From 3b18112cff9cf0bf5f734ebfd943118b28c43488 Mon Sep 17 00:00:00 2001 From: muralikrishnan Date: Fri, 24 Feb 2023 23:07:18 +0530 Subject: [PATCH 01/14] unbounce access request test --- data/saas/config/unbounce_config.yml | 51 +++++ data/saas/dataset/unbounce_dataset.yml | 177 ++++++++++++++++++ data/saas/saas_connector_registry.toml | 6 + tests/ops/fixtures/saas/unbounce_fixtures.py | 151 +++++++++++++++ .../saas/test_unbounce_task.py | 86 +++++++++ 5 files changed, 471 insertions(+) create mode 100644 data/saas/config/unbounce_config.yml create mode 100644 data/saas/dataset/unbounce_dataset.yml create mode 100644 tests/ops/fixtures/saas/unbounce_fixtures.py create mode 100644 tests/ops/integration_tests/saas/test_unbounce_task.py diff --git a/data/saas/config/unbounce_config.yml b/data/saas/config/unbounce_config.yml new file mode 100644 index 0000000000..b000b5ab39 --- /dev/null +++ b/data/saas/config/unbounce_config.yml @@ -0,0 +1,51 @@ +saas_config: + fides_key: + name: Unbounce SaaS Config + type: unbounce + description: A sample schema representing the Unbounce connector for Fides + version: 0.1.0 + + connector_params: + - name: domain + - name: api_key + - name: page_id + + client_config: + protocol: https + host: + authentication: + strategy: basic + configuration: + username: + password: '' + + test_request: + method: GET + path: /pages + headers: + - name: Accept + value: '*/*' + + endpoints: + - name: lead + requests: + read: + method: GET + path: /leads/ + param_values: + - name: all_leads_id + references: + - dataset: + field: all_leads.id + direction: from + - name: all_leads + requests: + read: + method: GET + path: /pages//leads + data_path: leads + param_values: + - name: page_id + connector_param: page_id + - name: placeholder + identity: email diff --git a/data/saas/dataset/unbounce_dataset.yml b/data/saas/dataset/unbounce_dataset.yml new file mode 100644 index 0000000000..e34991ab97 --- /dev/null +++ b/data/saas/dataset/unbounce_dataset.yml @@ -0,0 +1,177 @@ +dataset: + - fides_key: + name: unbounce + description: A sample dataset representing the Unbounce connector for Fides + collections: + - name: lead + fields: + - name: created_at + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: extra_data + fidesops_meta: + data_type: object + fields: + - name: cookies + fidesops_meta: + data_type: object + fields: + - name: ubvs + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: ubvt + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: form_data + fidesops_meta: + data_type: object + fields: + - name: email + data_categories: [system.operations] + fidesops_meta: + data_type: 'string[]' + - name: last_name + data_categories: [system.operations] + fidesops_meta: + data_type: 'string[]' + - name: first_name + data_categories: [system.operations] + fidesops_meta: + data_type: 'string[]' + - name: phone_number + data_categories: [system.operations] + fidesops_meta: + data_type: 'string[]' + - name: type_of_appointment + data_categories: [system.operations] + fidesops_meta: + data_type: 'string[]' + - name: page_id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: variant_id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: submitter_ip + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: metadata + fidesops_meta: + data_type: object + fields: + - name: documentation + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: location + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: related + fidesops_meta: + data_type: object + fields: + - name: page + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: sub_account + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: all_leads + fields: + - name: created_at + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: extra_data + fidesops_meta: + data_type: object + fields: + - name: cookies + fidesops_meta: + data_type: object + fields: + - name: ubvs + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: ubvt + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: form_data + fidesops_meta: + data_type: object + fields: + - name: email + data_categories: [system.operations] + fidesops_meta: + data_type: 'string[]' + - name: last_name + data_categories: [system.operations] + fidesops_meta: + data_type: 'string[]' + - name: first_name + data_categories: [system.operations] + fidesops_meta: + data_type: 'string[]' + - name: phone_number + data_categories: [system.operations] + fidesops_meta: + data_type: 'string[]' + - name: type_of_appointment + data_categories: [system.operations] + fidesops_meta: + data_type: 'string[]' + - name: page_id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: variant_id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: submitter_ip + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: metadata + fidesops_meta: + data_type: object + fields: + - name: documentation + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: location + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: related + fidesops_meta: + data_type: object + fields: + - name: page + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: sub_account + data_categories: [system.operations] + fidesops_meta: + data_type: string diff --git a/data/saas/saas_connector_registry.toml b/data/saas/saas_connector_registry.toml index b9936f41f0..5e4162fbf1 100644 --- a/data/saas/saas_connector_registry.toml +++ b/data/saas/saas_connector_registry.toml @@ -183,3 +183,9 @@ config = "data/saas/config/kustomer_config.yml" dataset = "data/saas/dataset/kustomer_dataset.yml" icon = "data/saas/icon/default.svg" human_readable = "Kustomer" + +[unbounce] +config = "data/saas/config/unbounce_config.yml" +dataset = "data/saas/dataset/unbounce_dataset.yml" +icon = "data/saas/icon/default.svg" +human_readable = "Unbounce" diff --git a/tests/ops/fixtures/saas/unbounce_fixtures.py b/tests/ops/fixtures/saas/unbounce_fixtures.py new file mode 100644 index 0000000000..551ef656f9 --- /dev/null +++ b/tests/ops/fixtures/saas/unbounce_fixtures.py @@ -0,0 +1,151 @@ +from time import sleep +from typing import Any, Dict, Generator + +import pydash +import pytest +import requests +from sqlalchemy.orm import Session + +from fides.api.ctl.sql_models import Dataset as CtlDataset +from fides.api.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fides.api.ops.models.datasetconfig import DatasetConfig +from fides.api.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) +from fides.lib.cryptography import cryptographic_util +from tests.ops.test_helpers.vault_client import get_secrets + +secrets = get_secrets("unbounce") + + +@pytest.fixture(scope="session") +def unbounce_secrets(saas_config): + return { + "domain": pydash.get(saas_config, "unbounce.domain") or secrets["domain"], + "page_id": pydash.get(saas_config, "unbounce.page_id") or secrets["page_id"], + "api_key": pydash.get(saas_config, "unbounce.api_key") or secrets["api_key"], + } + + +@pytest.fixture(scope="session") +def unbounce_identity_email(saas_config): + return ( + pydash.get(saas_config, "unbounce.identity_email") or secrets["identity_email"] + ) + + +@pytest.fixture(scope="function") +def unbounce_erasure_identity_email() -> str: + return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" + + +@pytest.fixture +def unbounce_config() -> Dict[str, Any]: + return load_config_with_replacement( + "data/saas/config/unbounce_config.yml", + "", + "unbounce_instance", + ) + + +@pytest.fixture +def unbounce_dataset() -> Dict[str, Any]: + return load_dataset_with_replacement( + "data/saas/dataset/unbounce_dataset.yml", + "", + "unbounce_instance", + )[0] + + +@pytest.fixture(scope="function") +def unbounce_connection_config( + db: Session, unbounce_config, unbounce_secrets +) -> Generator: + fides_key = unbounce_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": unbounce_secrets, + "saas_config": unbounce_config, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture +def unbounce_dataset_config( + db: Session, + unbounce_connection_config: ConnectionConfig, + unbounce_dataset: Dict[str, Any], +) -> Generator: + fides_key = unbounce_dataset["fides_key"] + unbounce_connection_config.name = fides_key + unbounce_connection_config.key = fides_key + unbounce_connection_config.save(db=db) + + ctl_dataset = CtlDataset.create_from_dataset_dict(db, unbounce_dataset) + + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": unbounce_connection_config.id, + "fides_key": fides_key, + "ctl_dataset_id": ctl_dataset.id, + }, + ) + yield dataset + dataset.delete(db=db) + ctl_dataset.delete(db=db) + + +@pytest.fixture(scope="function") +def unbounce_create_erasure_data( + unbounce_connection_config: ConnectionConfig, unbounce_erasure_identity_email: str +) -> None: + + sleep(60) + + unbounce_secrets = unbounce_connection_config.secrets + auth = unbounce_secrets["username"], unbounce_secrets["api_key"] + base_url = f"https://{unbounce_secrets['domain']}" + + # user + body = { + "user": { + "name": "Ethyca Test Erasure", + "email": unbounce_erasure_identity_email, + "verified": "true", + } + } + + users_response = requests.post(url=f"{base_url}/api/v2/users", auth=auth, json=body) + user = users_response.json()["user"] + user_id = user["id"] + + # ticket + ticket_data = { + "ticket": { + "comment": {"body": "Test Comment"}, + "priority": "urgent", + "subject": "Test Ticket", + "requester_id": user_id, + "submitter_id": user_id, + "description": "Test Description", + } + } + response = requests.post( + url=f"{base_url}/api/v2/tickets", auth=auth, json=ticket_data + ) + ticket = response.json()["ticket"] + ticket_id = ticket["id"] + yield ticket, user diff --git a/tests/ops/integration_tests/saas/test_unbounce_task.py b/tests/ops/integration_tests/saas/test_unbounce_task.py new file mode 100644 index 0000000000..82bb7834ad --- /dev/null +++ b/tests/ops/integration_tests/saas/test_unbounce_task.py @@ -0,0 +1,86 @@ +import random +import time + +import pytest +import requests + +from fides.api.ops.graph.graph import DatasetGraph +from fides.api.ops.models.privacy_request import PrivacyRequest +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.service.connectors import get_connector +from fides.api.ops.task import graph_task +from fides.api.ops.task.graph_task import get_cached_data_for_erasures +from fides.core.config import get_config +from tests.ops.graph.graph_test_util import assert_rows_match + +CONFIG = get_config() + + +@pytest.mark.integration_saas +@pytest.mark.integration_unbounce +def test_unbounce_connection_test(unbounce_connection_config) -> None: + get_connector(unbounce_connection_config).test_connection() + +@pytest.mark.integration_saas +@pytest.mark.integration_unbounce +@pytest.mark.asyncio +async def test_unbounce_access_request_task( + db, + policy, + unbounce_connection_config, + unbounce_dataset_config, + unbounce_identity_email, +) -> None: + """Full access request based on the unbounce SaaS config""" + + privacy_request = PrivacyRequest( + id=f"test_unbounce_access_request_task_{random.randint(0, 1000)}" + ) + identity = Identity(**{"email": unbounce_identity_email}) + privacy_request.cache_identity(identity) + + dataset_name = unbounce_connection_config.get_saas_config().fides_key + merged_graph = unbounce_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + v = await graph_task.run_access_request( + privacy_request, + policy, + graph, + [unbounce_connection_config], + {"email": unbounce_identity_email}, + db, + ) + + assert_rows_match( + v[f"{dataset_name}:all_leads"], + min_size=1, + keys=[ + "created_at", + "id", + "extra_data", + "form_data", + "page_id", + "variant_id", + "metadata", + ], + ) + + assert_rows_match( + v[f"{dataset_name}:lead"], + min_size=1, + keys=[ + "created_at", + "id", + "extra_data", + "form_data", + "page_id", + "variant_id", + "metadata", + ], + ) + + # verify we only returned data for our identity email + unbounce_secrets = unbounce_connection_config.secrets + for lead_list in v[f"{dataset_name}:lead"]: + assert lead_list["page_id"] == f"{unbounce_secrets['page_id']}" \ No newline at end of file From 72396451678a7ad80ecb27d6ddec58d7de0789e5 Mon Sep 17 00:00:00 2001 From: muralikrishnan Date: Fri, 3 Mar 2023 19:21:59 +0530 Subject: [PATCH 02/14] access request with lead id --- data/saas/config/unbounce_config.yml | 26 +--- data/saas/dataset/unbounce_dataset.yml | 123 +----------------- tests/ops/fixtures/saas/unbounce_fixtures.py | 2 +- .../saas/test_unbounce_task.py | 31 ++--- 4 files changed, 31 insertions(+), 151 deletions(-) diff --git a/data/saas/config/unbounce_config.yml b/data/saas/config/unbounce_config.yml index b000b5ab39..84d0795444 100644 --- a/data/saas/config/unbounce_config.yml +++ b/data/saas/config/unbounce_config.yml @@ -1,14 +1,14 @@ saas_config: fides_key: - name: Unbounce SaaS Config - type: unbounce - description: A sample schema representing the Unbounce connector for Fides + name: Unbounce_new SaaS Config + type: unbounce_new + description: A sample schema representing the Unbounce_new connector for Fides version: 0.1.0 connector_params: - name: domain - name: api_key - - name: page_id + - name: lead_id client_config: protocol: https @@ -31,21 +31,9 @@ saas_config: requests: read: method: GET - path: /leads/ + path: /leads/ param_values: - - name: all_leads_id - references: - - dataset: - field: all_leads.id - direction: from - - name: all_leads - requests: - read: - method: GET - path: /pages//leads - data_path: leads - param_values: - - name: page_id - connector_param: page_id + - name: lead_id + connector_param: lead_id - name: placeholder identity: email diff --git a/data/saas/dataset/unbounce_dataset.yml b/data/saas/dataset/unbounce_dataset.yml index e34991ab97..8943558a3b 100644 --- a/data/saas/dataset/unbounce_dataset.yml +++ b/data/saas/dataset/unbounce_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: - name: unbounce - description: A sample dataset representing the Unbounce connector for Fides + name: unbounce_new + description: A sample dataset representing the Unbounce_new connector for Fides collections: - name: lead fields: @@ -18,127 +18,22 @@ dataset: data_type: object fields: - name: cookies - fidesops_meta: - data_type: object - fields: - - name: ubvs - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: ubvt - data_categories: [system.operations] - fidesops_meta: - data_type: string - name: form_data fidesops_meta: data_type: object fields: - name: email - data_categories: [system.operations] - fidesops_meta: - data_type: 'string[]' - - name: last_name - data_categories: [system.operations] - fidesops_meta: - data_type: 'string[]' - - name: first_name - data_categories: [system.operations] - fidesops_meta: - data_type: 'string[]' - - name: phone_number - data_categories: [system.operations] - fidesops_meta: - data_type: 'string[]' - - name: type_of_appointment - data_categories: [system.operations] - fidesops_meta: - data_type: 'string[]' - - name: page_id - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: variant_id - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: submitter_ip - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: metadata - fidesops_meta: - data_type: object - fields: - - name: documentation - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: location - data_categories: [system.operations] + data_categories: [user.contact.email] fidesops_meta: data_type: string - - name: related - fidesops_meta: - data_type: object - fields: - - name: page - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: sub_account - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: all_leads - fields: - - name: created_at - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: id - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: extra_data - fidesops_meta: - data_type: object - fields: - - name: cookies - fidesops_meta: - data_type: object - fields: - - name: ubvs - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: ubvt - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: form_data - fidesops_meta: - data_type: object - fields: - - name: email - data_categories: [system.operations] - fidesops_meta: - data_type: 'string[]' - name: last_name - data_categories: [system.operations] + data_categories: [user.name] fidesops_meta: - data_type: 'string[]' + data_type: string - name: first_name - data_categories: [system.operations] + data_categories: [user.name] fidesops_meta: - data_type: 'string[]' - - name: phone_number - data_categories: [system.operations] - fidesops_meta: - data_type: 'string[]' - - name: type_of_appointment - data_categories: [system.operations] - fidesops_meta: - data_type: 'string[]' + data_type: string - name: page_id data_categories: [system.operations] fidesops_meta: @@ -147,10 +42,6 @@ dataset: data_categories: [system.operations] fidesops_meta: data_type: string - - name: submitter_ip - data_categories: [system.operations] - fidesops_meta: - data_type: string - name: metadata fidesops_meta: data_type: object diff --git a/tests/ops/fixtures/saas/unbounce_fixtures.py b/tests/ops/fixtures/saas/unbounce_fixtures.py index 551ef656f9..6b3eac19f3 100644 --- a/tests/ops/fixtures/saas/unbounce_fixtures.py +++ b/tests/ops/fixtures/saas/unbounce_fixtures.py @@ -27,7 +27,7 @@ def unbounce_secrets(saas_config): return { "domain": pydash.get(saas_config, "unbounce.domain") or secrets["domain"], - "page_id": pydash.get(saas_config, "unbounce.page_id") or secrets["page_id"], + "lead_id": pydash.get(saas_config, "unbounce.lead_id") or secrets["lead_id"], "api_key": pydash.get(saas_config, "unbounce.api_key") or secrets["api_key"], } diff --git a/tests/ops/integration_tests/saas/test_unbounce_task.py b/tests/ops/integration_tests/saas/test_unbounce_task.py index 82bb7834ad..13fd4606e4 100644 --- a/tests/ops/integration_tests/saas/test_unbounce_task.py +++ b/tests/ops/integration_tests/saas/test_unbounce_task.py @@ -52,19 +52,19 @@ async def test_unbounce_access_request_task( db, ) - assert_rows_match( - v[f"{dataset_name}:all_leads"], - min_size=1, - keys=[ - "created_at", - "id", - "extra_data", - "form_data", - "page_id", - "variant_id", - "metadata", - ], - ) + # assert_rows_match( + # v[f"{dataset_name}:all_leads"], + # min_size=1, + # keys=[ + # "created_at", + # "id", + # "extra_data", + # "form_data", + # "page_id", + # "variant_id", + # "metadata", + # ], + # ) assert_rows_match( v[f"{dataset_name}:lead"], @@ -82,5 +82,6 @@ async def test_unbounce_access_request_task( # verify we only returned data for our identity email unbounce_secrets = unbounce_connection_config.secrets - for lead_list in v[f"{dataset_name}:lead"]: - assert lead_list["page_id"] == f"{unbounce_secrets['page_id']}" \ No newline at end of file + # for lead_list in v[f"{dataset_name}:lead"]: + # assert lead_list["page_id"] == f"{unbounce_secrets['page_id']}" + assert v[f"{dataset_name}:lead"][0]["id"] == f"{unbounce_secrets['lead_id']}" \ No newline at end of file From c15f72c3c1b1f6e92b5d174f287cbc08501677d1 Mon Sep 17 00:00:00 2001 From: Kelsey Thomas <101993653+Kelsey-Ethyca@users.noreply.github.com> Date: Mon, 6 Mar 2023 19:57:06 -0800 Subject: [PATCH 03/14] add unbounce icon --- .../public/images/connector-logos/unbounce.svg | 11 +++++++++++ .../src/features/datastore-connections/constants.ts | 2 ++ data/saas/icon/unbounce.svg | 11 +++++++++++ data/saas/saas_connector_registry.toml | 2 +- 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 clients/admin-ui/public/images/connector-logos/unbounce.svg create mode 100644 data/saas/icon/unbounce.svg diff --git a/clients/admin-ui/public/images/connector-logos/unbounce.svg b/clients/admin-ui/public/images/connector-logos/unbounce.svg new file mode 100644 index 0000000000..d6189a3c12 --- /dev/null +++ b/clients/admin-ui/public/images/connector-logos/unbounce.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/clients/admin-ui/src/features/datastore-connections/constants.ts b/clients/admin-ui/src/features/datastore-connections/constants.ts index 6473a8ab84..020df02227 100644 --- a/clients/admin-ui/src/features/datastore-connections/constants.ts +++ b/clients/admin-ui/src/features/datastore-connections/constants.ts @@ -48,6 +48,7 @@ export enum SaasType { SQUARE = "square", STRIPE = "stripe", TWILIO_CONVERSATIONS = "twilio_conversations", + UNBOUNCE = "unbounce", UNIVERSAL_ANALYTICS = "universal_analytics", VEND = "vend", WUNDERKIND = "wunderkind", @@ -113,6 +114,7 @@ export const CONNECTION_TYPE_LOGO_MAP = new Map< [SaasType.SQUARE, "square.svg"], [SaasType.STRIPE, "stripe.svg"], [SaasType.TWILIO_CONVERSATIONS, "twilio.svg"], + [SaasType.UNBOUNCE, "unbounce.svg"], [SaasType.UNIVERSAL_ANALYTICS, "google_analytics.svg"], [SaasType.VEND, "vend.svg"], [SaasType.WUNDERKIND, "wunderkind.svg"], diff --git a/data/saas/icon/unbounce.svg b/data/saas/icon/unbounce.svg new file mode 100644 index 0000000000..d6189a3c12 --- /dev/null +++ b/data/saas/icon/unbounce.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/data/saas/saas_connector_registry.toml b/data/saas/saas_connector_registry.toml index 04287b2fe3..3846306dc4 100644 --- a/data/saas/saas_connector_registry.toml +++ b/data/saas/saas_connector_registry.toml @@ -199,7 +199,7 @@ human_readable = "Kustomer" [unbounce] config = "data/saas/config/unbounce_config.yml" dataset = "data/saas/dataset/unbounce_dataset.yml" -icon = "data/saas/icon/default.svg" +icon = "data/saas/icon/unbounce.svg" human_readable = "Unbounce" [yotpo_reviews] From f5f6ed52df9d66f5a0ea386ffae59f0663e78075 Mon Sep 17 00:00:00 2001 From: vivek Date: Tue, 7 Mar 2023 14:16:25 +0530 Subject: [PATCH 04/14] changed the name node in the config and dataset file --- data/saas/config/unbounce_config.yml | 6 +++--- data/saas/dataset/unbounce_dataset.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/data/saas/config/unbounce_config.yml b/data/saas/config/unbounce_config.yml index 84d0795444..6e3d8081fb 100644 --- a/data/saas/config/unbounce_config.yml +++ b/data/saas/config/unbounce_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Unbounce_new SaaS Config - type: unbounce_new - description: A sample schema representing the Unbounce_new connector for Fides + name: Unbounce SaaS Config + type: unbounce + description: A sample schema representing the Unbounce connector for Fides version: 0.1.0 connector_params: diff --git a/data/saas/dataset/unbounce_dataset.yml b/data/saas/dataset/unbounce_dataset.yml index 8943558a3b..82a353efa9 100644 --- a/data/saas/dataset/unbounce_dataset.yml +++ b/data/saas/dataset/unbounce_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: - name: unbounce_new - description: A sample dataset representing the Unbounce_new connector for Fides + name: unbounce + description: A sample dataset representing the Unbounce connector for Fides collections: - name: lead fields: From e5cba8bbf78c7b478fc4d0fcaeb541ddaa6c6083 Mon Sep 17 00:00:00 2001 From: Kelsey Thomas <101993653+Kelsey-Ethyca@users.noreply.github.com> Date: Mon, 13 Mar 2023 10:40:12 -0500 Subject: [PATCH 05/14] add default domain --- data/saas/config/unbounce_config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/data/saas/config/unbounce_config.yml b/data/saas/config/unbounce_config.yml index 6e3d8081fb..4a86e17e14 100644 --- a/data/saas/config/unbounce_config.yml +++ b/data/saas/config/unbounce_config.yml @@ -7,6 +7,7 @@ saas_config: connector_params: - name: domain + default_value: api.unbounce.com - name: api_key - name: lead_id @@ -17,14 +18,14 @@ saas_config: strategy: basic configuration: username: - password: '' + password: "" test_request: method: GET path: /pages headers: - name: Accept - value: '*/*' + value: "*/*" endpoints: - name: lead From 69eee9da4a9eddf516c6f3dfc632e8aef22bbd5b Mon Sep 17 00:00:00 2001 From: muralikrishnan Date: Fri, 24 Mar 2023 17:37:43 +0530 Subject: [PATCH 06/14] removed unwanted imports --- .../saas/test_unbounce_task.py | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/tests/ops/integration_tests/saas/test_unbounce_task.py b/tests/ops/integration_tests/saas/test_unbounce_task.py index 13fd4606e4..cf16bb9ce7 100644 --- a/tests/ops/integration_tests/saas/test_unbounce_task.py +++ b/tests/ops/integration_tests/saas/test_unbounce_task.py @@ -1,15 +1,12 @@ import random -import time import pytest -import requests from fides.api.ops.graph.graph import DatasetGraph from fides.api.ops.models.privacy_request import PrivacyRequest from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors import get_connector from fides.api.ops.task import graph_task -from fides.api.ops.task.graph_task import get_cached_data_for_erasures from fides.core.config import get_config from tests.ops.graph.graph_test_util import assert_rows_match @@ -31,7 +28,7 @@ async def test_unbounce_access_request_task( unbounce_dataset_config, unbounce_identity_email, ) -> None: - """Full access request based on the unbounce SaaS config""" + """Full access request based on the Unbounce SaaS config""" privacy_request = PrivacyRequest( id=f"test_unbounce_access_request_task_{random.randint(0, 1000)}" @@ -52,20 +49,6 @@ async def test_unbounce_access_request_task( db, ) - # assert_rows_match( - # v[f"{dataset_name}:all_leads"], - # min_size=1, - # keys=[ - # "created_at", - # "id", - # "extra_data", - # "form_data", - # "page_id", - # "variant_id", - # "metadata", - # ], - # ) - assert_rows_match( v[f"{dataset_name}:lead"], min_size=1, @@ -82,6 +65,4 @@ async def test_unbounce_access_request_task( # verify we only returned data for our identity email unbounce_secrets = unbounce_connection_config.secrets - # for lead_list in v[f"{dataset_name}:lead"]: - # assert lead_list["page_id"] == f"{unbounce_secrets['page_id']}" assert v[f"{dataset_name}:lead"][0]["id"] == f"{unbounce_secrets['lead_id']}" \ No newline at end of file From 4998194c4b30bd6beab4e027d51460188d8032e0 Mon Sep 17 00:00:00 2001 From: Soundarya Date: Wed, 29 Mar 2023 20:06:05 +0530 Subject: [PATCH 07/14] New changes in unbounce dataset, config and fixtures --- data/saas/config/unbounce_config.yml | 2 +- data/saas/dataset/unbounce_dataset.yml | 2 +- tests/fixtures/saas/unbounce_fixtures.py | 41 ------------------------ 3 files changed, 2 insertions(+), 43 deletions(-) diff --git a/data/saas/config/unbounce_config.yml b/data/saas/config/unbounce_config.yml index 4a86e17e14..fb8e8ffb18 100644 --- a/data/saas/config/unbounce_config.yml +++ b/data/saas/config/unbounce_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Unbounce SaaS Config + name: Unbounce type: unbounce description: A sample schema representing the Unbounce connector for Fides version: 0.1.0 diff --git a/data/saas/dataset/unbounce_dataset.yml b/data/saas/dataset/unbounce_dataset.yml index 82a353efa9..fe14f01477 100644 --- a/data/saas/dataset/unbounce_dataset.yml +++ b/data/saas/dataset/unbounce_dataset.yml @@ -1,6 +1,6 @@ dataset: - fides_key: - name: unbounce + name: Unbounce Dataset description: A sample dataset representing the Unbounce connector for Fides collections: - name: lead diff --git a/tests/fixtures/saas/unbounce_fixtures.py b/tests/fixtures/saas/unbounce_fixtures.py index 6b3eac19f3..57921b1576 100644 --- a/tests/fixtures/saas/unbounce_fixtures.py +++ b/tests/fixtures/saas/unbounce_fixtures.py @@ -108,44 +108,3 @@ def unbounce_dataset_config( ctl_dataset.delete(db=db) -@pytest.fixture(scope="function") -def unbounce_create_erasure_data( - unbounce_connection_config: ConnectionConfig, unbounce_erasure_identity_email: str -) -> None: - - sleep(60) - - unbounce_secrets = unbounce_connection_config.secrets - auth = unbounce_secrets["username"], unbounce_secrets["api_key"] - base_url = f"https://{unbounce_secrets['domain']}" - - # user - body = { - "user": { - "name": "Ethyca Test Erasure", - "email": unbounce_erasure_identity_email, - "verified": "true", - } - } - - users_response = requests.post(url=f"{base_url}/api/v2/users", auth=auth, json=body) - user = users_response.json()["user"] - user_id = user["id"] - - # ticket - ticket_data = { - "ticket": { - "comment": {"body": "Test Comment"}, - "priority": "urgent", - "subject": "Test Ticket", - "requester_id": user_id, - "submitter_id": user_id, - "description": "Test Description", - } - } - response = requests.post( - url=f"{base_url}/api/v2/tickets", auth=auth, json=ticket_data - ) - ticket = response.json()["ticket"] - ticket_id = ticket["id"] - yield ticket, user From 05485b712bfdfda705398fc00f72cbe60a1c636a Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 12 Apr 2023 16:33:34 -0700 Subject: [PATCH 08/14] Removing icon from admin-ui --- .../public/images/connector-logos/unbounce.svg | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 clients/admin-ui/public/images/connector-logos/unbounce.svg diff --git a/clients/admin-ui/public/images/connector-logos/unbounce.svg b/clients/admin-ui/public/images/connector-logos/unbounce.svg deleted file mode 100644 index d6189a3c12..0000000000 --- a/clients/admin-ui/public/images/connector-logos/unbounce.svg +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - From 4a4e2f3803697cea04b96709b11f4bcd99bf7635 Mon Sep 17 00:00:00 2001 From: muralikrishnan Date: Mon, 17 Apr 2023 19:00:37 +0530 Subject: [PATCH 09/14] unbounce access request --- data/saas/config/unbounce_config.yml | 37 +++++++-- data/saas/dataset/unbounce_dataset.yml | 78 ++++++++++++++++++- tests/fixtures/saas/unbounce_fixtures.py | 1 - .../saas/test_unbounce_task.py | 26 ++++++- 4 files changed, 128 insertions(+), 14 deletions(-) diff --git a/data/saas/config/unbounce_config.yml b/data/saas/config/unbounce_config.yml index fb8e8ffb18..d59a6e7c8f 100644 --- a/data/saas/config/unbounce_config.yml +++ b/data/saas/config/unbounce_config.yml @@ -2,14 +2,13 @@ saas_config: fides_key: name: Unbounce type: unbounce - description: A sample schema representing the Unbounce connector for Fides + description: A sample schema representing the Unbounce_latest connector for Fides version: 0.1.0 connector_params: - name: domain default_value: api.unbounce.com - name: api_key - - name: lead_id client_config: protocol: https @@ -18,23 +17,47 @@ saas_config: strategy: basic configuration: username: - password: "" + password: '12' test_request: method: GET path: /pages headers: - name: Accept - value: "*/*" + value: '*/*' endpoints: - name: lead requests: read: method: GET - path: /leads/ + path: /pages//leads/ + data_path: leads + postprocessors: + - strategy: filter + configuration: + field: form_data.email + value: + identity: email + exact: false + case_sensitive: false param_values: - - name: lead_id - connector_param: lead_id + - name: pages_id + references: + - dataset: + field: pages.id + direction: from - name: placeholder identity: email + - name: pages + requests: + read: + method: GET + path: /pages + headers: + - name: Accept + value: '*/*' + data_path: pages + param_values: + - name: placeholder + identity: email \ No newline at end of file diff --git a/data/saas/dataset/unbounce_dataset.yml b/data/saas/dataset/unbounce_dataset.yml index fe14f01477..ae98da928e 100644 --- a/data/saas/dataset/unbounce_dataset.yml +++ b/data/saas/dataset/unbounce_dataset.yml @@ -1,6 +1,6 @@ dataset: - fides_key: - name: Unbounce Dataset + name: unbounce description: A sample dataset representing the Unbounce connector for Fides collections: - name: lead @@ -10,7 +10,7 @@ dataset: fidesops_meta: data_type: string - name: id - data_categories: [system.operations] + data_categories: [user.unique_id] fidesops_meta: data_type: string - name: extra_data @@ -27,7 +27,7 @@ dataset: fidesops_meta: data_type: string - name: last_name - data_categories: [user.name] + data_categories: [system.operations] fidesops_meta: data_type: string - name: first_name @@ -66,3 +66,75 @@ dataset: data_categories: [system.operations] fidesops_meta: data_type: string + - name: pages + fields: + - name: subAccountId + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: integrations + - name: integrationsCount + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: integrationsErrorsCount + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: url + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: metadata + fidesops_meta: + data_type: object + fields: + - name: documentation + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: location + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: related + fidesops_meta: + data_type: object + fields: + - name: leads + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: subAccount + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: formFields + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: createdAt + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: name + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: state + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: lastPublishedAt + - name: variantsCount + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: domain + data_categories: [system.operations] + fidesops_meta: + data_type: string \ No newline at end of file diff --git a/tests/fixtures/saas/unbounce_fixtures.py b/tests/fixtures/saas/unbounce_fixtures.py index 57921b1576..178f451d04 100644 --- a/tests/fixtures/saas/unbounce_fixtures.py +++ b/tests/fixtures/saas/unbounce_fixtures.py @@ -27,7 +27,6 @@ def unbounce_secrets(saas_config): return { "domain": pydash.get(saas_config, "unbounce.domain") or secrets["domain"], - "lead_id": pydash.get(saas_config, "unbounce.lead_id") or secrets["lead_id"], "api_key": pydash.get(saas_config, "unbounce.api_key") or secrets["api_key"], } diff --git a/tests/ops/integration_tests/saas/test_unbounce_task.py b/tests/ops/integration_tests/saas/test_unbounce_task.py index cf16bb9ce7..5cba538ed7 100644 --- a/tests/ops/integration_tests/saas/test_unbounce_task.py +++ b/tests/ops/integration_tests/saas/test_unbounce_task.py @@ -49,6 +49,26 @@ async def test_unbounce_access_request_task( db, ) + assert_rows_match( + v[f"{dataset_name}:pages"], + min_size=1, + keys=[ + "subAccountId", + "integrations", + "integrationsCount", + "integrationsErrorsCount", + "id", + "url", + "metadata", + "createdAt", + "name", + "state", + "lastPublishedAt", + "variantsCount", + "domain" + ], + ) + assert_rows_match( v[f"{dataset_name}:lead"], min_size=1, @@ -59,10 +79,10 @@ async def test_unbounce_access_request_task( "form_data", "page_id", "variant_id", - "metadata", + "metadata" ], ) # verify we only returned data for our identity email - unbounce_secrets = unbounce_connection_config.secrets - assert v[f"{dataset_name}:lead"][0]["id"] == f"{unbounce_secrets['lead_id']}" \ No newline at end of file + for leads in v[f"{dataset_name}:lead"]: + assert leads["form_data"]['email'] == unbounce_identity_email \ No newline at end of file From 96a5759a0c793ec70425414082f4e7a1ed9a89f2 Mon Sep 17 00:00:00 2001 From: muralikrishnan Date: Tue, 18 Apr 2023 18:31:26 +0530 Subject: [PATCH 10/14] data categories --- data/saas/dataset/unbounce_dataset.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/saas/dataset/unbounce_dataset.yml b/data/saas/dataset/unbounce_dataset.yml index ae98da928e..83a0b3afed 100644 --- a/data/saas/dataset/unbounce_dataset.yml +++ b/data/saas/dataset/unbounce_dataset.yml @@ -27,7 +27,7 @@ dataset: fidesops_meta: data_type: string - name: last_name - data_categories: [system.operations] + data_categories: [user.name] fidesops_meta: data_type: string - name: first_name From 95305a55c4194060520d8860827993f04b3f2b54 Mon Sep 17 00:00:00 2001 From: muralikrishnan Date: Tue, 18 Apr 2023 19:20:50 +0530 Subject: [PATCH 11/14] config changes --- data/saas/config/unbounce_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/saas/config/unbounce_config.yml b/data/saas/config/unbounce_config.yml index d59a6e7c8f..7bdf23488c 100644 --- a/data/saas/config/unbounce_config.yml +++ b/data/saas/config/unbounce_config.yml @@ -2,7 +2,7 @@ saas_config: fides_key: name: Unbounce type: unbounce - description: A sample schema representing the Unbounce_latest connector for Fides + description: A sample schema representing the Unbounce connector for Fides version: 0.1.0 connector_params: From 9059f81c1d3a01c3b22c7c1c24a021f556c08099 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 25 Apr 2023 13:24:19 -0700 Subject: [PATCH 12/14] Adding erasure support and OAuth2 authentication --- data/saas/config/unbounce_config.yml | 105 +++++++++++--- data/saas/dataset/unbounce_dataset.yml | 129 +++++++++--------- tests/fixtures/saas/unbounce_fixtures.py | 58 +++++++- .../saas/test_unbounce_task.py | 100 +++++++++++++- 4 files changed, 298 insertions(+), 94 deletions(-) diff --git a/data/saas/config/unbounce_config.yml b/data/saas/config/unbounce_config.yml index 7bdf23488c..f96d4f5f3b 100644 --- a/data/saas/config/unbounce_config.yml +++ b/data/saas/config/unbounce_config.yml @@ -8,30 +8,86 @@ saas_config: connector_params: - name: domain default_value: api.unbounce.com - - name: api_key + - name: client_id + label: Client ID + - name: client_secret + - name: redirect_uri + lable: Redirect URI client_config: protocol: https host: authentication: - strategy: basic + strategy: oauth2_authorization_code configuration: - username: - password: '12' + authorization_request: + method: GET + path: /oauth/authorize + query_params: + - name: client_id + value: + - name: redirect_uri + value: + - name: response_type + value: code + - name: state + value: + token_request: + method: POST + path: /oauth/token + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + body: | + { + "client_id": "", + "client_secret": "", + "grant_type": "authorization_code", + "code": "", + "redirect_uri": "" + } + refresh_request: + method: POST + path: /oauth/token + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + body: | + { + "client_id": "", + "client_secret": "", + "grant_type": "refresh_token", + "refresh_token": "" + } test_request: method: GET - path: /pages headers: - name: Accept - value: '*/*' + value: application/json + path: /pages endpoints: - - name: lead + - name: pages + requests: + read: + method: GET + path: /pages + headers: + - name: Accept + value: application/json + data_path: pages + param_values: + - name: placeholder + identity: email + - name: leads requests: read: method: GET - path: /pages//leads/ + path: /pages//leads/ + headers: + - name: Accept + value: application/json data_path: leads postprocessors: - strategy: filter @@ -42,22 +98,27 @@ saas_config: exact: false case_sensitive: false param_values: - - name: pages_id + - name: page_id references: - dataset: field: pages.id direction: from - - name: placeholder - identity: email - - name: pages - requests: - read: - method: GET - path: /pages - headers: - - name: Accept - value: '*/*' - data_path: pages + delete: + method: POST + path: /pages//lead_deletion_request + grouped_inputs: [page_id, lead_id] param_values: - - name: placeholder - identity: email \ No newline at end of file + - name: page_id + references: + - dataset: + field: leads.page_id + direction: from + - name: lead_id + references: + - dataset: + field: leads.id + direction: from + body: | + { + "lead_ids": [""] + } diff --git a/data/saas/dataset/unbounce_dataset.yml b/data/saas/dataset/unbounce_dataset.yml index 83a0b3afed..71fdbfa70a 100644 --- a/data/saas/dataset/unbounce_dataset.yml +++ b/data/saas/dataset/unbounce_dataset.yml @@ -3,69 +3,6 @@ dataset: name: unbounce description: A sample dataset representing the Unbounce connector for Fides collections: - - name: lead - fields: - - name: created_at - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: id - data_categories: [user.unique_id] - fidesops_meta: - data_type: string - - name: extra_data - fidesops_meta: - data_type: object - fields: - - name: cookies - - name: form_data - fidesops_meta: - data_type: object - fields: - - name: email - data_categories: [user.contact.email] - fidesops_meta: - data_type: string - - name: last_name - data_categories: [user.name] - fidesops_meta: - data_type: string - - name: first_name - data_categories: [user.name] - fidesops_meta: - data_type: string - - name: page_id - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: variant_id - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: metadata - fidesops_meta: - data_type: object - fields: - - name: documentation - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: location - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: related - fidesops_meta: - data_type: object - fields: - - name: page - data_categories: [system.operations] - fidesops_meta: - data_type: string - - name: sub_account - data_categories: [system.operations] - fidesops_meta: - data_type: string - name: pages fields: - name: subAccountId @@ -137,4 +74,68 @@ dataset: - name: domain data_categories: [system.operations] fidesops_meta: - data_type: string \ No newline at end of file + data_type: string + - name: leads + fields: + - name: created_at + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: id + data_categories: [user.unique_id] + fidesops_meta: + data_type: string + primary_key: True + - name: extra_data + fidesops_meta: + data_type: object + fields: + - name: cookies + - name: form_data + fidesops_meta: + data_type: object + fields: + - name: email + data_categories: [user.contact.email] + fidesops_meta: + data_type: string + - name: last_name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: first_name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: page_id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: variant_id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: metadata + fidesops_meta: + data_type: object + fields: + - name: documentation + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: location + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: related + fidesops_meta: + data_type: object + fields: + - name: page + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: sub_account + data_categories: [system.operations] + fidesops_meta: + data_type: string diff --git a/tests/fixtures/saas/unbounce_fixtures.py b/tests/fixtures/saas/unbounce_fixtures.py index 178f451d04..28a8600515 100644 --- a/tests/fixtures/saas/unbounce_fixtures.py +++ b/tests/fixtures/saas/unbounce_fixtures.py @@ -1,9 +1,9 @@ -from time import sleep -from typing import Any, Dict, Generator +from typing import Any, Dict, Generator, cast import pydash import pytest import requests +from requests import Response from sqlalchemy.orm import Session from fides.api.ctl.sql_models import Dataset as CtlDataset @@ -27,7 +27,15 @@ def unbounce_secrets(saas_config): return { "domain": pydash.get(saas_config, "unbounce.domain") or secrets["domain"], - "api_key": pydash.get(saas_config, "unbounce.api_key") or secrets["api_key"], + "client_id": pydash.get(saas_config, "unbounce.client_id") + or secrets["client_id"], + "client_secret": pydash.get(saas_config, "unbounce.client_secret") + or secrets["client_secret"], + "redirect_uri": pydash.get(saas_config, "unbounce.redirect_uri") + or secrets["redirect_uri"], + "access_token": pydash.get(saas_config, "unbounce.access_token") + or secrets["access_token"], + "page_id": pydash.get(saas_config, "unbounce.page_id") or secrets["page_id"], } @@ -107,3 +115,47 @@ def unbounce_dataset_config( ctl_dataset.delete(db=db) +class UnbounceTestClient: + def __init__(self, unbounce_connection_config: ConnectionConfig): + self.unbounce_secrets = cast(Dict, unbounce_connection_config.secrets) + self.headers = { + "Authorization": f"Bearer {self.unbounce_secrets['access_token']}" + } + self.page_id = self.unbounce_secrets["page_id"] + self.base_url = f"https://{self.unbounce_secrets['domain']}" + + def create_lead(self, email: str) -> Response: + body = { + "conversion": "True", + "form_submission": { + "variant_id": "a", + "form_data": { + "first_name": "Test", + "last_name": "User", + "email": email, + }, + }, + } + + return requests.post( + url=f"{self.base_url}/pages/{self.page_id}/leads", + headers=self.headers, + json=body, + ) + + +@pytest.fixture(scope="function") +def unbounce_test_client( + unbounce_connection_config: ConnectionConfig, +) -> Generator: + test_client = UnbounceTestClient(unbounce_connection_config) + yield test_client + + +@pytest.fixture(scope="function") +def unbounce_create_erasure_data( + unbounce_test_client, + unbounce_erasure_identity_email: str, +) -> None: + # create lead + unbounce_test_client.create_lead(unbounce_erasure_identity_email) diff --git a/tests/ops/integration_tests/saas/test_unbounce_task.py b/tests/ops/integration_tests/saas/test_unbounce_task.py index 5cba538ed7..c08718d31c 100644 --- a/tests/ops/integration_tests/saas/test_unbounce_task.py +++ b/tests/ops/integration_tests/saas/test_unbounce_task.py @@ -7,6 +7,7 @@ from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors import get_connector from fides.api.ops.task import graph_task +from fides.api.ops.task.graph_task import get_cached_data_for_erasures from fides.core.config import get_config from tests.ops.graph.graph_test_util import assert_rows_match @@ -18,6 +19,7 @@ def test_unbounce_connection_test(unbounce_connection_config) -> None: get_connector(unbounce_connection_config).test_connection() + @pytest.mark.integration_saas @pytest.mark.integration_unbounce @pytest.mark.asyncio @@ -65,12 +67,12 @@ async def test_unbounce_access_request_task( "state", "lastPublishedAt", "variantsCount", - "domain" + "domain", ], ) assert_rows_match( - v[f"{dataset_name}:lead"], + v[f"{dataset_name}:leads"], min_size=1, keys=[ "created_at", @@ -79,10 +81,98 @@ async def test_unbounce_access_request_task( "form_data", "page_id", "variant_id", - "metadata" + "metadata", ], ) # verify we only returned data for our identity email - for leads in v[f"{dataset_name}:lead"]: - assert leads["form_data"]['email'] == unbounce_identity_email \ No newline at end of file + for leads in v[f"{dataset_name}:leads"]: + assert unbounce_identity_email in leads["form_data"]["email"] + + +@pytest.mark.integration_saas +@pytest.mark.integration_unbounce +@pytest.mark.asyncio +async def test_unbounce_erasure_request_task( + db, + policy, + erasure_policy_string_rewrite, + unbounce_connection_config, + unbounce_dataset_config, + unbounce_erasure_identity_email, + unbounce_create_erasure_data, +) -> None: + """Full erasure request based on the Unbounce SaaS config""" + + masking_strict = CONFIG.execution.masking_strict + CONFIG.execution.masking_strict = False # Allow Delete + + privacy_request = PrivacyRequest( + id=f"test_unbounce_erasure_request_task_{random.randint(0, 1000)}" + ) + identity = Identity(**{"email": unbounce_erasure_identity_email}) + privacy_request.cache_identity(identity) + + dataset_name = unbounce_connection_config.get_saas_config().fides_key + merged_graph = unbounce_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + v = await graph_task.run_access_request( + privacy_request, + policy, + graph, + [unbounce_connection_config], + {"email": unbounce_erasure_identity_email}, + db, + ) + + assert_rows_match( + v[f"{dataset_name}:pages"], + min_size=1, + keys=[ + "subAccountId", + "integrations", + "integrationsCount", + "integrationsErrorsCount", + "id", + "url", + "metadata", + "createdAt", + "name", + "state", + "lastPublishedAt", + "variantsCount", + "domain", + ], + ) + + assert_rows_match( + v[f"{dataset_name}:leads"], + min_size=1, + keys=[ + "created_at", + "id", + "extra_data", + "form_data", + "page_id", + "variant_id", + "metadata", + ], + ) + + x = await graph_task.run_erasure( + privacy_request, + erasure_policy_string_rewrite, + graph, + [unbounce_connection_config], + {"email": unbounce_erasure_identity_email}, + get_cached_data_for_erasures(privacy_request.id), + db, + ) + + assert x == { + f"{dataset_name}:pages": 0, + f"{dataset_name}:leads": 1, + } + + CONFIG.execution.masking_strict = masking_strict From 539892f2bd8690b002e86bc3709c0c9595f0ff0d Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 25 Apr 2023 13:54:50 -0700 Subject: [PATCH 13/14] Disabling tests --- tests/ops/integration_tests/saas/test_unbounce_task.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/ops/integration_tests/saas/test_unbounce_task.py b/tests/ops/integration_tests/saas/test_unbounce_task.py index c08718d31c..f87bf95bda 100644 --- a/tests/ops/integration_tests/saas/test_unbounce_task.py +++ b/tests/ops/integration_tests/saas/test_unbounce_task.py @@ -14,12 +14,14 @@ CONFIG = get_config() +@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors") @pytest.mark.integration_saas @pytest.mark.integration_unbounce def test_unbounce_connection_test(unbounce_connection_config) -> None: get_connector(unbounce_connection_config).test_connection() +@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors") @pytest.mark.integration_saas @pytest.mark.integration_unbounce @pytest.mark.asyncio @@ -90,6 +92,7 @@ async def test_unbounce_access_request_task( assert unbounce_identity_email in leads["form_data"]["email"] +@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors") @pytest.mark.integration_saas @pytest.mark.integration_unbounce @pytest.mark.asyncio From 2163acf875d9898b441339469b51a75254dded4e Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 25 Apr 2023 13:56:27 -0700 Subject: [PATCH 14/14] Adding changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1e865cdb6..d5933b1256 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The types of changes are: - Custom fields table [#3097](https://github.com/ethyca/fides/pull/3097) - Endpoints to save the new-style Privacy Preferences with respect to a fides user device id [#3132](https://github.com/ethyca/fides/pull/3132) - Support `privacy_declaration` as a resource type for custom fields [#3149](https://github.com/ethyca/fides/pull/3149) +- Access and erasure support for Unbounce [#2697](https://github.com/ethyca/fides/pull/2697) ### Changed