From ea5975e5f7c043dbf26f584b30da0ede603eef9c Mon Sep 17 00:00:00 2001 From: tmuralikrishnan <120677902+tmuralikrishnan@users.noreply.github.com> Date: Wed, 26 Apr 2023 21:33:27 +0530 Subject: [PATCH] Unbounce Connector (#2697) Co-authored-by: Kelsey Thomas <101993653+Kelsey-Ethyca@users.noreply.github.com> Co-authored-by: vivek Co-authored-by: Soundarya Co-authored-by: Adrian Galvan --- CHANGELOG.md | 1 + data/saas/config/unbounce_config.yml | 124 ++++++++++++ data/saas/dataset/unbounce_dataset.yml | 141 ++++++++++++++ data/saas/icon/unbounce.svg | 11 ++ tests/fixtures/saas/unbounce_fixtures.py | 161 ++++++++++++++++ .../saas/test_unbounce_task.py | 181 ++++++++++++++++++ 6 files changed, 619 insertions(+) create mode 100644 data/saas/config/unbounce_config.yml create mode 100644 data/saas/dataset/unbounce_dataset.yml create mode 100644 data/saas/icon/unbounce.svg create mode 100644 tests/fixtures/saas/unbounce_fixtures.py create mode 100644 tests/ops/integration_tests/saas/test_unbounce_task.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e3b8635a01..12b7cc0c36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ The types of changes are: - Endpoints to save the new-style Privacy Preferences with respect to a fides user device id [#3132](https://github.com/ethyca/fides/pull/3132) - Support `privacy_declaration` as a resource type for custom fields [#3149](https://github.com/ethyca/fides/pull/3149) - Expose `id` field of embedded `privacy_declarations` on `system` API responses [#3157](https://github.com/ethyca/fides/pull/3157) +- Access and erasure support for Unbounce [#2697](https://github.com/ethyca/fides/pull/2697) ### Changed diff --git a/data/saas/config/unbounce_config.yml b/data/saas/config/unbounce_config.yml new file mode 100644 index 0000000000..f96d4f5f3b --- /dev/null +++ b/data/saas/config/unbounce_config.yml @@ -0,0 +1,124 @@ +saas_config: + fides_key: + name: Unbounce + type: unbounce + description: A sample schema representing the Unbounce connector for Fides + version: 0.1.0 + + connector_params: + - name: domain + default_value: api.unbounce.com + - name: client_id + label: Client ID + - name: client_secret + - name: redirect_uri + lable: Redirect URI + + client_config: + protocol: https + host: + authentication: + strategy: oauth2_authorization_code + configuration: + authorization_request: + method: GET + path: /oauth/authorize + query_params: + - name: client_id + value: + - name: redirect_uri + value: + - name: response_type + value: code + - name: state + value: + token_request: + method: POST + path: /oauth/token + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + body: | + { + "client_id": "", + "client_secret": "", + "grant_type": "authorization_code", + "code": "", + "redirect_uri": "" + } + refresh_request: + method: POST + path: /oauth/token + headers: + - name: Content-Type + value: application/x-www-form-urlencoded + body: | + { + "client_id": "", + "client_secret": "", + "grant_type": "refresh_token", + "refresh_token": "" + } + + test_request: + method: GET + headers: + - name: Accept + value: application/json + path: /pages + + endpoints: + - name: pages + requests: + read: + method: GET + path: /pages + headers: + - name: Accept + value: application/json + data_path: pages + param_values: + - name: placeholder + identity: email + - name: leads + requests: + read: + method: GET + path: /pages//leads/ + headers: + - name: Accept + value: application/json + data_path: leads + postprocessors: + - strategy: filter + configuration: + field: form_data.email + value: + identity: email + exact: false + case_sensitive: false + param_values: + - name: page_id + references: + - dataset: + field: pages.id + direction: from + delete: + method: POST + path: /pages//lead_deletion_request + grouped_inputs: [page_id, lead_id] + param_values: + - name: page_id + references: + - dataset: + field: leads.page_id + direction: from + - name: lead_id + references: + - dataset: + field: leads.id + direction: from + body: | + { + "lead_ids": [""] + } diff --git a/data/saas/dataset/unbounce_dataset.yml b/data/saas/dataset/unbounce_dataset.yml new file mode 100644 index 0000000000..71fdbfa70a --- /dev/null +++ b/data/saas/dataset/unbounce_dataset.yml @@ -0,0 +1,141 @@ +dataset: + - fides_key: + name: unbounce + description: A sample dataset representing the Unbounce connector for Fides + collections: + - name: pages + fields: + - name: subAccountId + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: integrations + - name: integrationsCount + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: integrationsErrorsCount + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: url + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: metadata + fidesops_meta: + data_type: object + fields: + - name: documentation + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: location + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: related + fidesops_meta: + data_type: object + fields: + - name: leads + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: subAccount + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: formFields + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: createdAt + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: name + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: state + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: lastPublishedAt + - name: variantsCount + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: domain + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: leads + fields: + - name: created_at + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: id + data_categories: [user.unique_id] + fidesops_meta: + data_type: string + primary_key: True + - name: extra_data + fidesops_meta: + data_type: object + fields: + - name: cookies + - name: form_data + fidesops_meta: + data_type: object + fields: + - name: email + data_categories: [user.contact.email] + fidesops_meta: + data_type: string + - name: last_name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: first_name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: page_id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: variant_id + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: metadata + fidesops_meta: + data_type: object + fields: + - name: documentation + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: location + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: related + fidesops_meta: + data_type: object + fields: + - name: page + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: sub_account + data_categories: [system.operations] + fidesops_meta: + data_type: string diff --git a/data/saas/icon/unbounce.svg b/data/saas/icon/unbounce.svg new file mode 100644 index 0000000000..d6189a3c12 --- /dev/null +++ b/data/saas/icon/unbounce.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/tests/fixtures/saas/unbounce_fixtures.py b/tests/fixtures/saas/unbounce_fixtures.py new file mode 100644 index 0000000000..28a8600515 --- /dev/null +++ b/tests/fixtures/saas/unbounce_fixtures.py @@ -0,0 +1,161 @@ +from typing import Any, Dict, Generator, cast + +import pydash +import pytest +import requests +from requests import Response +from sqlalchemy.orm import Session + +from fides.api.ctl.sql_models import Dataset as CtlDataset +from fides.api.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fides.api.ops.models.datasetconfig import DatasetConfig +from fides.api.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) +from fides.lib.cryptography import cryptographic_util +from tests.ops.test_helpers.vault_client import get_secrets + +secrets = get_secrets("unbounce") + + +@pytest.fixture(scope="session") +def unbounce_secrets(saas_config): + return { + "domain": pydash.get(saas_config, "unbounce.domain") or secrets["domain"], + "client_id": pydash.get(saas_config, "unbounce.client_id") + or secrets["client_id"], + "client_secret": pydash.get(saas_config, "unbounce.client_secret") + or secrets["client_secret"], + "redirect_uri": pydash.get(saas_config, "unbounce.redirect_uri") + or secrets["redirect_uri"], + "access_token": pydash.get(saas_config, "unbounce.access_token") + or secrets["access_token"], + "page_id": pydash.get(saas_config, "unbounce.page_id") or secrets["page_id"], + } + + +@pytest.fixture(scope="session") +def unbounce_identity_email(saas_config): + return ( + pydash.get(saas_config, "unbounce.identity_email") or secrets["identity_email"] + ) + + +@pytest.fixture(scope="function") +def unbounce_erasure_identity_email() -> str: + return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" + + +@pytest.fixture +def unbounce_config() -> Dict[str, Any]: + return load_config_with_replacement( + "data/saas/config/unbounce_config.yml", + "", + "unbounce_instance", + ) + + +@pytest.fixture +def unbounce_dataset() -> Dict[str, Any]: + return load_dataset_with_replacement( + "data/saas/dataset/unbounce_dataset.yml", + "", + "unbounce_instance", + )[0] + + +@pytest.fixture(scope="function") +def unbounce_connection_config( + db: Session, unbounce_config, unbounce_secrets +) -> Generator: + fides_key = unbounce_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": unbounce_secrets, + "saas_config": unbounce_config, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture +def unbounce_dataset_config( + db: Session, + unbounce_connection_config: ConnectionConfig, + unbounce_dataset: Dict[str, Any], +) -> Generator: + fides_key = unbounce_dataset["fides_key"] + unbounce_connection_config.name = fides_key + unbounce_connection_config.key = fides_key + unbounce_connection_config.save(db=db) + + ctl_dataset = CtlDataset.create_from_dataset_dict(db, unbounce_dataset) + + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": unbounce_connection_config.id, + "fides_key": fides_key, + "ctl_dataset_id": ctl_dataset.id, + }, + ) + yield dataset + dataset.delete(db=db) + ctl_dataset.delete(db=db) + + +class UnbounceTestClient: + def __init__(self, unbounce_connection_config: ConnectionConfig): + self.unbounce_secrets = cast(Dict, unbounce_connection_config.secrets) + self.headers = { + "Authorization": f"Bearer {self.unbounce_secrets['access_token']}" + } + self.page_id = self.unbounce_secrets["page_id"] + self.base_url = f"https://{self.unbounce_secrets['domain']}" + + def create_lead(self, email: str) -> Response: + body = { + "conversion": "True", + "form_submission": { + "variant_id": "a", + "form_data": { + "first_name": "Test", + "last_name": "User", + "email": email, + }, + }, + } + + return requests.post( + url=f"{self.base_url}/pages/{self.page_id}/leads", + headers=self.headers, + json=body, + ) + + +@pytest.fixture(scope="function") +def unbounce_test_client( + unbounce_connection_config: ConnectionConfig, +) -> Generator: + test_client = UnbounceTestClient(unbounce_connection_config) + yield test_client + + +@pytest.fixture(scope="function") +def unbounce_create_erasure_data( + unbounce_test_client, + unbounce_erasure_identity_email: str, +) -> None: + # create lead + unbounce_test_client.create_lead(unbounce_erasure_identity_email) diff --git a/tests/ops/integration_tests/saas/test_unbounce_task.py b/tests/ops/integration_tests/saas/test_unbounce_task.py new file mode 100644 index 0000000000..f87bf95bda --- /dev/null +++ b/tests/ops/integration_tests/saas/test_unbounce_task.py @@ -0,0 +1,181 @@ +import random + +import pytest + +from fides.api.ops.graph.graph import DatasetGraph +from fides.api.ops.models.privacy_request import PrivacyRequest +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.service.connectors import get_connector +from fides.api.ops.task import graph_task +from fides.api.ops.task.graph_task import get_cached_data_for_erasures +from fides.core.config import get_config +from tests.ops.graph.graph_test_util import assert_rows_match + +CONFIG = get_config() + + +@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors") +@pytest.mark.integration_saas +@pytest.mark.integration_unbounce +def test_unbounce_connection_test(unbounce_connection_config) -> None: + get_connector(unbounce_connection_config).test_connection() + + +@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors") +@pytest.mark.integration_saas +@pytest.mark.integration_unbounce +@pytest.mark.asyncio +async def test_unbounce_access_request_task( + db, + policy, + unbounce_connection_config, + unbounce_dataset_config, + unbounce_identity_email, +) -> None: + """Full access request based on the Unbounce SaaS config""" + + privacy_request = PrivacyRequest( + id=f"test_unbounce_access_request_task_{random.randint(0, 1000)}" + ) + identity = Identity(**{"email": unbounce_identity_email}) + privacy_request.cache_identity(identity) + + dataset_name = unbounce_connection_config.get_saas_config().fides_key + merged_graph = unbounce_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + v = await graph_task.run_access_request( + privacy_request, + policy, + graph, + [unbounce_connection_config], + {"email": unbounce_identity_email}, + db, + ) + + assert_rows_match( + v[f"{dataset_name}:pages"], + min_size=1, + keys=[ + "subAccountId", + "integrations", + "integrationsCount", + "integrationsErrorsCount", + "id", + "url", + "metadata", + "createdAt", + "name", + "state", + "lastPublishedAt", + "variantsCount", + "domain", + ], + ) + + assert_rows_match( + v[f"{dataset_name}:leads"], + min_size=1, + keys=[ + "created_at", + "id", + "extra_data", + "form_data", + "page_id", + "variant_id", + "metadata", + ], + ) + + # verify we only returned data for our identity email + for leads in v[f"{dataset_name}:leads"]: + assert unbounce_identity_email in leads["form_data"]["email"] + + +@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors") +@pytest.mark.integration_saas +@pytest.mark.integration_unbounce +@pytest.mark.asyncio +async def test_unbounce_erasure_request_task( + db, + policy, + erasure_policy_string_rewrite, + unbounce_connection_config, + unbounce_dataset_config, + unbounce_erasure_identity_email, + unbounce_create_erasure_data, +) -> None: + """Full erasure request based on the Unbounce SaaS config""" + + masking_strict = CONFIG.execution.masking_strict + CONFIG.execution.masking_strict = False # Allow Delete + + privacy_request = PrivacyRequest( + id=f"test_unbounce_erasure_request_task_{random.randint(0, 1000)}" + ) + identity = Identity(**{"email": unbounce_erasure_identity_email}) + privacy_request.cache_identity(identity) + + dataset_name = unbounce_connection_config.get_saas_config().fides_key + merged_graph = unbounce_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + v = await graph_task.run_access_request( + privacy_request, + policy, + graph, + [unbounce_connection_config], + {"email": unbounce_erasure_identity_email}, + db, + ) + + assert_rows_match( + v[f"{dataset_name}:pages"], + min_size=1, + keys=[ + "subAccountId", + "integrations", + "integrationsCount", + "integrationsErrorsCount", + "id", + "url", + "metadata", + "createdAt", + "name", + "state", + "lastPublishedAt", + "variantsCount", + "domain", + ], + ) + + assert_rows_match( + v[f"{dataset_name}:leads"], + min_size=1, + keys=[ + "created_at", + "id", + "extra_data", + "form_data", + "page_id", + "variant_id", + "metadata", + ], + ) + + x = await graph_task.run_erasure( + privacy_request, + erasure_policy_string_rewrite, + graph, + [unbounce_connection_config], + {"email": unbounce_erasure_identity_email}, + get_cached_data_for_erasures(privacy_request.id), + db, + ) + + assert x == { + f"{dataset_name}:pages": 0, + f"{dataset_name}:leads": 1, + } + + CONFIG.execution.masking_strict = masking_strict