From 1aa32190a8901a99022932d7a03fb7fce49e1cb6 Mon Sep 17 00:00:00 2001 From: tmuralikrishnan <120677902+tmuralikrishnan@users.noreply.github.com> Date: Wed, 3 May 2023 05:04:51 +0530 Subject: [PATCH] Gorgias Connector (#2444) Co-authored-by: Adrian Galvan Co-authored-by: vivek --- CHANGELOG.md | 1 + data/saas/config/gorgias_config.yml | 126 +++++++ data/saas/dataset/gorgias_dataset.yml | 271 +++++++++++++++ data/saas/icon/gorgias.svg | 3 + tests/fixtures/saas/gorgias_fixtures.py | 148 ++++++++ .../saas/test_gorgias_task.py | 319 ++++++++++++++++++ 6 files changed, 868 insertions(+) create mode 100644 data/saas/config/gorgias_config.yml create mode 100644 data/saas/dataset/gorgias_dataset.yml create mode 100644 data/saas/icon/gorgias.svg create mode 100644 tests/fixtures/saas/gorgias_fixtures.py create mode 100644 tests/ops/integration_tests/saas/test_gorgias_task.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e76d1634c2..e26984ae55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The types of changes are: ### Added - Access and erasure support for Amplitude [#2569](https://github.com/ethyca/fides/pull/2569) +- Access and erasure support for Gorgias [#2444](https://github.com/ethyca/fides/pull/2444) ## [2.12.0](https://github.com/ethyca/fides/compare/2.11.0...2.12.0) diff --git a/data/saas/config/gorgias_config.yml b/data/saas/config/gorgias_config.yml new file mode 100644 index 0000000000..5dba8a4deb --- /dev/null +++ b/data/saas/config/gorgias_config.yml @@ -0,0 +1,126 @@ +saas_config: + fides_key: + name: Gorgias + type: gorgias + description: A sample schema representing the Gorgias connector for Fides + version: 0.1.0 + + connector_params: + - name: domain + - name: username + - name: api_key + label: API Key + + client_config: + protocol: https + host: + authentication: + strategy: basic + configuration: + username: + password: + + test_request: + method: GET + path: /api/customers + + endpoints: + - name: customer + requests: + read: + method: GET + path: /api/customers + query_params: + - name: email + value: + data_path: data + param_values: + - name: email + identity: email + update: + method: PUT + path: /api/customers// + body: | + { + + } + param_values: + - name: customer_id + references: + - dataset: + field: customer.id + direction: from + - name: tickets + requests: + read: + method: GET + path: /api/tickets + query_params: + - name: customer_id + value: + - name: limit + value: "30" + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: cursor + field: meta.next_cursor + param_values: + - name: customer_id + references: + - dataset: + field: customer.id + direction: from + update: + method: PUT + path: /api/tickets// + body: | + { + + } + param_values: + - name: tickets_id + references: + - dataset: + field: tickets.id + direction: from + - name: ticket_messages + requests: + read: + method: GET + path: /api/messages + query_params: + - name: ticket_id + value: + data_path: data + pagination: + strategy: cursor + configuration: + cursor_param: cursor + field: meta.next_cursor + param_values: + - name: tickets_id + references: + - dataset: + field: tickets.id + direction: from + update: + method: PUT + path: /api/tickets//messages// + grouped_inputs: [ticket_id, ticket_message_id] + body: | + { + + } + param_values: + - name: ticket_id + references: + - dataset: + field: ticket_messages.ticket_id + direction: from + - name: ticket_message_id + references: + - dataset: + field: ticket_messages.id + direction: from diff --git a/data/saas/dataset/gorgias_dataset.yml b/data/saas/dataset/gorgias_dataset.yml new file mode 100644 index 0000000000..90de07fcdc --- /dev/null +++ b/data/saas/dataset/gorgias_dataset.yml @@ -0,0 +1,271 @@ +dataset: + - fides_key: + name: Gorgias Dataset + description: A sample dataset representing the Gorgias connector for Fides + collections: + - name: customer + fields: + - name: id + data_categories: [user.unique_id] + fidesops_meta: + primary_key: True + data_type: integer + read_only: True + - name: external_id + - name: active + data_categories: [system.operations] + fidesops_meta: + data_type: boolean + - name: email + data_categories: [user.contact.email] + fidesops_meta: + data_type: string + - name: name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: firstname + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: lastname + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: language + - name: timezone + - name: created_datetime + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: updated_datetime + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: meta + fidesops_meta: + data_type: object + fields: + - name: name_set_via + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: data + - name: note + - name: tickets + fields: + - name: id + data_categories: [system.operations] + fidesops_meta: + primary_key: True + data_type: integer + - name: uri + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: external_id + - name: language + - name: status + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: priority + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: channel + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: via + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: from_agent + data_categories: [system.operations] + fidesops_meta: + data_type: boolean + - name: customer + fidesops_meta: + data_type: object + fields: + - name: id + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: email + data_categories: [user.contact.email] + fidesops_meta: + data_type: string + - name: name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: firstname + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: lastname + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: meta + fidesops_meta: + data_type: object + fields: + - name: name_set_via + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: assignee_user + - name: assignee_team + - name: subject + - name: meta + - name: tags + - name: is_unread + data_categories: [system.operations] + fidesops_meta: + data_type: boolean + - name: spam + data_categories: [system.operations] + fidesops_meta: + data_type: boolean + - name: created_datetime + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: opened_datetime + - name: last_received_message_datetime + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: last_message_datetime + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: updated_datetime + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: closed_datetime + - name: snooze_datetime + - name: trashed_datetime + - name: integrations + data_categories: [system.operations] + fidesops_meta: + read_only: True + data_type: integer + - name: messages_count + data_categories: [system.operations] + fidesops_meta: + read_only: True + data_type: integer + - name: excerpt + data_categories: [system.operations] + fidesops_meta: + read_only: True + data_type: string + - name: ticket_messages + fields: + - name: id + data_categories: [system.operations] + fidesops_meta: + primary_key: True + data_type: integer + - name: uri + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: message_id + - name: ticket_id + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: external_id + - name: public + data_categories: [system.operations] + fidesops_meta: + data_type: boolean + - name: channel + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: via + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: source + fidesops_meta: + data_type: object + fields: + - name: type + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: sender + fidesops_meta: + data_type: object + fields: + - name: id + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: email + data_categories: [user.contact.email] + fidesops_meta: + data_type: string + - name: name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: firstname + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: lastname + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: meta + fidesops_meta: + data_type: object + fields: + - name: name_set_via + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: integration_id + - name: intents + - name: rule_id + - name: from_agent + data_categories: [system.operations] + fidesops_meta: + data_type: boolean + - name: receiver + - name: subject + - name: body_text + - name: body_html + - name: stripped_text + - name: stripped_html + - name: stripped_signature + - name: headers + - name: attachments + - name: actions + - name: macros + - name: meta + - name: created_datetime + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: sent_datetime + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: failed_datetime + - name: deleted_datetime + - name: opened_datetime + - name: last_sending_error + - name: is_retriable + data_categories: [system.operations] + fidesops_meta: + data_type: boolean diff --git a/data/saas/icon/gorgias.svg b/data/saas/icon/gorgias.svg new file mode 100644 index 0000000000..3b08cbd9d8 --- /dev/null +++ b/data/saas/icon/gorgias.svg @@ -0,0 +1,3 @@ + + + diff --git a/tests/fixtures/saas/gorgias_fixtures.py b/tests/fixtures/saas/gorgias_fixtures.py new file mode 100644 index 0000000000..139365bccf --- /dev/null +++ b/tests/fixtures/saas/gorgias_fixtures.py @@ -0,0 +1,148 @@ +from time import sleep +from typing import Any, Dict, Generator + +import pydash +import pytest +import requests +from sqlalchemy.orm import Session + +from fides.api.ctl.sql_models import Dataset as CtlDataset +from fides.api.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fides.api.ops.models.datasetconfig import DatasetConfig +from fides.api.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) +from fides.lib.cryptography import cryptographic_util +from tests.ops.test_helpers.vault_client import get_secrets + +secrets = get_secrets("gorgias") + + +@pytest.fixture(scope="session") +def gorgias_secrets(saas_config): + return { + "domain": pydash.get(saas_config, "gorgias.domain") or secrets["domain"], + "username": pydash.get(saas_config, "gorgias.username") or secrets["username"], + "api_key": pydash.get(saas_config, "gorgias.api_key") or secrets["api_key"], + } + + +@pytest.fixture(scope="session") +def gorgias_identity_email(saas_config): + return ( + pydash.get(saas_config, "gorgias.identity_email") or secrets["identity_email"] + ) + + +@pytest.fixture(scope="function") +def gorgias_erasure_identity_email() -> str: + return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" + + +@pytest.fixture +def gorgias_config() -> Dict[str, Any]: + return load_config_with_replacement( + "data/saas/config/gorgias_config.yml", + "", + "gorgias_instance", + ) + + +@pytest.fixture +def gorgias_dataset() -> Dict[str, Any]: + return load_dataset_with_replacement( + "data/saas/dataset/gorgias_dataset.yml", + "", + "gorgias_instance", + )[0] + + +@pytest.fixture(scope="function") +def gorgias_connection_config( + db: Session, gorgias_config, gorgias_secrets +) -> Generator: + fides_key = gorgias_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": gorgias_secrets, + "saas_config": gorgias_config, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture +def gorgias_dataset_config( + db: Session, + gorgias_connection_config: ConnectionConfig, + gorgias_dataset: Dict[str, Any], +) -> Generator: + fides_key = gorgias_dataset["fides_key"] + gorgias_connection_config.name = fides_key + gorgias_connection_config.key = fides_key + gorgias_connection_config.save(db=db) + + ctl_dataset = CtlDataset.create_from_dataset_dict(db, gorgias_dataset) + + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": gorgias_connection_config.id, + "fides_key": fides_key, + "ctl_dataset_id": ctl_dataset.id, + }, + ) + yield dataset + dataset.delete(db=db) + ctl_dataset.delete(db=db) + + +@pytest.fixture(scope="function") +def gorgias_create_erasure_data( + gorgias_connection_config: ConnectionConfig, gorgias_erasure_identity_email: str +) -> None: + gorgias_secrets = gorgias_connection_config.secrets + auth = gorgias_secrets["username"], gorgias_secrets["api_key"] + base_url = f"https://{gorgias_secrets['domain']}" + + # user + body = { + "name": "Ethyca Test Erasure", + "email": gorgias_erasure_identity_email, + } + + users_response = requests.post( + url=f"{base_url}/api/customers", auth=auth, json=body + ) + user = users_response.json() + user_id = user["id"] + + ticket_data = { + "customer": {"id": user_id, "email": gorgias_erasure_identity_email}, + "messages": [ + { + "sender": {"id": user_id, "email": gorgias_erasure_identity_email}, + "channel": "twitter-direct-message", + "from_agent": "false", + "via": "instagram-ad-comment", + } + ], + "channel": "api", + "status": "open", + "subject": "Tested", + } + response = requests.post(url=f"{base_url}/api/tickets", auth=auth, json=ticket_data) + ticket = response.json() + sleep(60) + yield ticket, user diff --git a/tests/ops/integration_tests/saas/test_gorgias_task.py b/tests/ops/integration_tests/saas/test_gorgias_task.py new file mode 100644 index 0000000000..8dedee4938 --- /dev/null +++ b/tests/ops/integration_tests/saas/test_gorgias_task.py @@ -0,0 +1,319 @@ +import random + +import pytest +import requests + +from fides.api.ops.graph.graph import DatasetGraph +from fides.api.ops.models.privacy_request import PrivacyRequest +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.service.connectors import get_connector +from fides.api.ops.task import graph_task +from fides.api.ops.task.graph_task import get_cached_data_for_erasures +from fides.core.config import get_config +from tests.ops.graph.graph_test_util import assert_rows_match + +CONFIG = get_config() + + +@pytest.mark.integration_saas +@pytest.mark.integration_gorgias +def test_gorgias_connection_test(gorgias_connection_config) -> None: + get_connector(gorgias_connection_config).test_connection() + + +@pytest.mark.integration_saas +@pytest.mark.integration_gorgias +@pytest.mark.asyncio +async def test_gorgias_access_request_task( + db, + policy, + gorgias_connection_config, + gorgias_dataset_config, + gorgias_identity_email, +) -> None: + """Full access request based on the Gorgias SaaS config""" + + privacy_request = PrivacyRequest( + id=f"test_gorgias_access_request_task_{random.randint(0, 1000)}" + ) + identity = Identity(**{"email": gorgias_identity_email}) + privacy_request.cache_identity(identity) + + dataset_name = gorgias_connection_config.get_saas_config().fides_key + merged_graph = gorgias_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + v = await graph_task.run_access_request( + privacy_request, + policy, + graph, + [gorgias_connection_config], + {"email": gorgias_identity_email}, + db, + ) + + assert_rows_match( + v[f"{dataset_name}:customer"], + min_size=1, + keys=[ + "id", + "external_id", + "active", + "name", + "email", + "firstname", + "lastname", + "language", + "timezone", + "created_datetime", + "updated_datetime", + "meta", + "data", + "note", + ], + ) + + assert_rows_match( + v[f"{dataset_name}:tickets"], + min_size=1, + keys=[ + "id", + "uri", + "external_id", + "language", + "status", + "priority", + "channel", + "via", + "from_agent", + "customer", + "assignee_user", + "assignee_team", + "subject", + "meta", + "tags", + "is_unread", + "spam", + "created_datetime", + "opened_datetime", + "last_received_message_datetime", + "last_message_datetime", + "updated_datetime", + "closed_datetime", + "snooze_datetime", + "trashed_datetime", + "integrations", + "messages_count", + "excerpt", + ], + ) + + assert_rows_match( + v[f"{dataset_name}:ticket_messages"], + min_size=1, + keys=[ + "id", + "uri", + "message_id", + "ticket_id", + "external_id", + "public", + "channel", + "via", + "source", + "sender", + "integration_id", + "intents", + "rule_id", + "from_agent", + "receiver", + "subject", + "body_text", + "body_html", + "stripped_text", + "stripped_html", + "stripped_signature", + "headers", + "attachments", + "actions", + "macros", + "meta", + "created_datetime", + "sent_datetime", + "failed_datetime", + "deleted_datetime", + "opened_datetime", + "last_sending_error", + "is_retriable", + ], + ) + + # verify we only returned data for our identity email + assert v[f"{dataset_name}:customer"][0]["email"] == gorgias_identity_email + user_id = v[f"{dataset_name}:customer"][0]["id"] + + for ticket in v[f"{dataset_name}:tickets"]: + assert ticket["customer"]["id"] == user_id + + +@pytest.mark.integration_saas +@pytest.mark.integration_gorgias +@pytest.mark.asyncio +async def test_gorgias_erasure_request_task( + db, + policy, + erasure_policy_string_rewrite, + gorgias_connection_config, + gorgias_dataset_config, + gorgias_erasure_identity_email, + gorgias_create_erasure_data, +) -> None: + """Full erasure request based on the Gorgias SaaS config""" + + masking_strict = CONFIG.execution.masking_strict + CONFIG.execution.masking_strict = True + + privacy_request = PrivacyRequest( + id=f"test_gorgias_erasure_request_task_{random.randint(0, 1000)}" + ) + identity = Identity(**{"email": gorgias_erasure_identity_email}) + privacy_request.cache_identity(identity) + + dataset_name = gorgias_connection_config.get_saas_config().fides_key + merged_graph = gorgias_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + v = await graph_task.run_access_request( + privacy_request, + policy, + graph, + [gorgias_connection_config], + {"email": gorgias_erasure_identity_email}, + db, + ) + + assert_rows_match( + v[f"{dataset_name}:customer"], + min_size=1, + keys=[ + "id", + "external_id", + "active", + "name", + "email", + "firstname", + "lastname", + "language", + "timezone", + "created_datetime", + "updated_datetime", + "meta", + "data", + "note", + ], + ) + + assert_rows_match( + v[f"{dataset_name}:tickets"], + min_size=1, + keys=[ + "id", + "uri", + "external_id", + "language", + "status", + "priority", + "channel", + "via", + "from_agent", + "customer", + "assignee_user", + "assignee_team", + "subject", + "meta", + "tags", + "is_unread", + "spam", + "created_datetime", + "opened_datetime", + "last_received_message_datetime", + "last_message_datetime", + "updated_datetime", + "closed_datetime", + "snooze_datetime", + "trashed_datetime", + "integrations", + "messages_count", + "excerpt", + ], + ) + + assert_rows_match( + v[f"{dataset_name}:ticket_messages"], + min_size=1, + keys=[ + "id", + "uri", + "message_id", + "ticket_id", + "external_id", + "public", + "channel", + "via", + "source", + "sender", + "integration_id", + "intents", + "rule_id", + "from_agent", + "receiver", + "subject", + "body_text", + "body_html", + "stripped_text", + "stripped_html", + "stripped_signature", + "headers", + "attachments", + "actions", + "macros", + "meta", + "created_datetime", + "sent_datetime", + "failed_datetime", + "deleted_datetime", + "opened_datetime", + "last_sending_error", + "is_retriable", + ], + ) + + x = await graph_task.run_erasure( + privacy_request, + erasure_policy_string_rewrite, + graph, + [gorgias_connection_config], + {"email": gorgias_erasure_identity_email}, + get_cached_data_for_erasures(privacy_request.id), + db, + ) + + assert x == { + f"{dataset_name}:customer": 1, + f"{dataset_name}:tickets": 1, + f"{dataset_name}:ticket_messages": 1, + } + + gorgias_secrets = gorgias_connection_config.secrets + auth = gorgias_secrets["username"], gorgias_secrets["api_key"] + base_url = f"https://{gorgias_secrets['domain']}" + + # user + response = requests.get( + url=f"{base_url}/api/customers", + auth=auth, + params={"email": gorgias_erasure_identity_email}, + ) + assert response.status_code == 200 + + CONFIG.execution.masking_strict = masking_strict