From 1c393998584b51e1a82a820c5107b9966e722600 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 6 Mar 2023 09:30:03 -0800 Subject: [PATCH] SaaS connector test refactoring (#1795) --- CHANGELOG.md | 1 + data/saas/config/zendesk_config.yml | 23 +- data/saas/dataset/zendesk_dataset.yml | 4 +- .../new_fixtures.jinja | 69 +++ .../test_new_task.jinja | 51 +++ noxfiles/utils_nox.py | 57 +++ tests/fixtures/saas/doordash_fixtures.py | 162 +------- .../saas/external_datasets/doordash.sql | 7 - .../saas/external_datasets/yotpo_reviews.sql | 7 - tests/fixtures/saas/yotpo_loyalty_fixtures.py | 6 +- tests/fixtures/saas/yotpo_reviews_fixtures.py | 232 +++-------- tests/fixtures/saas/zendesk_fixtures.py | 188 ++++----- .../saas/connector_runner.py | 343 +++++++++++++++ .../saas/test_doordash_task.py | 97 +---- .../saas/test_yotpo_reviews_task.py | 209 ++-------- .../saas/test_zendesk_task.py | 393 +++--------------- tests/ops/test_helpers/saas_test_utils.py | 15 +- tests/ops/test_helpers/vault_client.py | 9 +- 18 files changed, 808 insertions(+), 1065 deletions(-) create mode 100644 data/saas/saas_connector_scaffolding/new_fixtures.jinja create mode 100644 data/saas/saas_connector_scaffolding/test_new_task.jinja delete mode 100644 tests/fixtures/saas/external_datasets/doordash.sql delete mode 100644 tests/fixtures/saas/external_datasets/yotpo_reviews.sql create mode 100644 tests/ops/integration_tests/saas/connector_runner.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d66dfed1c..b9935b0f52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The types of changes are: * Improve "Upload a new dataset YAML" [#1531](https://github.com/ethyca/fides/pull/2258) * Access and erasure support for Yotpo [#2708](https://github.com/ethyca/fides/pull/2708) * Allow SendGrid template usage [#2728](https://github.com/ethyca/fides/pull/2728) +* Added ConnectorRunner to simplify SaaS connector testing [#1795](https://github.com/ethyca/fides/pull/1795) ### Changed diff --git a/data/saas/config/zendesk_config.yml b/data/saas/config/zendesk_config.yml index b0bccfc384..38d35a3896 100644 --- a/data/saas/config/zendesk_config.yml +++ b/data/saas/config/zendesk_config.yml @@ -2,14 +2,13 @@ saas_config: fides_key: name: Zendesk SaaS Config type: zendesk - description: A sample schema representing the Zendesk connector for Fidesops + description: A sample schema representing the Zendesk connector for Fides version: 0.0.1 connector_params: - name: domain - name: username - name: api_key - - name: page_size client_config: protocol: https @@ -28,7 +27,7 @@ saas_config: value: test@ethyca endpoints: - - name: users + - name: user requests: read: method: GET @@ -47,7 +46,7 @@ saas_config: - name: user_id references: - dataset: - field: users.id + field: user.id direction: from - name: user_identities requests: @@ -56,15 +55,13 @@ saas_config: path: /api/v2/users//identities.json query_params: - name: page[size] - value: + value: 100 param_values: - name: user_id references: - dataset: - field: users.id + field: user.id direction: from - - name: page_size - connector_param: page_size data_path: identities pagination: strategy: link @@ -78,15 +75,13 @@ saas_config: path: /api/v2/users//tickets/requested.json query_params: - name: page[size] - value: + value: 100 param_values: - name: user_id references: - dataset: - field: users.id + field: user.id direction: from - - name: page_size - connector_param: page_size data_path: tickets pagination: strategy: link @@ -109,15 +104,13 @@ saas_config: path: /api/v2/tickets//comments.json query_params: - name: page[size] - value: + value: 100 param_values: - name: ticket_id references: - dataset: field: tickets.id direction: from - - name: page_size - connector_param: page_size data_path: comments pagination: strategy: link diff --git a/data/saas/dataset/zendesk_dataset.yml b/data/saas/dataset/zendesk_dataset.yml index fdbc5a3e9b..c525bb5289 100644 --- a/data/saas/dataset/zendesk_dataset.yml +++ b/data/saas/dataset/zendesk_dataset.yml @@ -1,9 +1,9 @@ dataset: - fides_key: name: Zendesk Dataset - description: A sample dataset representing the Zendesk connector for Fidesops + description: A sample dataset representing the Zendesk connector for Fides collections: - - name: users + - name: user fields: - name: id data_categories: [system.operations] diff --git a/data/saas/saas_connector_scaffolding/new_fixtures.jinja b/data/saas/saas_connector_scaffolding/new_fixtures.jinja new file mode 100644 index 0000000000..b950aded1e --- /dev/null +++ b/data/saas/saas_connector_scaffolding/new_fixtures.jinja @@ -0,0 +1,69 @@ +from typing import Any, Dict, Generator + +import pydash +import pytest + +from tests.ops.integration_tests.saas.connector_runner import ( + ConnectorRunner, + generate_random_email, +) +from tests.ops.test_helpers.vault_client import get_secrets + +secrets = get_secrets("{{ connector_id }}") + + +@pytest.fixture(scope="session") +def {{ connector_id }}_secrets(saas_config) -> Dict[str, Any]: + return { + "domain": pydash.get(saas_config, "{{ connector_id }}.domain") + or secrets["domain"] + # add the rest of your secrets here + } + + +@pytest.fixture(scope="session") +def {{ connector_id }}_identity_email(saas_config) -> str: + return ( + pydash.get(saas_config, "{{ connector_id }}.identity_email") or secrets["identity_email"] + ) + + +@pytest.fixture +def {{ connector_id }}_erasure_identity_email() -> str: + return generate_random_email() + + +@pytest.fixture +def {{ connector_id }}_external_references() -> Dict[str, Any]: + return {} + + +@pytest.fixture +def {{ connector_id }}_erasure_external_references() -> Dict[str, Any]: + return {} + + +@pytest.fixture +def {{ connector_id }}_erasure_data( + {{ connector_id }}_erasure_identity_email: str, +) -> Generator: + # create the data needed for erasure tests here + yield {} + + +@pytest.fixture +def {{ connector_id }}_runner( + db, + cache, + {{ connector_id }}_secrets, + {{ connector_id }}_external_references, + {{ connector_id }}_erasure_external_references, +) -> ConnectorRunner: + return ConnectorRunner( + db, + cache, + "{{ connector_id }}", + {{ connector_id }}_secrets, + external_references={{ connector_id }}_external_references, + erasure_external_references={{ connector_id }}_erasure_external_references, + ) diff --git a/data/saas/saas_connector_scaffolding/test_new_task.jinja b/data/saas/saas_connector_scaffolding/test_new_task.jinja new file mode 100644 index 0000000000..413d6af60c --- /dev/null +++ b/data/saas/saas_connector_scaffolding/test_new_task.jinja @@ -0,0 +1,51 @@ +import pytest + +from fides.api.ops.models.policy import Policy +from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner + + +@pytest.mark.integration_saas +class Test{{ connector_name }}Connector: + def test_connection(self, {{ connector_id }}_runner: ConnectorRunner): + {{ connector_id }}_runner.test_connection() + + async def test_access_request( + self, {{ connector_id }}_runner: ConnectorRunner, policy, {{ connector_id }}_identity_email: str + ): + access_results = await {{ connector_id }}_runner.access_request( + access_policy=policy, identities={"email": {{ connector_id }}_identity_email} + ) + + async def test_strict_erasure_request( + self, + {{ connector_id }}_runner: ConnectorRunner, + policy: Policy, + erasure_policy_string_rewrite: Policy, + {{ connector_id }}_erasure_identity_email: str, + {{ connector_id }}_erasure_data, + ): + ( + access_results, + erasure_results, + ) = await {{ connector_id }}_runner.strict_erasure_request( + access_policy=policy, + erasure_policy=erasure_policy_string_rewrite, + identities={"email": {{ connector_id }}_erasure_identity_email}, + ) + + async def test_non_strict_erasure_request( + self, + {{ connector_id }}_runner: ConnectorRunner, + policy: Policy, + erasure_policy_string_rewrite: Policy, + {{ connector_id }}_erasure_identity_email: str, + {{ connector_id }}_erasure_data, + ): + ( + access_results, + erasure_results, + ) = await {{ connector_id }}_runner.non_strict_erasure_request( + access_policy=policy, + erasure_policy=erasure_policy_string_rewrite, + identities={"email": {{ connector_id }}_erasure_identity_email}, + ) diff --git a/noxfiles/utils_nox.py b/noxfiles/utils_nox.py index c1669bc1bb..0422f92e87 100644 --- a/noxfiles/utils_nox.py +++ b/noxfiles/utils_nox.py @@ -1,4 +1,6 @@ """Contains various utility-related nox sessions.""" +from pathlib import Path + import nox from constants_nox import COMPOSE_FILE, INTEGRATION_COMPOSE_FILE, TEST_ENV_COMPOSE_FILE @@ -60,3 +62,58 @@ def teardown(session: nox.Session) -> None: def install_requirements(session: nox.Session) -> None: session.install("-r", "requirements.txt") session.install("-r", "dev-requirements.txt") + + +@nox.session() +def init_saas_connector(session: nox.Session) -> None: + connector_name = session.posargs[0].replace(" ", "") + connector_id = "_".join(session.posargs[0].lower().split(" ")) + variable_map = {"connector_name": connector_name, "connector_id": connector_id} + + # create empty config and dataset files + try: + Path(f"data/saas/config/{variable_map['connector_id']}_config.yml").touch( + exist_ok=False + ) + Path(f"data/saas/dataset/{variable_map['connector_id']}_dataset.yml").touch( + exist_ok=False + ) + except Exception: + session.error( + f"Files for {session.posargs[0]} already exist, skipping initialization" + ) + + # location of Jinja templates + from jinja2 import Environment, FileSystemLoader + + environment = Environment( + loader=FileSystemLoader("data/saas/saas_connector_scaffolding/") + ) + + # render fixtures file + fixtures_template = environment.get_template("new_fixtures.jinja") + filename = f"tests/fixtures/saas/{variable_map['connector_id']}_fixtures.py" + contents = fixtures_template.render(variable_map) + try: + with open(filename, mode="x", encoding="utf-8") as fixtures: + fixtures.write(contents) + fixtures.close() + except FileExistsError: + session.error( + f"Files for {session.posargs[0]} already exist, skipping initialization" + ) + + # render tests file + test_template = environment.get_template("test_new_task.jinja") + filename = ( + f"tests/ops/integration_tests/saas/test_{variable_map['connector_id']}_task.py" + ) + contents = test_template.render(variable_map) + try: + with open(filename, mode="x", encoding="utf-8") as tests: + tests.write(contents) + tests.close() + except FileExistsError: + session.error( + f"Files for {session.posargs[0]} already exist, skipping initialization" + ) diff --git a/tests/fixtures/saas/doordash_fixtures.py b/tests/fixtures/saas/doordash_fixtures.py index 81a0b5be7a..dc69cd8d7d 100644 --- a/tests/fixtures/saas/doordash_fixtures.py +++ b/tests/fixtures/saas/doordash_fixtures.py @@ -1,28 +1,11 @@ -from typing import Any, Dict, Generator +from typing import Any, Dict import pydash import pytest -from sqlalchemy.orm import Session -from sqlalchemy_utils.functions import drop_database -from fides.api.ctl.sql_models import Dataset as CtlDataset -from fides.api.ops.models.connectionconfig import ( - AccessLevel, - ConnectionConfig, - ConnectionType, -) -from fides.api.ops.models.datasetconfig import DatasetConfig -from fides.api.ops.util.saas_util import ( - load_config_with_replacement, - load_dataset_with_replacement, -) -from fides.lib.cryptography import cryptographic_util -from fides.lib.db import session -from tests.ops.test_helpers.db_utils import seed_postgres_data +from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner from tests.ops.test_helpers.vault_client import get_secrets -from ..application_fixtures import load_dataset - secrets = get_secrets("doordash") @@ -35,11 +18,6 @@ def doordash_secrets(saas_config): "key_id": pydash.get(saas_config, "doordash.key_id") or secrets["key_id"], "signing_secret": pydash.get(saas_config, "doordash.signing_secret") or secrets["signing_secret"], - "doordash_delivery_id": { - "dataset": "doordash_postgres", - "field": "doordash_deliveries.delivery_id", - "direction": "from", - }, } @@ -50,130 +28,22 @@ def doordash_identity_email(saas_config): ) -@pytest.fixture(scope="session") -def doordash_erasure_identity_email(): - return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" - - -@pytest.fixture -def doordash_config() -> Dict[str, Any]: - return load_config_with_replacement( - "data/saas/config/doordash_config.yml", - "", - "doordash_instance", - ) - - -@pytest.fixture -def doordash_dataset() -> Dict[str, Any]: - return load_dataset_with_replacement( - "data/saas/dataset/doordash_dataset.yml", - "", - "doordash_instance", - )[0] - - -@pytest.fixture(scope="function") -def doordash_connection_config( - db: session, doordash_config, doordash_secrets -) -> Generator: - fides_key = doordash_config["fides_key"] - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": fides_key, - "name": fides_key, - "connection_type": ConnectionType.saas, - "access": AccessLevel.write, - "secrets": doordash_secrets, - "saas_config": doordash_config, - }, - ) - yield connection_config - connection_config.delete(db) - - @pytest.fixture -def doordash_dataset_config( - db: Session, - doordash_connection_config: ConnectionConfig, - doordash_dataset: Dict[str, Any], -) -> Generator: - fides_key = doordash_dataset["fides_key"] - doordash_connection_config.name = fides_key - doordash_connection_config.key = fides_key - doordash_connection_config.save(db=db) - ctl_dataset = CtlDataset.create_from_dataset_dict(db, doordash_dataset) - - dataset = DatasetConfig.create( - db=db, - data={ - "connection_config_id": doordash_connection_config.id, - "fides_key": fides_key, - "ctl_dataset_id": ctl_dataset.id, - }, - ) - yield dataset - dataset.delete(db=db) - ctl_dataset.delete(db=db) - - -@pytest.fixture() -def doordash_postgres_dataset() -> Dict[str, Any]: - return { - "fides_key": "doordash_postgres", - "name": "Doordash Postgres", - "description": "Lookup for Doordash delivery IDs", - "collections": [ - { - "name": "doordash_deliveries", - "fields": [ - { - "name": "email", - "data_categories": ["user.contact.email"], - "fidesops_meta": {"data_type": "string", "identity": "email"}, - }, - { - "name": "delivery_id", - "fidesops_meta": {"data_type": "string"}, - }, - ], - } - ], - } +def doordash_external_references() -> Dict[str, Any]: + return {"doordash_delivery_id": "D-12345"} @pytest.fixture -def doordash_postgres_dataset_config( - connection_config: ConnectionConfig, - doordash_postgres_dataset: Dict[str, Any], - db: Session, -) -> Generator: - fides_key = doordash_postgres_dataset["fides_key"] - connection_config.name = fides_key - connection_config.key = fides_key - connection_config.save(db=db) - - ctl_dataset = CtlDataset.create_from_dataset_dict(db, doordash_postgres_dataset) - - dataset = DatasetConfig.create( - db=db, - data={ - "connection_config_id": connection_config.id, - "fides_key": fides_key, - "ctl_dataset_id": ctl_dataset.id, - }, - ) - yield dataset - dataset.delete(db=db) - ctl_dataset.delete(db) - - -@pytest.fixture(scope="function") -def doordash_postgres_db(postgres_integration_session): - postgres_integration_session = seed_postgres_data( - postgres_integration_session, - "./tests/fixtures/saas/external_datasets/doordash.sql", +def doordash_runner( + db, + cache, + doordash_secrets, + doordash_external_references, +) -> ConnectorRunner: + return ConnectorRunner( + db, + cache, + "doordash", + doordash_secrets, + external_references=doordash_external_references, ) - yield postgres_integration_session - drop_database(postgres_integration_session.bind.url) diff --git a/tests/fixtures/saas/external_datasets/doordash.sql b/tests/fixtures/saas/external_datasets/doordash.sql deleted file mode 100644 index 44d94b737d..0000000000 --- a/tests/fixtures/saas/external_datasets/doordash.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE public.doordash_deliveries ( - email CHARACTER VARYING(100) PRIMARY KEY, - delivery_id CHARACTER VARYING(100) -); - -INSERT INTO public.doordash_deliveries VALUES -('test@example.com', 'D-12345') \ No newline at end of file diff --git a/tests/fixtures/saas/external_datasets/yotpo_reviews.sql b/tests/fixtures/saas/external_datasets/yotpo_reviews.sql deleted file mode 100644 index 3662b5a09f..0000000000 --- a/tests/fixtures/saas/external_datasets/yotpo_reviews.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE public.yotpo_customer ( - email CHARACTER VARYING(100) PRIMARY KEY, - external_id CHARACTER VARYING(100) -); - -INSERT INTO public.yotpo_customer VALUES -('test@example.com', 'ak123798684365sdfkj') \ No newline at end of file diff --git a/tests/fixtures/saas/yotpo_loyalty_fixtures.py b/tests/fixtures/saas/yotpo_loyalty_fixtures.py index 2cb4df9afd..59a6dec0e6 100644 --- a/tests/fixtures/saas/yotpo_loyalty_fixtures.py +++ b/tests/fixtures/saas/yotpo_loyalty_fixtures.py @@ -126,7 +126,7 @@ def yotpo_loyalty_dataset_config( ctl_dataset.delete(db=db) -class YotpoReviewsTestClient: +class YotpoLoyaltyTestClient: def __init__(self, connection_config: ConnectionConfig): yotpo_loyalty_secrets = connection_config.secrets self.domain = yotpo_loyalty_secrets["domain"] @@ -161,13 +161,13 @@ def get_customer(self, email: str) -> Any: def yotpo_loyalty_test_client( yotpo_loyalty_connection_config: ConnectionConfig, ) -> Generator: - test_client = YotpoReviewsTestClient(yotpo_loyalty_connection_config) + test_client = YotpoLoyaltyTestClient(yotpo_loyalty_connection_config) yield test_client @pytest.fixture(scope="function") def yotpo_loyalty_erasure_data( - yotpo_loyalty_test_client: YotpoReviewsTestClient, + yotpo_loyalty_test_client: YotpoLoyaltyTestClient, yotpo_loyalty_erasure_identity_email, ) -> None: # create customer diff --git a/tests/fixtures/saas/yotpo_reviews_fixtures.py b/tests/fixtures/saas/yotpo_reviews_fixtures.py index 3ae7af737e..b1229ea7b5 100644 --- a/tests/fixtures/saas/yotpo_reviews_fixtures.py +++ b/tests/fixtures/saas/yotpo_reviews_fixtures.py @@ -1,5 +1,5 @@ from time import sleep -from typing import Any, Dict, Generator +from typing import Any, Dict, Generator, Optional from uuid import uuid4 import pydash @@ -7,22 +7,12 @@ import requests from faker import Faker from requests import Response -from sqlalchemy.orm import Session -from sqlalchemy_utils.functions import create_database, database_exists, drop_database -from fides.api.ctl.sql_models import Dataset as CtlDataset -from fides.api.ops.models.connectionconfig import ( - AccessLevel, - ConnectionConfig, - ConnectionType, +from fides.api.ops.models.connectionconfig import ConnectionConfig +from tests.ops.integration_tests.saas.connector_runner import ( + ConnectorRunner, + generate_random_email, ) -from fides.api.ops.models.datasetconfig import DatasetConfig -from fides.api.ops.util.saas_util import ( - load_config_with_replacement, - load_dataset_with_replacement, -) -from fides.lib.cryptography import cryptographic_util -from tests.ops.test_helpers.db_utils import seed_postgres_data from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets @@ -30,112 +20,54 @@ @pytest.fixture(scope="session") -def yotpo_reviews_secrets(saas_config): +def yotpo_reviews_secrets(saas_config) -> Dict[str, Any]: return { "domain": pydash.get(saas_config, "yotpo_reviews.domain") or secrets["domain"], "store_id": pydash.get(saas_config, "yotpo_reviews.store_id") or secrets["store_id"], "secret_key": pydash.get(saas_config, "yotpo_reviews.secret_key") or secrets["secret_key"], - "yotpo_external_id": { - "dataset": "yotpo_reviews_postgres", - "field": "yotpo_customer.external_id", - "direction": "from", - }, } @pytest.fixture(scope="session") -def yotpo_reviews_identity_email(saas_config): +def yotpo_reviews_identity_email(saas_config) -> str: return ( pydash.get(saas_config, "yotpo_reviews.identity_email") or secrets["identity_email"] ) -@pytest.fixture(scope="function") +@pytest.fixture def yotpo_reviews_erasure_identity_email() -> str: - return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" + return generate_random_email() -@pytest.fixture(scope="session") +@pytest.fixture def yotpo_reviews_erasure_yotpo_external_id() -> str: return f"ext-{uuid4()}" @pytest.fixture -def yotpo_reviews_config() -> Dict[str, Any]: - return load_config_with_replacement( - "data/saas/config/yotpo_reviews_config.yml", - "", - "yotpo_reviews_instance", - ) - - -@pytest.fixture -def yotpo_reviews_dataset() -> Dict[str, Any]: - return load_dataset_with_replacement( - "data/saas/dataset/yotpo_reviews_dataset.yml", - "", - "yotpo_reviews_instance", - )[0] - - -@pytest.fixture(scope="function") -def yotpo_reviews_connection_config( - db: Session, yotpo_reviews_config, yotpo_reviews_secrets -) -> Generator: - fides_key = yotpo_reviews_config["fides_key"] - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": fides_key, - "name": fides_key, - "connection_type": ConnectionType.saas, - "access": AccessLevel.write, - "secrets": yotpo_reviews_secrets, - "saas_config": yotpo_reviews_config, - }, - ) - yield connection_config - connection_config.delete(db) +def yotpo_reviews_external_references() -> Dict[str, Any]: + return {"yotpo_external_id": "ak123798684365sdfkj"} @pytest.fixture -def yotpo_reviews_dataset_config( - db: Session, - yotpo_reviews_connection_config: ConnectionConfig, - yotpo_reviews_dataset: Dict[str, Any], -) -> Generator: - fides_key = yotpo_reviews_dataset["fides_key"] - yotpo_reviews_connection_config.name = fides_key - yotpo_reviews_connection_config.key = fides_key - yotpo_reviews_connection_config.save(db=db) - - ctl_dataset = CtlDataset.create_from_dataset_dict(db, yotpo_reviews_dataset) - - dataset = DatasetConfig.create( - db=db, - data={ - "connection_config_id": yotpo_reviews_connection_config.id, - "fides_key": fides_key, - "ctl_dataset_id": ctl_dataset.id, - }, - ) - yield dataset - dataset.delete(db=db) - ctl_dataset.delete(db=db) +def yotpo_reviews_erasure_external_references( + yotpo_reviews_erasure_yotpo_external_id, +) -> Dict[str, Any]: + return {"yotpo_external_id": yotpo_reviews_erasure_yotpo_external_id} class YotpoReviewsTestClient: - def __init__(self, connection_config: ConnectionConfig): - yotpo_reviews_secrets = connection_config.secrets - self.domain = yotpo_reviews_secrets["domain"] - self.store_id = yotpo_reviews_secrets["store_id"] + def __init__(self, secrets: Dict[str, Any]): + self.domain = secrets["domain"] + self.store_id = secrets["store_id"] response = requests.post( url=f"https://{self.domain}/core/v3/stores/{self.store_id}/access_tokens", json={ - "secret": f"{yotpo_reviews_secrets['secret_key']}", + "secret": f"{secrets['secret_key']}", }, ) assert response.ok @@ -173,124 +105,58 @@ def create_customer(self, external_id: str, email: str) -> Response: }, ) - def get_customer(self, external_id: str) -> Response: - return requests.get( + def get_customer(self, external_id: str) -> Optional[Response]: + response = requests.get( url=f"https://{self.domain}/core/v3/stores/{self.store_id}/customers", headers={"X-Yotpo-Token": self.access_token}, params={"external_ids": external_id}, ) + return response if response.json().get("customers") else None -@pytest.fixture(scope="function") +@pytest.fixture def yotpo_reviews_test_client( - yotpo_reviews_connection_config: ConnectionConfig, + yotpo_reviews_secrets, ) -> Generator: - test_client = YotpoReviewsTestClient(yotpo_reviews_connection_config) + test_client = YotpoReviewsTestClient(yotpo_reviews_secrets) yield test_client -@pytest.fixture(scope="function") +@pytest.fixture def yotpo_reviews_erasure_data( yotpo_reviews_test_client: YotpoReviewsTestClient, yotpo_reviews_erasure_yotpo_external_id, yotpo_reviews_erasure_identity_email, -) -> None: +) -> Generator: # create customer response = yotpo_reviews_test_client.create_customer( yotpo_reviews_erasure_yotpo_external_id, yotpo_reviews_erasure_identity_email ) assert response.ok - # takes a while for this data to propagate, success from poll_for_existence doesn't - # guarantee the data will be available for the actual test - sleep(240) - - -@pytest.fixture() -def yotpo_reviews_postgres_dataset() -> Dict[str, Any]: - return { - "fides_key": "yotpo_reviews_postgres", - "name": "Yotpo Reviews Postgres", - "description": "Lookup for Yotpo Reviews external IDs", - "collections": [ - { - "name": "yotpo_customer", - "fields": [ - { - "name": "email", - "data_categories": ["user.contact.email"], - "fidesops_meta": {"data_type": "string", "identity": "email"}, - }, - { - "name": "external_id", - "fidesops_meta": {"data_type": "string"}, - }, - ], - } - ], - } - - -@pytest.fixture -def yotpo_reviews_postgres_dataset_config( - connection_config: ConnectionConfig, - yotpo_reviews_postgres_dataset: Dict[str, Any], - db: Session, -) -> Generator: - fides_key = yotpo_reviews_postgres_dataset["fides_key"] - connection_config.name = fides_key - connection_config.key = fides_key - connection_config.save(db=db) - - ctl_dataset = CtlDataset.create_from_dataset_dict( - db, yotpo_reviews_postgres_dataset - ) - - dataset = DatasetConfig.create( - db=db, - data={ - "connection_config_id": connection_config.id, - "fides_key": fides_key, - "ctl_dataset_id": ctl_dataset.id, - }, - ) - yield dataset - dataset.delete(db=db) - ctl_dataset.delete(db) - - -@pytest.fixture(scope="function") -def yotpo_reviews_postgres_db(postgres_integration_session): - postgres_integration_session = seed_postgres_data( - postgres_integration_session, - "./tests/fixtures/saas/external_datasets/yotpo_reviews.sql", + poll_for_existence( + yotpo_reviews_test_client.get_customer, + (yotpo_reviews_erasure_yotpo_external_id,), + interval=30, + verification_count=3, ) - yield postgres_integration_session - drop_database(postgres_integration_session.bind.url) + yield yotpo_reviews_erasure_identity_email, yotpo_reviews_erasure_yotpo_external_id -@pytest.fixture(scope="function") -def yotpo_reviews_postgres_erasure_db( - postgres_integration_session, - yotpo_reviews_erasure_identity_email, - yotpo_reviews_erasure_yotpo_external_id, -): - if database_exists(postgres_integration_session.bind.url): - # Postgres cannot drop databases from within a transaction block, so - # we should drop the DB this way instead - drop_database(postgres_integration_session.bind.url) - create_database(postgres_integration_session.bind.url) - create_table_query = "CREATE TABLE public.yotpo_customer (email CHARACTER VARYING(100) PRIMARY KEY, external_id CHARACTER VARYING(100));" - postgres_integration_session.execute(create_table_query) - insert_query = ( - "INSERT INTO public.yotpo_customer VALUES('" - + yotpo_reviews_erasure_identity_email - + "', '" - + yotpo_reviews_erasure_yotpo_external_id - + "')" +@pytest.fixture +def yotpo_reviews_runner( + db, + cache, + yotpo_reviews_secrets, + yotpo_reviews_external_references, + yotpo_reviews_erasure_external_references, +) -> ConnectorRunner: + return ConnectorRunner( + db, + cache, + "yotpo_reviews", + yotpo_reviews_secrets, + external_references=yotpo_reviews_external_references, + erasure_external_references=yotpo_reviews_erasure_external_references, ) - postgres_integration_session.execute(insert_query) - - yield postgres_integration_session - drop_database(postgres_integration_session.bind.url) diff --git a/tests/fixtures/saas/zendesk_fixtures.py b/tests/fixtures/saas/zendesk_fixtures.py index b813dd736b..7c1a70cd49 100644 --- a/tests/fixtures/saas/zendesk_fixtures.py +++ b/tests/fixtures/saas/zendesk_fixtures.py @@ -1,30 +1,21 @@ -from time import sleep from typing import Any, Dict, Generator import pydash import pytest import requests -from sqlalchemy.orm import Session -from fides.api.ctl.sql_models import Dataset as CtlDataset -from fides.api.ops.models.connectionconfig import ( - AccessLevel, - ConnectionConfig, - ConnectionType, +from fides.api.ops.models.connectionconfig import ConnectionConfig +from tests.ops.integration_tests.saas.connector_runner import ( + ConnectorRunner, + generate_random_email, ) -from fides.api.ops.models.datasetconfig import DatasetConfig -from fides.api.ops.util.saas_util import ( - load_config_with_replacement, - load_dataset_with_replacement, -) -from fides.lib.cryptography import cryptographic_util from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("zendesk") @pytest.fixture(scope="session") -def zendesk_secrets(saas_config): +def zendesk_secrets(saas_config) -> Dict[str, Any]: return { "domain": pydash.get(saas_config, "zendesk.domain") or secrets["domain"], "username": pydash.get(saas_config, "zendesk.username") or secrets["username"], @@ -35,119 +26,92 @@ def zendesk_secrets(saas_config): @pytest.fixture(scope="session") -def zendesk_identity_email(saas_config): +def zendesk_identity_email(saas_config) -> str: return ( pydash.get(saas_config, "zendesk.identity_email") or secrets["identity_email"] ) -@pytest.fixture(scope="function") +@pytest.fixture def zendesk_erasure_identity_email() -> str: - return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" + return generate_random_email() + + +class ZendeskClient: + def __init__(self, secrets: Dict[str, Any]): + self.base_url = f"https://{secrets['domain']}" + self.auth = secrets["username"], secrets["api_key"] + + def create_user(self, email): + return requests.post( + url=f"{self.base_url}/api/v2/users", + auth=self.auth, + json={ + "user": { + "name": "Ethyca Test Erasure", + "email": email, + "verified": "true", + } + }, + ) + + def get_user(self, email): + return requests.get( + url=f"{self.base_url}/v2/users", + auth=self.auth, + params={"email": email}, + ) + + def create_ticket(self, user_id: str): + return requests.post( + url=f"{self.base_url}/api/v2/tickets", + auth=self.auth, + json={ + "ticket": { + "comment": {"body": "Test Comment"}, + "priority": "urgent", + "subject": "Test Ticket", + "requester_id": user_id, + "submitter_id": user_id, + "description": "Test Description", + } + }, + ) + + def get_ticket(self, ticket_id: str): + return requests.get( + url=f"{self.base_url}/v2/tickets/{ticket_id}.json", + auth=self.auth, + ) @pytest.fixture -def zendesk_config() -> Dict[str, Any]: - return load_config_with_replacement( - "data/saas/config/zendesk_config.yml", - "", - "zendesk_instance", - ) +def zendesk_client(zendesk_secrets) -> Generator: + yield ZendeskClient(zendesk_secrets) @pytest.fixture -def zendesk_dataset() -> Dict[str, Any]: - return load_dataset_with_replacement( - "data/saas/dataset/zendesk_dataset.yml", - "", - "zendesk_instance", - )[0] - - -@pytest.fixture(scope="function") -def zendesk_connection_config( - db: Session, zendesk_config, zendesk_secrets +def zendesk_erasure_data( + zendesk_client: ZendeskClient, + zendesk_erasure_identity_email: str, ) -> Generator: - fides_key = zendesk_config["fides_key"] - connection_config = ConnectionConfig.create( - db=db, - data={ - "key": fides_key, - "name": fides_key, - "connection_type": ConnectionType.saas, - "access": AccessLevel.write, - "secrets": zendesk_secrets, - "saas_config": zendesk_config, - }, - ) - yield connection_config - connection_config.delete(db) - - -@pytest.fixture -def zendesk_dataset_config( - db: Session, - zendesk_connection_config: ConnectionConfig, - zendesk_dataset: Dict[str, Any], -) -> Generator: - fides_key = zendesk_dataset["fides_key"] - zendesk_connection_config.name = fides_key - zendesk_connection_config.key = fides_key - zendesk_connection_config.save(db=db) - - ctl_dataset = CtlDataset.create_from_dataset_dict(db, zendesk_dataset) - - dataset = DatasetConfig.create( - db=db, - data={ - "connection_config_id": zendesk_connection_config.id, - "fides_key": fides_key, - "ctl_dataset_id": ctl_dataset.id, - }, - ) - yield dataset - dataset.delete(db=db) - ctl_dataset.delete(db=db) - -@pytest.fixture(scope="function") -def zendesk_create_erasure_data( - zendesk_connection_config: ConnectionConfig, zendesk_erasure_identity_email: str -) -> None: - - sleep(60) - - zendesk_secrets = zendesk_connection_config.secrets - auth = zendesk_secrets["username"], zendesk_secrets["api_key"] - base_url = f"https://{zendesk_secrets['domain']}" - - # user - body = { - "user": { - "name": "Ethyca Test Erasure", - "email": zendesk_erasure_identity_email, - "verified": "true", - } - } - - users_response = requests.post(url=f"{base_url}/api/v2/users", auth=auth, json=body) - user = users_response.json()["user"] - user_id = user["id"] + # customer + response = zendesk_client.create_user(zendesk_erasure_identity_email) + assert response.ok + user = response.json()["user"] # ticket - ticket_data = { - "ticket": { - "comment": {"body": "Test Comment"}, - "priority": "urgent", - "subject": "Test Ticket", - "requester_id": user_id, - "submitter_id": user_id, - "description": "Test Description", - } - } - response = requests.post( - url=f"{base_url}/api/v2/tickets", auth=auth, json=ticket_data - ) + response = zendesk_client.create_ticket(user["id"]) + assert response.ok ticket = response.json()["ticket"] - ticket_id = ticket["id"] yield ticket, user + + +@pytest.fixture +def zendesk_runner( + db, + cache, + zendesk_secrets, +) -> ConnectorRunner: + return ConnectorRunner(db, cache, "zendesk", zendesk_secrets) diff --git a/tests/ops/integration_tests/saas/connector_runner.py b/tests/ops/integration_tests/saas/connector_runner.py new file mode 100644 index 0000000000..685e0c154f --- /dev/null +++ b/tests/ops/integration_tests/saas/connector_runner.py @@ -0,0 +1,343 @@ +import random +from typing import Any, Dict, List, Optional, Tuple + +from sqlalchemy.orm import Session + +from fides.api.ctl.sql_models import Dataset as CtlDataset +from fides.api.ops.graph.config import GraphDataset +from fides.api.ops.graph.graph import DatasetGraph +from fides.api.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fides.api.ops.models.datasetconfig import DatasetConfig +from fides.api.ops.models.policy import Policy +from fides.api.ops.models.privacy_request import PrivacyRequest +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.service.connectors import get_connector +from fides.api.ops.task import graph_task +from fides.api.ops.task.graph_task import get_cached_data_for_erasures +from fides.api.ops.util.cache import FidesopsRedis +from fides.api.ops.util.collection_util import Row +from fides.api.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) +from fides.core.config import get_config +from fides.lib.cryptography import cryptographic_util + +CONFIG = get_config() + + +class ConnectorRunner: + """ + A util class responsible for creating the entities required for + testing connectivity and access/erasure requests for a SaaS connector + """ + + def __init__( + self, + db, + cache: FidesopsRedis, + connector_type: str, + secrets: Dict[str, Any], + external_references: Optional[Dict[str, Any]] = None, + erasure_external_references: Optional[Dict[str, Any]] = None, + ): + self.db = db + self.cache = cache + self.connector_type = connector_type + self.external_references = external_references + self.erasure_external_references = erasure_external_references + + # load the config and dataset from the yaml files + self.config = _config(connector_type) + self.dataset = _dataset(connector_type) + + # update secrets with external reference dataset + for external_reference in self.config.get("external_references", []): + external_reference_name = external_reference["name"] + secrets[external_reference_name] = { + "dataset": f"{connector_type}_external_dataset", + "field": f"{connector_type}_external_collection.{external_reference_name}", + "direction": "from", + } + + # create and save the connection config and dataset config to the database + self.connection_config = _connection_config(db, self.config, secrets) + self.dataset_config = _dataset_config(db, self.connection_config, self.dataset) + + def test_connection(self): + """Connection test using the connectors test_request""" + get_connector(self.connection_config).test_connection() + + async def access_request( + self, + access_policy: Policy, + identities: Dict[str, Any], + ) -> Dict[str, List[Row]]: + """Access request for a given access policy and identities""" + fides_key = self.connection_config.key + privacy_request = PrivacyRequest( + id=f"test_{fides_key}_access_request_{random.randint(0, 1000)}" + ) + identity = Identity(**identities) + privacy_request.cache_identity(identity) + + # cache external dataset data + if self.external_references: + self.cache.set_encoded_object( + f"{privacy_request.id}__access_request__{self.connector_type}_external_dataset:{self.connector_type}_external_collection", + [self.external_references], + ) + + graph_list = [self.dataset_config.get_graph()] + connection_config_list = [self.connection_config] + _process_external_references(self.db, graph_list, connection_config_list) + dataset_graph = DatasetGraph(*graph_list) + + access_results = await graph_task.run_access_request( + privacy_request, + access_policy, + dataset_graph, + connection_config_list, + identities, + self.db, + ) + # verify we returned at least one row for each collection in the dataset + for collection in self.dataset["collections"]: + assert len(access_results[f"{fides_key}:{collection['name']}"]) + return access_results + + async def strict_erasure_request( + self, + access_policy: Policy, + erasure_policy: Policy, + identities: Dict[str, Any], + ) -> Tuple[Dict, Dict]: + """ + Erasure request with masking_strict set to true, + meaning we will only update data, not delete it + """ + + # store the existing masking_strict value so we can reset it at the end of the test + masking_strict = CONFIG.execution.masking_strict + CONFIG.execution.masking_strict = True + + access_results, erasure_results = await self._base_erasure_request( + access_policy, erasure_policy, identities + ) + + # reset masking_strict value + CONFIG.execution.masking_strict = masking_strict + return access_results, erasure_results + + async def non_strict_erasure_request( + self, + access_policy: Policy, + erasure_policy: Policy, + identities: Dict[str, Any], + ) -> Tuple[Dict, Dict]: + """ + Erasure request with masking_strict set to false, + meaning we will use deletes to mask data if an update + is not available + """ + + # store the existing masking_strict value so we can reset it at the end of the test + masking_strict = CONFIG.execution.masking_strict + CONFIG.execution.masking_strict = False + + access_results, erasure_results = await self._base_erasure_request( + access_policy, erasure_policy, identities + ) + + # reset masking_strict value + CONFIG.execution.masking_strict = masking_strict + return access_results, erasure_results + + async def _base_erasure_request( + self, + access_policy: Policy, + erasure_policy: Policy, + identities: Dict[str, Any], + ) -> Tuple[Dict, Dict]: + + fides_key = self.connection_config.key + privacy_request = PrivacyRequest( + id=f"test_{fides_key}_access_request_{random.randint(0, 1000)}" + ) + identity = Identity(**identities) + privacy_request.cache_identity(identity) + + # cache external dataset data + if self.erasure_external_references: + self.cache.set_encoded_object( + f"{privacy_request.id}__access_request__{self.connector_type}_external_dataset:{self.connector_type}_external_collection", + [self.erasure_external_references], + ) + + graph_list = [self.dataset_config.get_graph()] + connection_config_list = [self.connection_config] + _process_external_references(self.db, graph_list, connection_config_list) + dataset_graph = DatasetGraph(*graph_list) + + access_results = await graph_task.run_access_request( + privacy_request, + access_policy, + dataset_graph, + connection_config_list, + identities, + self.db, + ) + + # verify we returned at least one row for each collection in the dataset + for collection in self.dataset["collections"]: + assert len(access_results[f"{fides_key}:{collection['name']}"]) + + erasure_results = await graph_task.run_erasure( + privacy_request, + erasure_policy, + dataset_graph, + connection_config_list, + identities, + get_cached_data_for_erasures(privacy_request.id), + self.db, + ) + + return access_results, erasure_results + + +def _config(connector_type: str) -> Dict[str, Any]: + return load_config_with_replacement( + f"data/saas/config/{connector_type}_config.yml", + "", + f"{connector_type}_instance", + ) + + +def _dataset(connector_type: str) -> Dict[str, Any]: + return load_dataset_with_replacement( + f"data/saas/dataset/{connector_type}_dataset.yml", + "", + f"{connector_type}_instance", + )[0] + + +def _connection_config(db: Session, config, secrets) -> ConnectionConfig: + fides_key = config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": secrets, + "saas_config": config, + }, + ) + return connection_config + + +def _external_connection_config(db: Session, fides_key) -> ConnectionConfig: + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.postgres, + "access": AccessLevel.write, + }, + ) + return connection_config + + +def _dataset_config( + db: Session, + connection_config: ConnectionConfig, + dataset: Dict[str, Any], +) -> DatasetConfig: + fides_key = dataset["fides_key"] + connection_config.name = fides_key + connection_config.key = fides_key + connection_config.save(db=db) + + ctl_dataset = CtlDataset.create_from_dataset_dict(db, dataset) + + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": connection_config.id, + "fides_key": fides_key, + "ctl_dataset_id": ctl_dataset.id, + }, + ) + return dataset + + +def _external_dataset( + connector_type: str, external_references: List[Dict[str, Any]] +) -> Dict[str, Any]: + """Generate a dataset that contains each external reference as a collection field""" + # define the email field as an entry point to this dataset + fields = [ + { + "name": "email", + "data_categories": ["user.contact.email"], + "fidesops_meta": {"data_type": "string", "identity": "email"}, + } + ] + # add every external reference as an additional field in this collection + for external_reference in external_references: + fields.append( + { + "name": external_reference["name"], + "fidesops_meta": {"data_type": "string"}, + } + ) + return { + "fides_key": f"{connector_type}_external_dataset", + "name": f"{connector_type}_external_dataset", + "description": f"{connector_type}_external_dataset", + "collections": [ + { + "name": f"{connector_type}_external_collection", + "fields": fields, + } + ], + } + + +def _process_external_references( + db: Session, + graph_list: List[GraphDataset], + connection_config_list: List[ConnectionConfig], +): + """ + Read the external references from the base connection config + and generate the connection config and dataset to represent + the external references + """ + # we start with the base connection config + connection_config = connection_config_list[0] + if connection_config.saas_config.get("external_references"): + connector_type = connection_config.saas_config["type"] + external_connection_config = _external_connection_config( + db, f"{connector_type}_external_dataset" + ) + graph_list.append( + _dataset_config( + db, + external_connection_config, + _external_dataset( + connector_type, connection_config.saas_config["external_references"] + ), + ).get_graph() + ) + connection_config_list.append(external_connection_config) + + +def generate_random_email() -> str: + return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" diff --git a/tests/ops/integration_tests/saas/test_doordash_task.py b/tests/ops/integration_tests/saas/test_doordash_task.py index 3468ab06d9..1f1269a8f9 100644 --- a/tests/ops/integration_tests/saas/test_doordash_task.py +++ b/tests/ops/integration_tests/saas/test_doordash_task.py @@ -1,87 +1,20 @@ -import random - import pytest -from fides.api.ops.graph.graph import DatasetGraph -from fides.api.ops.models.privacy_request import PrivacyRequest -from fides.api.ops.schemas.redis_cache import Identity -from fides.api.ops.service.connectors import get_connector -from fides.api.ops.task import graph_task -from tests.ops.graph.graph_test_util import assert_rows_match - - -@pytest.mark.integration_saas -@pytest.mark.integration_doordash -def test_doordash_connection_test(doordash_connection_config) -> None: - get_connector(doordash_connection_config).test_connection() +from fides.api.ops.models.policy import Policy +from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner @pytest.mark.integration_saas -@pytest.mark.integration_doordash -@pytest.mark.asyncio -async def test_doordash_access_request_task( - db, - policy, - doordash_connection_config, - doordash_dataset_config, - doordash_identity_email, - connection_config, - doordash_postgres_dataset_config, - doordash_postgres_db, -) -> None: - """Full access request based on the Doordash SaaS config""" - - privacy_request = PrivacyRequest( - id=f"test_doordash_access_request_task_{random.randint(0, 1000)}" - ) - identity_attribute = "email" - identity_value = doordash_identity_email - identity_kwargs = {identity_attribute: identity_value} - identity = Identity(**identity_kwargs) - privacy_request.cache_identity(identity) - - dataset_name = doordash_connection_config.get_saas_config().fides_key - merged_graph = doordash_dataset_config.get_graph() - graph = DatasetGraph(*[merged_graph, doordash_postgres_dataset_config.get_graph()]) - - v = await graph_task.run_access_request( - privacy_request, - policy, - graph, - [doordash_connection_config, connection_config], - {"email": doordash_identity_email}, - db, - ) - - assert_rows_match( - v[f"{dataset_name}:deliveries"], - min_size=1, - keys=[ - "external_delivery_id", - "currency", - "delivery_status", - "fee", - "pickup_address", - "pickup_phone_number", - "pickup_instructions", - "pickup_reference_tag", - "dropoff_address", - "dropoff_business_name", - "dropoff_phone_number", - "dropoff_instructions", - "dropoff_contact_given_name", - "dropoff_contact_family_name", - "dropoff_contact_send_notifications", - "order_value", - "cancellation_reason", - "updated_at", - "pickup_time_estimated", - "dropoff_time_estimated", - "support_reference", - "tracking_url", - "contactless_dropoff", - "action_if_undeliverable", - "tip", - "order_contains", - ], - ) +class TestDoordashConnector: + def test_connection(self, doordash_runner: ConnectorRunner): + doordash_runner.test_connection() + + async def test_access_request( + self, + doordash_runner: ConnectorRunner, + policy: Policy, + doordash_identity_email: str, + ): + await doordash_runner.access_request( + access_policy=policy, identities={"email": doordash_identity_email} + ) diff --git a/tests/ops/integration_tests/saas/test_yotpo_reviews_task.py b/tests/ops/integration_tests/saas/test_yotpo_reviews_task.py index 10c9f18ee7..b3b4f8ade3 100644 --- a/tests/ops/integration_tests/saas/test_yotpo_reviews_task.py +++ b/tests/ops/integration_tests/saas/test_yotpo_reviews_task.py @@ -1,174 +1,53 @@ -import random from time import sleep import pytest -from fides.api.ops.graph.graph import DatasetGraph -from fides.api.ops.models.privacy_request import PrivacyRequest -from fides.api.ops.schemas.redis_cache import Identity -from fides.api.ops.service.connectors import get_connector -from fides.api.ops.task import graph_task -from fides.api.ops.task.graph_task import get_cached_data_for_erasures -from fides.core.config import get_config -from tests.ops.graph.graph_test_util import assert_rows_match - -CONFIG = get_config() - - -@pytest.mark.integration_saas -@pytest.mark.integration_yotpo -def test_yotpo_reviews_connection_test(yotpo_reviews_connection_config) -> None: - get_connector(yotpo_reviews_connection_config).test_connection() - - -@pytest.mark.integration_saas -@pytest.mark.integration_yotpo -@pytest.mark.asyncio -async def test_yotpo_reviews_access_request_task( - db, - policy, - yotpo_reviews_connection_config, - yotpo_reviews_dataset_config, - yotpo_reviews_identity_email, - connection_config, - yotpo_reviews_postgres_dataset_config, - yotpo_reviews_postgres_db, -) -> None: - """Full access request based on the Yotpo Reviews SaaS config""" - - privacy_request = PrivacyRequest( - id=f"test_yotpo_reviews_access_request_task_{random.randint(0, 1000)}" - ) - identity = Identity(**{"email": yotpo_reviews_identity_email}) - privacy_request.cache_identity(identity) - - dataset_name = yotpo_reviews_connection_config.get_saas_config().fides_key - merged_graph = yotpo_reviews_dataset_config.get_graph() - graph = DatasetGraph( - *[merged_graph, yotpo_reviews_postgres_dataset_config.get_graph()] - ) - - v = await graph_task.run_access_request( - privacy_request, - policy, - graph, - [yotpo_reviews_connection_config, connection_config], - {"email": yotpo_reviews_identity_email}, - db, - ) - - assert_rows_match( - v[f"{dataset_name}:customer"], - min_size=1, - keys=[ - "external_id", - "email", - "phone_number", - "first_name", - "last_name", - "gender", - "account_created_at", - "account_status", - "default_language", - "default_currency", - "tags", - "address", - "custom_properties", - "accepts_email_marketing", - "accepts_sms_marketing", - ], - ) +from fides.api.ops.models.policy import Policy +from tests.fixtures.saas.yotpo_reviews_fixtures import YotpoReviewsTestClient +from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner @pytest.mark.integration_saas -@pytest.mark.integration_yotpo -@pytest.mark.asyncio -async def test_yotpo_reviews_erasure_request_task( - db, - policy, - erasure_policy_string_rewrite, - yotpo_reviews_connection_config, - yotpo_reviews_dataset_config, - yotpo_reviews_erasure_identity_email, - yotpo_reviews_erasure_yotpo_external_id, - yotpo_reviews_postgres_dataset_config, - connection_config, - yotpo_reviews_postgres_erasure_db, - yotpo_reviews_erasure_data, - yotpo_reviews_test_client, -) -> None: - """Full erasure request based on the Yotpo Reviews SaaS config""" - - masking_strict = CONFIG.execution.masking_strict - CONFIG.execution.masking_strict = True - - privacy_request = PrivacyRequest( - id=f"test_yotpo_reviews_erasure_request_task_{random.randint(0, 1000)}" - ) - identity = Identity(**{"email": yotpo_reviews_erasure_identity_email}) - privacy_request.cache_identity(identity) +class TestYotpoReviewsConnector: + def test_connection(self, yotpo_reviews_runner: ConnectorRunner): + yotpo_reviews_runner.test_connection() - dataset_name = yotpo_reviews_connection_config.get_saas_config().fides_key - merged_graph = yotpo_reviews_dataset_config.get_graph() - graph = DatasetGraph( - *[merged_graph, yotpo_reviews_postgres_dataset_config.get_graph()] - ) - - v = await graph_task.run_access_request( - privacy_request, + async def test_access_request( + self, + yotpo_reviews_runner: ConnectorRunner, policy, - graph, - [yotpo_reviews_connection_config, connection_config], - {"email": yotpo_reviews_erasure_identity_email}, - db, - ) - - assert_rows_match( - v[f"{dataset_name}:customer"], - min_size=1, - keys=[ - "external_id", - "email", - "phone_number", - "first_name", - "last_name", - "gender", - "account_created_at", - "account_status", - "default_language", - "default_currency", - "tags", - "address", - "custom_properties", - "accepts_email_marketing", - "accepts_sms_marketing", - ], - ) - - x = await graph_task.run_erasure( - privacy_request, - erasure_policy_string_rewrite, - graph, - [yotpo_reviews_connection_config, connection_config], - {"email": yotpo_reviews_erasure_identity_email}, - get_cached_data_for_erasures(privacy_request.id), - db, - ) - - assert x == { - "yotpo_reviews_instance:customer": 1, - "yotpo_reviews_postgres:yotpo_customer": 0, - } - - sleep(120) - - response = yotpo_reviews_test_client.get_customer( - yotpo_reviews_erasure_yotpo_external_id - ) - assert response.ok - - customer = response.json()["customers"][0] - assert customer["first_name"] == "MASKED" - assert customer["last_name"] == "MASKED" - - CONFIG.execution.masking_strict = masking_strict + yotpo_reviews_identity_email: str, + ): + await yotpo_reviews_runner.access_request( + access_policy=policy, identities={"email": yotpo_reviews_identity_email} + ) + + async def test_strict_erasure_request( + self, + yotpo_reviews_runner: ConnectorRunner, + policy: Policy, + erasure_policy_string_rewrite: Policy, + yotpo_reviews_erasure_data, + yotpo_reviews_test_client: YotpoReviewsTestClient, + ): + email, external_id = yotpo_reviews_erasure_data + (_, erasure_results) = await yotpo_reviews_runner.strict_erasure_request( + access_policy=policy, + erasure_policy=erasure_policy_string_rewrite, + identities={"email": email}, + ) + + assert erasure_results == { + "yotpo_reviews_instance:customer": 1, + "yotpo_reviews_external_dataset:yotpo_reviews_external_collection": 0, + } + + # wait for the update to propagate + sleep(180) + + response = yotpo_reviews_test_client.get_customer(external_id) + assert response and response.ok + + customer = response.json()["customers"][0] + assert customer["first_name"] == "MASKED" + assert customer["last_name"] == "MASKED" diff --git a/tests/ops/integration_tests/saas/test_zendesk_task.py b/tests/ops/integration_tests/saas/test_zendesk_task.py index be67754e4a..156cb1a8b5 100644 --- a/tests/ops/integration_tests/saas/test_zendesk_task.py +++ b/tests/ops/integration_tests/saas/test_zendesk_task.py @@ -1,345 +1,72 @@ -import random -import time - import pytest -import requests - -from fides.api.ops.graph.graph import DatasetGraph -from fides.api.ops.models.privacy_request import PrivacyRequest -from fides.api.ops.schemas.redis_cache import Identity -from fides.api.ops.service.connectors import get_connector -from fides.api.ops.task import graph_task -from fides.api.ops.task.graph_task import get_cached_data_for_erasures -from fides.core.config import CONFIG -from tests.ops.graph.graph_test_util import assert_rows_match - -@pytest.mark.integration_saas -@pytest.mark.integration_zendesk -def test_zendesk_connection_test(zendesk_connection_config) -> None: - get_connector(zendesk_connection_config).test_connection() +from fides.api.ops.models.policy import Policy +from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner @pytest.mark.integration_saas -@pytest.mark.integration_zendesk -@pytest.mark.asyncio -async def test_zendesk_access_request_task( - db, - policy, - zendesk_connection_config, - zendesk_dataset_config, - zendesk_identity_email, -) -> None: - """Full access request based on the Zendesk SaaS config""" - - privacy_request = PrivacyRequest( - id=f"test_zendesk_access_request_task_{random.randint(0, 1000)}" - ) - identity = Identity(**{"email": zendesk_identity_email}) - privacy_request.cache_identity(identity) - - dataset_name = zendesk_connection_config.get_saas_config().fides_key - merged_graph = zendesk_dataset_config.get_graph() - graph = DatasetGraph(merged_graph) - - v = await graph_task.run_access_request( - privacy_request, - policy, - graph, - [zendesk_connection_config], - {"email": zendesk_identity_email}, - db, - ) - - assert_rows_match( - v[f"{dataset_name}:users"], - min_size=1, - keys=[ - "id", - "url", - "name", - "email", - "created_at", - "updated_at", - "time_zone", - "iana_time_zone", - "phone", - "shared_phone_number", - "photo", - "locale_id", - "locale", - "organization_id", - "role", - "verified", - "external_id", - "tags", - "alias", - "active", - "shared", - "shared_agent", - "last_login_at", - "two_factor_auth_enabled", - "signature", - "details", - "notes", - "role_type", - "custom_role_id", - "moderator", - "ticket_restriction", - "only_private_comments", - "restricted_agent", - "suspended", - "default_group_id", - "report_csv", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:user_identities"], - min_size=2, - keys=[ - "url", - "id", - "user_id", - "type", - "value", - "verified", - "primary", - "created_at", - "updated_at", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:tickets"], - min_size=1, - keys=[ - "url", - "id", - "external_id", - "via", - "created_at", - "updated_at", - "type", - "subject", - "raw_subject", - "description", - "priority", - "status", - "recipient", - "requester_id", - "submitter_id", - "assignee_id", - "organization_id", - "group_id", - "collaborator_ids", - "follower_ids", - "email_cc_ids", - "forum_topic_id", - "problem_id", - "has_incidents", - "is_public", - "due_at", - "tags", - "custom_fields", - "satisfaction_rating", - "sharing_agreement_ids", - "followup_ids", - "brand_id", - "allow_channelback", - "allow_attachments", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:ticket_comments"], - min_size=1, - keys=[ - "id", - "type", - "author_id", - "body", - "html_body", - "plain_body", - "public", - "attachments", - "audit_id", - "via", - "created_at", - "metadata", - ], - ) - - # verify we only returned data for our identity email - assert v[f"{dataset_name}:users"][0]["email"] == zendesk_identity_email - user_id = v[f"{dataset_name}:users"][0]["id"] - - assert v[f"{dataset_name}:user_identities"][0]["value"] == zendesk_identity_email - - for ticket in v[f"{dataset_name}:tickets"]: - assert ticket["requester_id"] == user_id - - for ticket_comment in v[f"{dataset_name}:ticket_comments"]: - assert ticket_comment["author_id"] == user_id - - -@pytest.mark.integration_saas -@pytest.mark.integration_zendesk -@pytest.mark.asyncio -async def test_zendesk_erasure_request_task( - db, - policy, - erasure_policy_string_rewrite, - zendesk_connection_config, - zendesk_dataset_config, - zendesk_erasure_identity_email, - zendesk_create_erasure_data, -) -> None: - """Full erasure request based on the Zendesk SaaS config""" - - masking_strict = CONFIG.execution.masking_strict - CONFIG.execution.masking_strict = False # Allow Delete - - privacy_request = PrivacyRequest( - id=f"test_zendesk_erasure_request_task_{random.randint(0, 1000)}" - ) - identity = Identity(**{"email": zendesk_erasure_identity_email}) - privacy_request.cache_identity(identity) - - dataset_name = zendesk_connection_config.get_saas_config().fides_key - merged_graph = zendesk_dataset_config.get_graph() - graph = DatasetGraph(merged_graph) - - v = await graph_task.run_access_request( - privacy_request, - policy, - graph, - [zendesk_connection_config], - {"email": zendesk_erasure_identity_email}, - db, - ) - - assert_rows_match( - v[f"{dataset_name}:users"], - min_size=1, - keys=[ - "id", - "url", - "name", - "email", - "created_at", - "updated_at", - "time_zone", - "iana_time_zone", - "phone", - "shared_phone_number", - "photo", - "locale_id", - "locale", - "organization_id", - "role", - "verified", - "external_id", - "tags", - "alias", - "active", - "shared", - "shared_agent", - "last_login_at", - "two_factor_auth_enabled", - "signature", - "details", - "notes", - "role_type", - "custom_role_id", - "moderator", - "ticket_restriction", - "only_private_comments", - "restricted_agent", - "suspended", - "default_group_id", - "report_csv", - ], - ) - - assert_rows_match( - v[f"{dataset_name}:tickets"], - min_size=1, - keys=[ - "url", - "id", - "external_id", - "via", - "created_at", - "updated_at", - "type", - "subject", - "raw_subject", - "description", - "priority", - "status", - "recipient", - "requester_id", - "submitter_id", - "assignee_id", - "organization_id", - "group_id", - "collaborator_ids", - "follower_ids", - "email_cc_ids", - "forum_topic_id", - "problem_id", - "has_incidents", - "is_public", - "due_at", - "tags", - "custom_fields", - "satisfaction_rating", - "sharing_agreement_ids", - "followup_ids", - "brand_id", - "allow_channelback", - "allow_attachments", - ], - ) +class TestZendeskConnector: + def test_connection(self, zendesk_runner: ConnectorRunner): + zendesk_runner.test_connection() + + async def test_access_request( + self, + zendesk_runner: ConnectorRunner, + policy: Policy, + zendesk_identity_email: str, + ): + access_results = await zendesk_runner.access_request( + access_policy=policy, identities={"email": zendesk_identity_email} + ) - x = await graph_task.run_erasure( - privacy_request, - erasure_policy_string_rewrite, - graph, - [zendesk_connection_config], - {"email": zendesk_erasure_identity_email}, - get_cached_data_for_erasures(privacy_request.id), - db, - ) + # verify we only returned data for our identity email + assert ( + access_results["zendesk_instance:user"][0]["email"] + == zendesk_identity_email + ) + user_id = access_results["zendesk_instance:user"][0]["id"] - assert x == { - f"{dataset_name}:users": 1, - f"{dataset_name}:user_identities": 0, - f"{dataset_name}:tickets": 1, - f"{dataset_name}:ticket_comments": 0, - } + assert ( + access_results["zendesk_instance:user_identities"][0]["value"] + == zendesk_identity_email + ) - zendesk_secrets = zendesk_connection_config.secrets - auth = zendesk_secrets["username"], zendesk_secrets["api_key"] - base_url = f"https://{zendesk_secrets['domain']}" + for ticket in access_results["zendesk_instance:tickets"]: + assert ticket["requester_id"] == user_id + + for ticket_comment in access_results["zendesk_instance:ticket_comments"]: + assert ticket_comment["author_id"] == user_id + + async def test_non_strict_erasure_request( + self, + zendesk_runner: ConnectorRunner, + policy: Policy, + erasure_policy_string_rewrite: Policy, + zendesk_erasure_identity_email: str, + zendesk_erasure_data, + zendesk_client, + ): + ( + access_results, + erasure_results, + ) = await zendesk_runner.non_strict_erasure_request( + access_policy=policy, + erasure_policy=erasure_policy_string_rewrite, + identities={"email": zendesk_erasure_identity_email}, + ) - # user - response = requests.get( - url=f"{base_url}/v2/users", - auth=auth, - params={"email": zendesk_erasure_identity_email}, - ) - # Since user is deleted, it won't be available so response is 404 - assert response.status_code == 404 + assert erasure_results == { + "zendesk_instance:user": 1, + "zendesk_instance:user_identities": 0, + "zendesk_instance:tickets": 1, + "zendesk_instance:ticket_comments": 0, + } - for ticket in v[f"{dataset_name}:tickets"]: - ticket_id = ticket["id"] - response = requests.get( - url=f"{base_url}/v2/tickets/{ticket_id}.json", - auth=auth, - ) - # Since ticket is deleted, it won't be available so response is 404 + response = zendesk_client.get_user(zendesk_erasure_identity_email) + # Since user is deleted, it won't be available so response is 404 assert response.status_code == 404 - CONFIG.execution.masking_strict = masking_strict + for ticket in access_results["zendesk_instance:tickets"]: + response = zendesk_client.get_ticket(ticket["id"]) + # Since ticket is deleted, it won't be available so response is 404 + assert response.status_code == 404 diff --git a/tests/ops/test_helpers/saas_test_utils.py b/tests/ops/test_helpers/saas_test_utils.py index a2ee393c75..e617cf0d14 100644 --- a/tests/ops/test_helpers/saas_test_utils.py +++ b/tests/ops/test_helpers/saas_test_utils.py @@ -14,12 +14,15 @@ def poll_for_existence( retries: int = 10, interval: int = 5, existence_desired=True, + verification_count=1, ) -> Any: # we continue polling if poller is None OR if poller is not None but we don't desire existence, i.e. we are polling for removal - while (return_val := poller(*args, **kwargs) is None) is existence_desired: - if not retries: - raise Exception(error_message) - retries -= 1 - time.sleep(interval) - + original_retries = retries + for _ in range(verification_count): + while (return_val := poller(*args, **kwargs) is None) is existence_desired: + if not retries: + raise Exception(error_message) + retries -= 1 + time.sleep(interval) + retries = original_retries return return_val diff --git a/tests/ops/test_helpers/vault_client.py b/tests/ops/test_helpers/vault_client.py index 23fdb31934..e827bbdfd2 100644 --- a/tests/ops/test_helpers/vault_client.py +++ b/tests/ops/test_helpers/vault_client.py @@ -1,5 +1,5 @@ import os -from typing import Any, Dict, Optional +from typing import Any, Dict from hvac import Client from loguru import logger @@ -26,12 +26,13 @@ raise FidesopsException(f"Unable to create Vault client: {str(exc)}") -def get_secrets(connector: str) -> Optional[Dict[str, Any]]: +def get_secrets(connector: str) -> Dict[str, Any]: """Returns a map of secrets for the given connector.""" + secrets_map: Dict[str, Any] = {} + if not _client: - return + return secrets_map - secrets_map = {} try: secrets = _client.secrets.kv.v2.read_secret_version( mount_point=_environment, path=connector