From f0da0631f5563e632902339e19b0fecdac19b8e0 Mon Sep 17 00:00:00 2001 From: Noonari Date: Thu, 12 Jan 2023 08:40:27 +0500 Subject: [PATCH] 1388 recharge erasure (#1709) --- CHANGELOG.md | 1 + data/saas/config/recharge_config.yml | 82 ++++++ data/saas/dataset/recharge_dataset.yml | 171 ++++++++++++ data/saas/saas_connector_registry.toml | 8 +- pyproject.toml | 1 + tests/ops/fixtures/saas/recharge_fixtures.py | 238 ++++++++++++++++ tests/ops/fixtures/saas_example_fixtures.py | 4 +- .../saas/test_recharge_tasks.py | 256 ++++++++++++++++++ 8 files changed, 758 insertions(+), 3 deletions(-) create mode 100644 data/saas/config/recharge_config.yml create mode 100644 data/saas/dataset/recharge_dataset.yml create mode 100644 tests/ops/fixtures/saas/recharge_fixtures.py create mode 100644 tests/ops/integration_tests/saas/test_recharge_tasks.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d3f4b75fef..60a6dbcca3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The types of changes are: * `Fides.shopify` integration function. [#2152](https://github.com/ethyca/fides/pull/2152) * Dedicated folder for integrations. * Adds support for Twilio email service (Sendgrid) [#2154](https://github.com/ethyca/fides/pull/2154) +* Access and erasure support for Recharge [#1709](https://github.com/ethyca/fides/pull/1709) ### Changed diff --git a/data/saas/config/recharge_config.yml b/data/saas/config/recharge_config.yml new file mode 100644 index 0000000000..9f249a06d0 --- /dev/null +++ b/data/saas/config/recharge_config.yml @@ -0,0 +1,82 @@ +saas_config: + fides_key: + name: Recharge SaaS Config + type: recharge + description: A sample schema representing the Recharge connector for Fides + version: 0.0.1 + + connector_params: + - name: domain + default_value: api.rechargeapps.com + - name: api_key + + client_config: + protocol: https + host: + authentication: + strategy: api_key + configuration: + headers: + - name: X-Recharge-Access-Token + value: + + test_request: + method: GET + path: / + + endpoints: + - name: customer + requests: + read: + method: GET + path: /customers + query_params: + - name: email + value: + param_values: + - name: email + identity: email + data_path: customers + update: + method: PUT + path: /customers/ + body: | + { + + } + param_values: + - name: customer_id + references: + - dataset: + field: customer.id + direction: from + - name: addresses + requests: + read: + method: GET + path: /addresses + query_params: + - name: customer_id + value: + param_values: + - name: customer_id + references: + - dataset: + field: customer.id + direction: from + data_path: addresses + update: + method: PUT + path: /addresses/ + body: | + { + "address": { + + } + } + param_values: + - name: address_id + references: + - dataset: + field: addresses.id + direction: from diff --git a/data/saas/dataset/recharge_dataset.yml b/data/saas/dataset/recharge_dataset.yml new file mode 100644 index 0000000000..e790d2987e --- /dev/null +++ b/data/saas/dataset/recharge_dataset.yml @@ -0,0 +1,171 @@ +dataset: + - fides_key: + name: Recharge Dataset + description: A sample dataset representing the Recharge connector for Fides + collections: + - name: customer + fields: + - name: billing_address1 + data_categories: [user.contact.address] + fidesops_meta: + data_type: string + - name: billing_address2 + data_categories: [user.contact.address] + fidesops_meta: + data_type: string + - name: billing_city + data_categories: [user.contact.address.city] + fidesops_meta: + data_type: string + - name: billing_country + data_categories: [user.contact.address.country] + fidesops_meta: + data_type: string + - name: billing_phone + data_categories: [user.contact.phone_number] + fidesops_meta: + data_type: string + - name: billing_province + data_categories: [user.contact.address.state] + fidesops_meta: + data_type: string + - name: billing_zip + data_categories: [user.contact.address.postal_code] + fidesops_meta: + data_type: string + - name: created_at + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: email + data_categories: [user.contact.email] + fidesops_meta: + data_type: string + - name: first_charge_processed_at + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: first_name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: has_card_error_in_dunning + data_categories: [system.operations] + fidesops_meta: + data_type: boolean + - name: has_valid_payment_method + data_categories: [system.operations] + fidesops_meta: + data_type: boolean + - name: hash + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: id + data_categories: [system.operations] + fidesops_meta: + data_type: string + primary_key: True + - name: last_name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: number_active_subscriptions + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: number_subscriptions + data_categories: [system.operations] + fidesops_meta: + data_type: integer + - name: phone + data_categories: [user.contact.phone_number] + fidesops_meta: + data_type: string + - name: processor_type + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: reason_payment_method_not_valid + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: shopify_customer_id + data_categories: [system.operations] + fidesops_meta: + data_type: object + - name: status + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: tax_exempt + data_categories: [system.operations] + fidesops_meta: + data_type: boolean + - name: updated_at + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: addresses + fields: + - name: address1 + data_categories: [user.contact.address] + fidesops_meta: + data_type: string + - name: address2 + data_categories: [user.contact.address] + fidesops_meta: + data_type: string + - name: city + data_categories: [user.contact.address.city] + fidesops_meta: + data_type: string + - name: company + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: country + data_categories: [user.contact.address.country] + fidesops_meta: + data_type: string + - name: created_at + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: customer_id + data_categories: [user.unique_id] + fidesops_meta: + data_type: integer + - name: first_name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: id + data_categories: [system.operations] + fidesops_meta: + data_type: integer + primary_key: True + - name: last_name + data_categories: [user.name] + fidesops_meta: + data_type: string + - name: phone + data_categories: [user.contact.phone_number] + fidesops_meta: + data_type: string + - name: presentment_currency + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: province + data_categories: [user.contact.address.state] + fidesops_meta: + data_type: string + - name: updated_at + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: zip + data_categories: [user.contact.address.postal_code] + fidesops_meta: + data_type: string diff --git a/data/saas/saas_connector_registry.toml b/data/saas/saas_connector_registry.toml index 0e281b1ea6..d1b1727b80 100644 --- a/data/saas/saas_connector_registry.toml +++ b/data/saas/saas_connector_registry.toml @@ -134,4 +134,10 @@ human_readable = "Domo" config = "data/saas/config/slack_enterprise_config.yml" dataset = "data/saas/dataset/slack_enterprise_dataset.yml" icon = "data/saas/icon/slack.svg" -human_readable = "Slack Enterprise" \ No newline at end of file +human_readable = "Slack Enterprise" + +[recharge] +config = "data/saas/config/recharge_config.yml" +dataset = "data/saas/dataset/recharge_dataset.yml" +icon = "data/saas/icon/default.svg" +human_readable = "Recharge" diff --git a/pyproject.toml b/pyproject.toml index 416767bd25..95bf5d738e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -198,6 +198,7 @@ markers = [ "integration_shopify", "integration_square", "integration_outreach", + "integration_recharge", "integration_salesforce", "integration_twilio_conversations", "integration_adobe_campaign", diff --git a/tests/ops/fixtures/saas/recharge_fixtures.py b/tests/ops/fixtures/saas/recharge_fixtures.py new file mode 100644 index 0000000000..3c3944b4b5 --- /dev/null +++ b/tests/ops/fixtures/saas/recharge_fixtures.py @@ -0,0 +1,238 @@ +import uuid +from typing import Any, Dict, Generator + +import pydash +import pytest +import requests +from faker import Faker +from requests import Response +from sqlalchemy.orm import Session + +from fides.api.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fides.api.ops.models.datasetconfig import DatasetConfig +from fides.api.ops.util.saas_util import ( + load_config_with_replacement, + load_dataset_with_replacement, +) +from fides.lib.db import session +from tests.ops.test_helpers.saas_test_utils import poll_for_existence +from tests.ops.test_helpers.vault_client import get_secrets + +secrets = get_secrets("recharge") + + +@pytest.fixture(scope="function") +def recharge_secrets(saas_config): + return { + "domain": pydash.get(saas_config, "recharge.domain") or secrets["domain"], + "api_key": pydash.get(saas_config, "recharge.api_key") or secrets["api_key"], + } + + +@pytest.fixture(scope="function") +def recharge_identity_email(saas_config): + return ( + pydash.get(saas_config, "recharge.identity_email") or secrets["identity_email"] + ) + + +@pytest.fixture(scope="function") +def recharge_erasure_identity_email(): + return f"{uuid.uuid4().hex}@email.com" + + +@pytest.fixture +def recharge_config() -> Dict[str, Any]: + return load_config_with_replacement( + "data/saas/config/recharge_config.yml", + "", + "recharge_instance", + ) + + +@pytest.fixture +def recharge_dataset() -> Dict[str, Any]: + return load_dataset_with_replacement( + "data/saas/dataset/recharge_dataset.yml", + "", + "recharge_instance", + )[0] + + +@pytest.fixture(scope="function") +def recharge_connection_config( + db: session, recharge_config, recharge_secrets +) -> Generator: + fides_key = recharge_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": recharge_secrets, + "saas_config": recharge_config, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture +def recharge_dataset_config( + db: Session, + recharge_connection_config: ConnectionConfig, + recharge_dataset: Dict[str, Any], +) -> Generator: + fides_key = recharge_dataset["fides_key"] + recharge_connection_config.name = fides_key + recharge_connection_config.key = fides_key + recharge_connection_config.save(db=db) + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": recharge_connection_config.id, + "fides_key": fides_key, + "dataset": recharge_dataset, + }, + ) + yield dataset + dataset.delete(db=db) + + +class RechargeTestClient: + """Helper to call various Recharge data management requests""" + + def __init__(self, recharge_connection_config: ConnectionConfig): + self.recharge_secrets = recharge_connection_config.secrets + self.headers = { + "X-Recharge-Access-Token": self.recharge_secrets["api_key"], + "Content-Type": "application/json", + } + self.base_url = f"https://{self.recharge_secrets['domain']}" + self.faker = Faker() + self.first_name = self.faker.first_name() + self.last_name = self.faker.last_name() + self.street_address = self.faker.street_address() + + # 1: Creates, checks for existance and deletes customer + def create_customer(self, email) -> Response: + customer_body = { + "first_name": self.first_name, + "last_name": self.last_name, + "email": email, + "billing_address1": self.street_address, + "billing_city": "New York City", + "billing_province": "New York", + "billing_country": "United States", + "billing_first_name": self.first_name, + "billing_last_name": self.last_name, + "billing_zip": "10001", + } + + customer_response: Response = requests.post( + url=f"{self.base_url}/customers", + json=customer_body, + headers=self.headers, + ) + assert customer_response.ok + + return customer_response + + def get_customer(self, email): + customer_response: Response = requests.get( + url=f"{self.base_url}/customers", + params={"email": email}, + headers=self.headers, + ) + assert customer_response.ok + return customer_response.json() + + def delete_customer(self, customer_id): + customer_response: Response = requests.delete( + url=f"{self.base_url}/customers/{customer_id}", headers=self.headers + ) + assert customer_response.ok + + # 2: Creates, checks for existance and deletes address + def create_address(self, customer_id) -> Response: + address_body = { + "customer_id": customer_id, + "address1": self.street_address, + "address2": self.street_address, + "city": "Los Angeles", + "company": "Recharge", + "country_code": "US", + "country": "United States", + "first_name": self.first_name, + "last_name": self.last_name, + "order_attributes": [{"name": "custom name", "value": "custom value"}], + "phone": "5551234567", + "province": "California", + "zip": "90001", + } + address_response = requests.post( + url=f"{self.base_url}/addresses", + headers=self.headers, + json=address_body, + ) + assert address_response.ok + return address_response + + def get_addresses(self, customer_id): + address_response: Response = requests.get( + url=f"{self.base_url}/addresses", + params={"customer_id": customer_id}, + headers=self.headers, + ) + assert address_response.ok + return address_response.json() + + def delete_address(self, address_id): + address_response: Response = requests.delete( + url=f"{self.base_url}/addresses/{address_id}", headers=self.headers + ) + assert address_response.ok + + +@pytest.fixture(scope="function") +def recharge_test_client(recharge_connection_config: RechargeTestClient) -> Generator: + test_client = RechargeTestClient( + recharge_connection_config=recharge_connection_config + ) + yield test_client + + +@pytest.fixture(scope="function") +def recharge_erasure_data( + recharge_test_client: RechargeTestClient, recharge_erasure_identity_email: str +) -> Generator: + customer_response = recharge_test_client.create_customer( + recharge_erasure_identity_email + ) + error_message = f"customer with email {recharge_erasure_identity_email} could not be created in Recharge" + poll_for_existence( + recharge_test_client.get_customer, + (recharge_erasure_identity_email,), + error_message=error_message, + ) + customer_id = customer_response.json()["customer"]["id"] + + address_response = recharge_test_client.create_address(customer_id) + error_message = f"address for customer '{recharge_erasure_identity_email}' could not be created in Recharge" + poll_for_existence( + recharge_test_client.get_addresses, + args=(customer_id,), + error_message=error_message, + ) + address_id = address_response.json()["address"]["id"] + + yield customer_response, address_response + + recharge_test_client.delete_address(address_id) + recharge_test_client.delete_customer(customer_id) diff --git a/tests/ops/fixtures/saas_example_fixtures.py b/tests/ops/fixtures/saas_example_fixtures.py index 24db889597..e608ef06b7 100644 --- a/tests/ops/fixtures/saas_example_fixtures.py +++ b/tests/ops/fixtures/saas_example_fixtures.py @@ -406,8 +406,8 @@ def erasure_policy_complete_mask( "name": "user_contact_address_country Erasure Rule", "policy_id": erasure_policy.id, "masking_strategy": { - "strategy": NullMaskingStrategy, - "configuration": {}, + "strategy": StringRewriteMaskingStrategy.name, + "configuration": {"rewrite_value": "Masked"}, }, }, ) diff --git a/tests/ops/integration_tests/saas/test_recharge_tasks.py b/tests/ops/integration_tests/saas/test_recharge_tasks.py new file mode 100644 index 0000000000..226e74948e --- /dev/null +++ b/tests/ops/integration_tests/saas/test_recharge_tasks.py @@ -0,0 +1,256 @@ +import random + +import pytest + +from fides.api.ops.graph.graph import DatasetGraph +from fides.api.ops.models.privacy_request import PrivacyRequest +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.service.connectors import get_connector +from fides.api.ops.task import graph_task +from fides.api.ops.task.graph_task import get_cached_data_for_erasures +from fides.core.config import get_config +from tests.ops.graph.graph_test_util import assert_rows_match + +CONFIG = get_config() + + +@pytest.mark.integration_saas +@pytest.mark.integration_recharge +def test_recharge_connection_test(recharge_connection_config) -> None: + get_connector(recharge_connection_config).test_connection() + + +@pytest.mark.integration_saas +@pytest.mark.integration_recharge +@pytest.mark.asyncio +async def test_recharge_access_request_task( + db, + policy, + recharge_connection_config, + recharge_dataset_config, + recharge_identity_email, +) -> None: + """Full access request based on the Recharge SaaS config""" + + privacy_request = PrivacyRequest( + id=f"test_recharge_access_request_task_{random.randint(0, 1000)}" + ) + identity_attribute = "email" + identity_value = recharge_identity_email + identity_kwargs = {identity_attribute: identity_value} + identity = Identity(**identity_kwargs) + privacy_request.cache_identity(identity) + + dataset_name = recharge_connection_config.get_saas_config().fides_key + merged_graph = recharge_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + v = await graph_task.run_access_request( + privacy_request, + policy, + graph, + [recharge_connection_config], + {"email": recharge_identity_email}, + db, + ) + + key = f"{dataset_name}:customer" + assert_rows_match( + v[key], + min_size=1, + keys=[ + "accepts_marketing", + "billing_address1", + "billing_address2", + "billing_city", + "billing_company", + "billing_country", + "billing_phone", + "billing_province", + "billing_zip", + "created_at", + "email", + "first_charge_processed_at", + "first_name", + "has_card_error_in_dunning", + "has_valid_payment_method", + "hash", + "id", + "last_name", + "number_active_subscriptions", + "number_subscriptions", + "phone", + "processor_type", + "reason_payment_method_not_valid", + "shopify_customer_id", + "status", + "tax_exempt", + "updated_at", + ], + ) + for item in v[key]: + assert item["email"] == recharge_identity_email + + customer_id = v[key][0]["id"] + + key = f"{dataset_name}:addresses" + assert_rows_match( + v[key], + min_size=1, + keys=[ + "address1", + "address2", + "cart_attributes", + "cart_note", + "city", + "company", + "country", + "created_at", + "customer_id", + "discount_id", + "first_name", + "id", + "last_name", + "note_attributes", + "original_shipping_lines", + "phone", + "presentment_currency", + "province", + "shipping_lines_override", + "updated_at", + "zip", + ], + ) + + for item in v[key]: + assert item["customer_id"] == customer_id + + +@pytest.mark.integration_saas +@pytest.mark.integration_recharge +@pytest.mark.asyncio +async def test_recharge_erasure_request_task( + db, + policy, + erasure_policy_complete_mask, + recharge_connection_config, + recharge_dataset_config, + recharge_erasure_identity_email, + recharge_erasure_data, + recharge_test_client, +) -> None: + privacy_request = PrivacyRequest( + id=f"test_recharge_erasure_request_task_{random.randint(0, 1000)}" + ) + identity_attribute = "email" + identity_value = recharge_erasure_identity_email + identity_kwargs = {identity_attribute: identity_value} + identity = Identity(**identity_kwargs) + privacy_request.cache_identity(identity) + + dataset_name = recharge_connection_config.get_saas_config().fides_key + + merged_graph = recharge_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + v = await graph_task.run_access_request( + privacy_request, + policy, + graph, + [recharge_connection_config], + {"email": recharge_erasure_identity_email}, + db, + ) + + key = f"{dataset_name}:customer" + assert_rows_match( + v[key], + min_size=1, + keys=[ + "billing_address1", + "billing_address2", + "billing_city", + "billing_company", + "billing_country", + "billing_phone", + "billing_province", + "billing_zip", + "created_at", + "email", + "first_charge_processed_at", + "first_name", + "has_card_error_in_dunning", + "has_valid_payment_method", + "hash", + "id", + "last_name", + "number_active_subscriptions", + "number_subscriptions", + "phone", + "processor_type", + "reason_payment_method_not_valid", + "shopify_customer_id", + "status", + "tax_exempt", + "updated_at", + ], + ) + for item in v[key]: + assert item["email"] == recharge_erasure_identity_email + + customer_id = v[key][0]["id"] + + key = f"{dataset_name}:addresses" + assert_rows_match( + v[key], + min_size=1, + keys=[ + "address1", + "address2", + "cart_attributes", + "cart_note", + "city", + "company", + "country", + "created_at", + "customer_id", + "discount_id", + "first_name", + "id", + "last_name", + "note_attributes", + "original_shipping_lines", + "phone", + "presentment_currency", + "province", + "shipping_lines_override", + "updated_at", + "zip", + ], + ) + + for item in v[key]: + assert item["customer_id"] == customer_id + + temp_masking = CONFIG.execution.masking_strict + CONFIG.execution.masking_strict = False + + x = await graph_task.run_erasure( + privacy_request, + erasure_policy_complete_mask, + graph, + [recharge_connection_config], + identity_kwargs, + get_cached_data_for_erasures(privacy_request.id), + db, + ) + + assert x == {f"{dataset_name}:addresses": 1, f"{dataset_name}:customer": 1} + + address = recharge_test_client.get_addresses( + recharge_erasure_data[1].json().get("address", {}).get("id") + ) + assert not address["addresses"] + + customer = recharge_test_client.get_customer(recharge_erasure_identity_email) + assert not customer["customers"] + + CONFIG.execution.masking_strict = temp_masking