diff --git a/CHANGELOG.md b/CHANGELOG.md index d679f73a6d..a4cd4c04f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The types of changes are: * Added the datamap UI to make it open source [#2988](https://github.com/ethyca/fides/pull/2988) * Introduced a `FixedLayout` component (from the datamap UI) for pages that need to be a fixed height and scroll within [#2992](https://github.com/ethyca/fides/pull/2992) * Added preliminary privacy notice page [#2995](https://github.com/ethyca/fides/pull/2995) +* Query params on connection type endpoint to filter by supported action type [#2996](https://github.com/ethyca/fides/pull/2996) ### Changed * Set `privacyDeclarationDeprecatedFields` flags to false and set `userCannotModify` to true [2987](https://github.com/ethyca/fides/pull/2987) diff --git a/docs/fides/docs/development/postman/Fides.postman_collection.json b/docs/fides/docs/development/postman/Fides.postman_collection.json index 92d46575a1..0b5536f843 100644 --- a/docs/fides/docs/development/postman/Fides.postman_collection.json +++ b/docs/fides/docs/development/postman/Fides.postman_collection.json @@ -4239,6 +4239,39 @@ }, "response": [] }, + { + "name": "Get available connectors - consent", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection_type?consent=true", + "host": [ + "{{host}}" + ], + "path": [ + "connection_type" + ], + "query": [ + { + "key": "consent", + "value": "true" + } + ] + } + }, + "response": [] + }, { "name": "Get connection secrets schema", "request": { diff --git a/src/fides/api/ops/api/v1/endpoints/connection_type_endpoints.py b/src/fides/api/ops/api/v1/endpoints/connection_type_endpoints.py index bacfd4d436..ef38b01f39 100644 --- a/src/fides/api/ops/api/v1/endpoints/connection_type_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/connection_type_endpoints.py @@ -13,6 +13,7 @@ V1_URL_PREFIX, ) from fides.api.ops.common_exceptions import NoSuchConnectionTypeSecretSchemaError +from fides.api.ops.models.policy import ActionType from fides.api.ops.schemas.connection_configuration.connection_config import ( ConnectionSystemTypeMap, SystemType, @@ -36,11 +37,32 @@ def get_all_connection_types( params: Params = Depends(), search: Optional[str] = None, system_type: Optional[SystemType] = None, + consent: Optional[bool] = None, + access: Optional[bool] = None, + erasure: Optional[bool] = None, ) -> AbstractPage[ConnectionSystemTypeMap]: - """Returns a list of connection options in Fidesops - includes only database and saas options here.""" + """ + Returns a list of connection options in Fides - includes only database and saas options here. + + Query params for types of requests supported - `consent`, `access` and `erasure` - act as filters. + If set to `true`, only connections that support the specified type of request will be returned. + If no filters are specified, then no filtering is performed. + When applied together, the filters act as a union: result sets are additive. + """ + action_types = set() + # special-case when no action type filters are provided + if consent is None and access is None and erasure is None: + action_types = {ActionType.access, ActionType.erasure, ActionType.consent} + else: + if access: + action_types.add(ActionType.access) + if erasure: + action_types.add(ActionType.erasure) + if consent: + action_types.add(ActionType.consent) return paginate( - get_connection_types(search, system_type), + get_connection_types(search, system_type, action_types), params, ) diff --git a/src/fides/api/ops/models/policy.py b/src/fides/api/ops/models/policy.py index 70b76bec89..a8708fe80d 100644 --- a/src/fides/api/ops/models/policy.py +++ b/src/fides/api/ops/models/policy.py @@ -50,6 +50,10 @@ class ActionType(str, EnumType): update = "update" +# action types we actively support in policies/requests +SUPPORTED_ACTION_TYPES = {ActionType.access, ActionType.consent, ActionType.erasure} + + class DrpAction(EnumType): """ Enum to hold valid DRP actions. For more details, see: diff --git a/src/fides/api/ops/util/connection_type.py b/src/fides/api/ops/util/connection_type.py index d8cb2bf05c..2ddfd70c6a 100644 --- a/src/fides/api/ops/util/connection_type.py +++ b/src/fides/api/ops/util/connection_type.py @@ -1,9 +1,12 @@ from __future__ import annotations -from typing import Any +from typing import Any, Set + +import yaml from fides.api.ops.common_exceptions import NoSuchConnectionTypeSecretSchemaError from fides.api.ops.models.connectionconfig import ConnectionType +from fides.api.ops.models.policy import SUPPORTED_ACTION_TYPES, ActionType from fides.api.ops.schemas.connection_configuration import ( SaaSSchemaFactory, secrets_schemas, @@ -70,14 +73,65 @@ def connection_type_secret_schema(*, connection_type: str) -> dict[str, Any]: def get_connection_types( - search: str | None = None, system_type: SystemType | None = None + search: str | None = None, + system_type: SystemType | None = None, + action_types: Set[ActionType] = SUPPORTED_ACTION_TYPES, ) -> list[ConnectionSystemTypeMap]: def is_match(elem: str) -> bool: """If a search query param was included, is it a substring of an available connector type?""" return search.lower() in elem.lower() if search else True + def saas_request_type_filter(connection_type: str) -> bool: + """ + If any of the request type filters are set to true, + ensure the given saas connector supports requests of at least one of those types. + """ + if SUPPORTED_ACTION_TYPES == action_types: + # if none of our filters are enabled, pass quickly to avoid unnecessary overhead + return True + + template = ConnectorRegistry.get_connector_template(connection_type) + if template is None: # shouldn't happen, but we can be safe + return False + + saas_config = SaaSConfig(**yaml.safe_load(template.config).get("saas_config")) + has_access = bool( + next( + ( + request.read + for request in [ + endpoint.requests for endpoint in saas_config.endpoints + ] + ), + None, + ) + ) + has_erasure = ( + bool( + next( + ( + request.update or request.delete + for request in [ + endpoint.requests for endpoint in saas_config.endpoints + ] + ), + None, + ) + ) + or saas_config.data_protection_request + ) + has_consent = saas_config.consent_requests + + return bool( + (ActionType.consent in action_types and has_consent) + or (ActionType.access in action_types and has_access) + or (ActionType.erasure in action_types and has_erasure) + ) + connection_system_types: list[ConnectionSystemTypeMap] = [] - if system_type == SystemType.database or system_type is None: + if (system_type == SystemType.database or system_type is None) and ( + ActionType.access in action_types or ActionType.erasure in action_types + ): database_types: list[str] = sorted( [ conn_type.value @@ -110,7 +164,7 @@ def is_match(elem: str) -> bool: [ saas_type for saas_type in ConnectorRegistry.connector_types() - if is_match(saas_type) + if is_match(saas_type) and saas_request_type_filter(saas_type) ] ) @@ -131,7 +185,9 @@ def is_match(elem: str) -> bool: ) ) - if system_type == SystemType.manual or system_type is None: + if ( + system_type == SystemType.manual or system_type is None + ) and ActionType.access in action_types: manual_types: list[str] = sorted( [ manual_type.value @@ -159,6 +215,16 @@ def is_match(elem: str) -> bool: if email_type in ERASURE_EMAIL_CONNECTOR_TYPES + CONSENT_EMAIL_CONNECTOR_TYPES and is_match(email_type.value) + and ( # include consent or erasure connectors if requested, respectively + ( + ActionType.consent in action_types + and email_type in CONSENT_EMAIL_CONNECTOR_TYPES + ) + or ( + ActionType.erasure in action_types + and email_type in ERASURE_EMAIL_CONNECTOR_TYPES + ) + ) ] ) connection_system_types.extend( diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 5031b7c049..d111fa6c9c 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -1,3 +1,4 @@ +from typing import List, Set from unittest import mock import pytest @@ -20,6 +21,7 @@ ConnectionType, ) from fides.api.ops.models.datasetconfig import DatasetConfig +from fides.api.ops.models.policy import ActionType from fides.api.ops.schemas.connection_configuration.connection_config import SystemType from fides.api.ops.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, @@ -316,6 +318,289 @@ def test_search_email_type(self, api_client, generate_auth_header, url): ] +DOORDASH = "doordash" +GOOGLE_ANALYTICS = "google_analytics" +MAILCHIMP_TRANSACTIONAL = "mailchimp_transactional" +SEGMENT = "segment" +STRIPE = "stripe" +ZENDESK = "zendesk" + + +class TestGetConnectionsActionTypeParams: + """ + Class specifically for testing the "action type" query params for the get connection types endpoint. + + This testing approach (and the fixtures) mimic what's done within `test_connection_type.py` to evaluate + the `action_type` filtering logic. + + That test specifically tests the underlying utility that is leveraged by this endpoint. + """ + + @pytest.fixture(scope="function") + def url(self) -> str: + return V1_URL_PREFIX + CONNECTION_TYPES + + @pytest.fixture(scope="function") + def url_with_params(self) -> str: + return ( + V1_URL_PREFIX + + CONNECTION_TYPES + + "?" + + "consent={consent}" + + "&access={access}" + + "&erasure={erasure}" + ) + + @pytest.fixture + def connection_type_objects(self): + google_analytics_template = ConnectorRegistry.get_connector_template( + GOOGLE_ANALYTICS + ) + mailchimp_transactional_template = ConnectorRegistry.get_connector_template( + MAILCHIMP_TRANSACTIONAL + ) + stripe_template = ConnectorRegistry.get_connector_template("stripe") + zendesk_template = ConnectorRegistry.get_connector_template("zendesk") + doordash_template = ConnectorRegistry.get_connector_template(DOORDASH) + segment_template = ConnectorRegistry.get_connector_template(SEGMENT) + + return { + ConnectionType.postgres.value: { + "identifier": ConnectionType.postgres.value, + "type": SystemType.database.value, + "human_readable": "PostgreSQL", + "encoded_icon": None, + }, + ConnectionType.manual_webhook.value: { + "identifier": ConnectionType.manual_webhook.value, + "type": SystemType.manual.value, + "human_readable": "Manual Process", + "encoded_icon": None, + }, + GOOGLE_ANALYTICS: { + "identifier": GOOGLE_ANALYTICS, + "type": SystemType.saas.value, + "human_readable": google_analytics_template.human_readable, + "encoded_icon": google_analytics_template.icon, + }, + MAILCHIMP_TRANSACTIONAL: { + "identifier": MAILCHIMP_TRANSACTIONAL, + "type": SystemType.saas.value, + "human_readable": mailchimp_transactional_template.human_readable, + "encoded_icon": mailchimp_transactional_template.icon, + }, + SEGMENT: { + "identifier": SEGMENT, + "type": SystemType.saas.value, + "human_readable": segment_template.human_readable, + "encoded_icon": segment_template.icon, + }, + STRIPE: { + "identifier": STRIPE, + "type": SystemType.saas.value, + "human_readable": stripe_template.human_readable, + "encoded_icon": stripe_template.icon, + }, + ZENDESK: { + "identifier": ZENDESK, + "type": SystemType.saas.value, + "human_readable": zendesk_template.human_readable, + "encoded_icon": zendesk_template.icon, + }, + DOORDASH: { + "identifier": DOORDASH, + "type": SystemType.saas.value, + "human_readable": doordash_template.human_readable, + "encoded_icon": doordash_template.icon, + }, + ConnectionType.sovrn.value: { + "identifier": ConnectionType.sovrn.value, + "type": SystemType.email.value, + "human_readable": "Sovrn", + "encoded_icon": None, + }, + ConnectionType.attentive.value: { + "identifier": ConnectionType.attentive.value, + "type": SystemType.email.value, + "human_readable": "Attentive", + "encoded_icon": None, + }, + } + + @pytest.mark.parametrize( + "action_types, assert_in_data, assert_not_in_data", + [ + ( + [], # no filters should give us all connectors + [ + ConnectionType.postgres.value, + ConnectionType.manual_webhook.value, + DOORDASH, + STRIPE, + ZENDESK, + SEGMENT, + ConnectionType.attentive.value, + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.sovrn.value, + ], + [], + ), + ( + [ActionType.consent], + [GOOGLE_ANALYTICS, MAILCHIMP_TRANSACTIONAL, ConnectionType.sovrn.value], + [ + ConnectionType.postgres.value, + ConnectionType.manual_webhook.value, + DOORDASH, + STRIPE, + ZENDESK, + SEGMENT, + ConnectionType.attentive.value, + ], + ), + ( + [ActionType.access], + [ + ConnectionType.postgres.value, + ConnectionType.manual_webhook.value, + DOORDASH, + SEGMENT, + STRIPE, + ZENDESK, + ], + [ + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.sovrn.value, + ConnectionType.attentive.value, + ], + ), + ( + [ActionType.erasure], + [ + ConnectionType.postgres.value, + SEGMENT, # segment has DPR so it is an erasure + STRIPE, + ZENDESK, + ConnectionType.attentive.value, + ], + [ + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.manual_webhook.value, # manual webhook is not erasure + DOORDASH, # doordash does not have erasures + ConnectionType.sovrn.value, + ], + ), + ( + [ActionType.consent, ActionType.access], + [ + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.sovrn.value, + ConnectionType.postgres.value, + ConnectionType.manual_webhook.value, + DOORDASH, + SEGMENT, + STRIPE, + ZENDESK, + ], + [ + ConnectionType.attentive.value, + ], + ), + ( + [ActionType.consent, ActionType.erasure], + [ + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.sovrn.value, + ConnectionType.postgres.value, + SEGMENT, # segment has DPR so it is an erasure + STRIPE, + ZENDESK, + ConnectionType.attentive.value, + ], + [ + ConnectionType.manual_webhook.value, # manual webhook is not erasure + DOORDASH, # doordash does not have erasures + ], + ), + ( + [ActionType.access, ActionType.erasure], + [ + ConnectionType.postgres.value, + ConnectionType.manual_webhook.value, + DOORDASH, + SEGMENT, + STRIPE, + ZENDESK, + ConnectionType.attentive.value, + ], + [ + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.sovrn.value, + ], + ), + ], + ) + def test_get_connection_types_action_type_filter( + self, + action_types, + assert_in_data, + assert_not_in_data, + connection_type_objects, + generate_auth_header, + api_client, + url, + url_with_params, + ): + the_url = url + auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ]) + if action_types: + the_url = url_with_params.format( + consent=ActionType.consent in action_types, + access=ActionType.access in action_types, + erasure=ActionType.erasure in action_types, + ) + resp = api_client.get(the_url, headers=auth_header) + data = resp.json()["items"] + assert resp.status_code == 200 + + for connection_type in assert_in_data: + obj = connection_type_objects[connection_type] + assert obj in data + + for connection_type in assert_not_in_data: + obj = connection_type_objects[connection_type] + assert obj not in data + + # now run another request, this time omitting non-specified filter params + # rather than setting them to false explicitly. we should get identical results. + if action_types: + the_url = url + "?" + if ActionType.consent in action_types: + the_url += "consent=true&" + if ActionType.access in action_types: + the_url += "access=true&" + if ActionType.erasure in action_types: + the_url += "erasure=true&" + + resp = api_client.get(the_url, headers=auth_header) + data = resp.json()["items"] + assert resp.status_code == 200 + + for connection_type in assert_in_data: + obj = connection_type_objects[connection_type] + assert obj in data + + for connection_type in assert_not_in_data: + obj = connection_type_objects[connection_type] + assert obj not in data + + class TestGetConnectionSecretSchema: @pytest.fixture(scope="function") def base_url(self, oauth_client: ClientDetail, policy) -> str: diff --git a/tests/ops/util/test_connection_type.py b/tests/ops/util/test_connection_type.py index 36b7aa42d2..05740aa5a3 100644 --- a/tests/ops/util/test_connection_type.py +++ b/tests/ops/util/test_connection_type.py @@ -1,4 +1,7 @@ +import pytest + from fides.api.ops.models.connectionconfig import ConnectionType +from fides.api.ops.models.policy import ActionType from fides.api.ops.schemas.connection_configuration.connection_config import SystemType from fides.api.ops.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, @@ -38,3 +41,205 @@ def test_get_connection_types(): "human_readable": "Sovrn", "encoded_icon": None, } in data + + +DOORDASH = "doordash" +GOOGLE_ANALYTICS = "google_analytics" +MAILCHIMP_TRANSACTIONAL = "mailchimp_transactional" +SEGMENT = "segment" +STRIPE = "stripe" +ZENDESK = "zendesk" + + +@pytest.fixture +def connection_type_objects(): + google_analytics_template = ConnectorRegistry.get_connector_template( + GOOGLE_ANALYTICS + ) + mailchimp_transactional_template = ConnectorRegistry.get_connector_template( + MAILCHIMP_TRANSACTIONAL + ) + stripe_template = ConnectorRegistry.get_connector_template("stripe") + zendesk_template = ConnectorRegistry.get_connector_template("zendesk") + doordash_template = ConnectorRegistry.get_connector_template(DOORDASH) + segment_template = ConnectorRegistry.get_connector_template(SEGMENT) + + return { + ConnectionType.postgres.value: { + "identifier": ConnectionType.postgres.value, + "type": SystemType.database.value, + "human_readable": "PostgreSQL", + "encoded_icon": None, + }, + ConnectionType.manual_webhook.value: { + "identifier": ConnectionType.manual_webhook.value, + "type": SystemType.manual.value, + "human_readable": "Manual Process", + "encoded_icon": None, + }, + GOOGLE_ANALYTICS: { + "identifier": GOOGLE_ANALYTICS, + "type": SystemType.saas.value, + "human_readable": google_analytics_template.human_readable, + "encoded_icon": google_analytics_template.icon, + }, + MAILCHIMP_TRANSACTIONAL: { + "identifier": MAILCHIMP_TRANSACTIONAL, + "type": SystemType.saas.value, + "human_readable": mailchimp_transactional_template.human_readable, + "encoded_icon": mailchimp_transactional_template.icon, + }, + SEGMENT: { + "identifier": SEGMENT, + "type": SystemType.saas.value, + "human_readable": segment_template.human_readable, + "encoded_icon": segment_template.icon, + }, + STRIPE: { + "identifier": STRIPE, + "type": SystemType.saas.value, + "human_readable": stripe_template.human_readable, + "encoded_icon": stripe_template.icon, + }, + ZENDESK: { + "identifier": ZENDESK, + "type": SystemType.saas.value, + "human_readable": zendesk_template.human_readable, + "encoded_icon": zendesk_template.icon, + }, + DOORDASH: { + "identifier": DOORDASH, + "type": SystemType.saas.value, + "human_readable": doordash_template.human_readable, + "encoded_icon": doordash_template.icon, + }, + ConnectionType.sovrn.value: { + "identifier": ConnectionType.sovrn.value, + "type": SystemType.email.value, + "human_readable": "Sovrn", + "encoded_icon": None, + }, + ConnectionType.attentive.value: { + "identifier": ConnectionType.attentive.value, + "type": SystemType.email.value, + "human_readable": "Attentive", + "encoded_icon": None, + }, + } + + +@pytest.mark.parametrize( + "action_types, assert_in_data, assert_not_in_data", + [ + ( + [ActionType.consent], + [GOOGLE_ANALYTICS, MAILCHIMP_TRANSACTIONAL, ConnectionType.sovrn.value], + [ + ConnectionType.postgres.value, + ConnectionType.manual_webhook.value, + DOORDASH, + STRIPE, + ZENDESK, + SEGMENT, + ConnectionType.attentive.value, + ], + ), + ( + [ActionType.access], + [ + ConnectionType.postgres.value, + ConnectionType.manual_webhook.value, + DOORDASH, + SEGMENT, + STRIPE, + ZENDESK, + ], + [ + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.sovrn.value, + ConnectionType.attentive.value, + ], + ), + ( + [ActionType.erasure], + [ + ConnectionType.postgres.value, + SEGMENT, # segment has DPR so it is an erasure + STRIPE, + ZENDESK, + ConnectionType.attentive.value, + ], + [ + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.manual_webhook.value, # manual webhook is not erasure + DOORDASH, # doordash does not have erasures + ConnectionType.sovrn.value, + ], + ), + ( + [ActionType.consent, ActionType.access], + [ + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.sovrn.value, + ConnectionType.postgres.value, + ConnectionType.manual_webhook.value, + DOORDASH, + SEGMENT, + STRIPE, + ZENDESK, + ], + [ + ConnectionType.attentive.value, + ], + ), + ( + [ActionType.consent, ActionType.erasure], + [ + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.sovrn.value, + ConnectionType.postgres.value, + SEGMENT, # segment has DPR so it is an erasure + STRIPE, + ZENDESK, + ConnectionType.attentive.value, + ], + [ + ConnectionType.manual_webhook.value, # manual webhook is not erasure + DOORDASH, # doordash does not have erasures + ], + ), + ( + [ActionType.access, ActionType.erasure], + [ + ConnectionType.postgres.value, + ConnectionType.manual_webhook.value, + DOORDASH, + SEGMENT, + STRIPE, + ZENDESK, + ConnectionType.attentive.value, + ], + [ + GOOGLE_ANALYTICS, + MAILCHIMP_TRANSACTIONAL, + ConnectionType.sovrn.value, + ], + ), + ], +) +def test_get_connection_types_action_type_filter( + action_types, assert_in_data, assert_not_in_data, connection_type_objects +): + data = get_connection_types(action_types=action_types) + + for connection_type in assert_in_data: + obj = connection_type_objects[connection_type] + assert obj in data + + for connection_type in assert_not_in_data: + obj = connection_type_objects[connection_type] + assert obj not in data