diff --git a/CHANGELOG.md b/CHANGELOG.md index ea9e730e30..9e63c028fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The types of changes are: * Custom Field Library Tab [#527](https://github.com/ethyca/fides/pull/2693) * Allow SendGrid template usage [#2728](https://github.com/ethyca/fides/pull/2728) * Added ConnectorRunner to simplify SaaS connector testing [#1795](https://github.com/ethyca/fides/pull/1795) +* Adds support for Mailchimp Transactional as a messaging config [#2742](https://github.com/ethyca/fides/pull/2742) ### Changed diff --git a/src/fides/api/ctl/migrations/versions/39b209861471_adds_mailchimp_transactional_enum.py b/src/fides/api/ctl/migrations/versions/39b209861471_adds_mailchimp_transactional_enum.py new file mode 100644 index 0000000000..042dce8975 --- /dev/null +++ b/src/fides/api/ctl/migrations/versions/39b209861471_adds_mailchimp_transactional_enum.py @@ -0,0 +1,48 @@ +"""adds MAILCHIMP_TRANSACTIONAL enum + +Revision ID: 39b209861471 +Revises: eb1e6ec39b83 +Create Date: 2023-03-03 21:57:14.368385 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "39b209861471" +down_revision = "9f38dad37628" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("alter type messagingservicetype rename to messagingservicetype_old") + op.execute( + "create type messagingservicetype as enum('MAILCHIMP_TRANSACTIONAL', 'MAILGUN', 'TWILIO_TEXT', 'TWILIO_EMAIL')" + ) + op.execute( + ( + "alter table messagingconfig alter column service_type type messagingservicetype using " + "service_type::text::messagingservicetype" + ) + ) + op.execute("drop type messagingservicetype_old") + + +def downgrade(): + # return + op.execute( + "delete from messagingconfig where service_type in ('MAILCHIMP_TRANSACTIONAL')" + ) + op.execute("alter type messagingservicetype rename to messagingservicetype_old") + op.execute( + "create type messagingservicetype as enum('MAILGUN', 'TWILIO_TEXT', 'TWILIO_EMAIL')" + ) + op.execute( + ( + "alter table messagingconfig alter column service_type type messagingservicetype using " + "service_type::text::messagingservicetype" + ) + ) + op.execute("drop type messagingservicetype_old") diff --git a/src/fides/api/ops/models/messaging.py b/src/fides/api/ops/models/messaging.py index 55eaae6336..ad2c8659f7 100644 --- a/src/fides/api/ops/models/messaging.py +++ b/src/fides/api/ops/models/messaging.py @@ -20,6 +20,7 @@ SMS_MESSAGING_SERVICES, SUPPORTED_MESSAGING_SERVICE_SECRETS, MessagingMethod, + MessagingServiceSecretsMailchimpTransactional, MessagingServiceSecretsMailgun, MessagingServiceSecretsTwilioEmail, MessagingServiceSecretsTwilioSMS, @@ -55,6 +56,7 @@ def get_schema_for_secrets( """ try: schema = { + MessagingServiceType.MAILCHIMP_TRANSACTIONAL: MessagingServiceSecretsMailchimpTransactional, MessagingServiceType.MAILGUN: MessagingServiceSecretsMailgun, MessagingServiceType.TWILIO_TEXT: MessagingServiceSecretsTwilioSMS, MessagingServiceType.TWILIO_EMAIL: MessagingServiceSecretsTwilioEmail, diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py index 2503ee5d87..3606702580 100644 --- a/src/fides/api/ops/schemas/messaging/messaging.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -23,6 +23,8 @@ class MessagingMethod(Enum): class MessagingServiceType(Enum): """Enum for messaging service type. Upper-cased in the database""" + MAILCHIMP_TRANSACTIONAL = "MAILCHIMP_TRANSACTIONAL" + MAILGUN = "MAILGUN" TWILIO_TEXT = "TWILIO_TEXT" @@ -40,6 +42,7 @@ def _missing_( EMAIL_MESSAGING_SERVICES: Tuple[str, ...] = ( + MessagingServiceType.MAILCHIMP_TRANSACTIONAL.value, MessagingServiceType.MAILGUN.value, MessagingServiceType.TWILIO_EMAIL.value, ) @@ -167,15 +170,29 @@ class EmailForActionType(BaseModel): class MessagingServiceDetails(Enum): """Enum for messaging service details""" + # Generic + DOMAIN = "domain" + EMAIL_FROM = "email_from" + # Mailgun IS_EU_DOMAIN = "is_eu_domain" API_VERSION = "api_version" - DOMAIN = "domain" # Twilio Email TWILIO_EMAIL_FROM = "twilio_email_from" +class MessagingServiceDetailsMailchimpTransactional(BaseModel): + """The details required to represent a Mailchimp Transactional email configuration.""" + + email_from: str + + class Config: + """Restrict adding other fields through this schema.""" + + extra = Extra.forbid + + class MessagingServiceDetailsMailgun(BaseModel): """The details required to represent a Mailgun email configuration.""" @@ -203,6 +220,9 @@ class Config: class MessagingServiceSecrets(Enum): """Enum for message service secrets""" + # Mailchimp Transactional + MAILCHIMP_TRANSACTIONAL_API_KEY = "mailchimp_transactional_api_key" + # Mailgun MAILGUN_API_KEY = "mailgun_api_key" @@ -216,8 +236,19 @@ class MessagingServiceSecrets(Enum): TWILIO_API_KEY = "twilio_api_key" +class MessagingServiceSecretsMailchimpTransactional(BaseModel): + """The secrets required to connect to Mailchimp Transactional.""" + + mailchimp_transactional_api_key: str + + class Config: + """Restrict adding other fields through this schema.""" + + extra = Extra.forbid + + class MessagingServiceSecretsMailgun(BaseModel): - """The secrets required to connect to mailgun.""" + """The secrets required to connect to Mailgun.""" mailgun_api_key: str @@ -228,7 +259,7 @@ class Config: class MessagingServiceSecretsTwilioSMS(BaseModel): - """The secrets required to connect to twilio SMS.""" + """The secrets required to connect to Twilio SMS.""" twilio_account_sid: str twilio_auth_token: str @@ -272,7 +303,11 @@ class MessagingConfigBase(BaseModel): service_type: MessagingServiceType details: Optional[ - Union[MessagingServiceDetailsMailgun, MessagingServiceDetailsTwilioEmail] + Union[ + MessagingServiceDetailsMailchimpTransactional, + MessagingServiceDetailsMailgun, + MessagingServiceDetailsTwilioEmail, + ] ] class Config: @@ -333,6 +368,7 @@ class Config: SUPPORTED_MESSAGING_SERVICE_SECRETS = Union[ + MessagingServiceSecretsMailchimpTransactional, MessagingServiceSecretsMailgun, MessagingServiceSecretsTwilioSMS, MessagingServiceSecretsTwilioEmail, diff --git a/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py b/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py index 18e24a52f4..43eb7b66d5 100644 --- a/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py +++ b/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py @@ -2,12 +2,20 @@ from fides.api.ops.schemas.base_class import NoValidationSchema from fides.api.ops.schemas.messaging.messaging import ( + MessagingServiceSecretsMailchimpTransactional, MessagingServiceSecretsMailgun, MessagingServiceSecretsTwilioEmail, MessagingServiceSecretsTwilioSMS, ) +class MessagingServiceSecretsMailchimpTransactionalDocs( + MessagingServiceSecretsMailchimpTransactional, + NoValidationSchema, +): + """The secrets required to connect Mailchimp Transactional, for documentation""" + + class MessagingSecretsMailgunDocs(MessagingServiceSecretsMailgun, NoValidationSchema): """The secrets required to connect to Mailgun, for documentation""" @@ -25,6 +33,7 @@ class MessagingSecretsTwilioEmailDocs( possible_messaging_secrets = Union[ + MessagingServiceSecretsMailchimpTransactionalDocs, MessagingSecretsMailgunDocs, MessagingSecretsTwilioSMSDocs, MessagingSecretsTwilioEmailDocs, diff --git a/src/fides/api/ops/service/connectors/email_connector.py b/src/fides/api/ops/service/connectors/email_connector.py index f507ccaf5b..0c6e98dfa4 100644 --- a/src/fides/api/ops/service/connectors/email_connector.py +++ b/src/fides/api/ops/service/connectors/email_connector.py @@ -237,6 +237,7 @@ def _get_email_messaging_config_service_type(db: Session) -> Optional[str]: if not messaging_configs: # let messaging dispatch service handle non-existent service return None + twilio_email_config = next( ( config @@ -245,6 +246,10 @@ def _get_email_messaging_config_service_type(db: Session) -> Optional[str]: ), None, ) + if twilio_email_config: + # First choice: use Twilio + return MessagingServiceType.TWILIO_EMAIL.value + mailgun_config = next( ( config @@ -253,9 +258,20 @@ def _get_email_messaging_config_service_type(db: Session) -> Optional[str]: ), None, ) - if twilio_email_config: - # we prefer twilio over mailgun - return MessagingServiceType.TWILIO_EMAIL.value if mailgun_config: + # Second choice: use Mailgun return MessagingServiceType.MAILGUN.value + + mailchimp_transactional_config = next( + ( + config + for config in messaging_configs + if config.service_type == MessagingServiceType.MAILCHIMP_TRANSACTIONAL + ), + None, + ) + if mailchimp_transactional_config: + # Third choice: use Mailchimp Transactional + return MessagingServiceType.MAILCHIMP_TRANSACTIONAL.value + return None diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py index 9ef4900d85..7db3547105 100644 --- a/src/fides/api/ops/service/messaging/message_dispatch_service.py +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -168,9 +168,9 @@ def dispatch_message( logger.info( "Retrieving appropriate dispatcher for email service: {}", messaging_service ) - dispatcher: Optional[ - Callable[[MessagingConfig, Any, Optional[str]], None] - ] = _get_dispatcher_from_config_type(message_service_type=messaging_service) + dispatcher: Optional[Callable] = _get_dispatcher_from_config_type( + message_service_type=messaging_service + ) if not dispatcher: logger.error( "Dispatcher has not been implemented for message service type: {}", @@ -343,15 +343,76 @@ def _build_email( # pylint: disable=too-many-return-statements def _get_dispatcher_from_config_type( message_service_type: MessagingServiceType, -) -> Optional[Callable[[MessagingConfig, Any, Optional[str]], None]]: +) -> Optional[Callable]: """Determines which dispatcher to use based on message service type""" - if message_service_type == MessagingServiceType.MAILGUN: - return _mailgun_dispatcher - if message_service_type == MessagingServiceType.TWILIO_TEXT: - return _twilio_sms_dispatcher - if message_service_type == MessagingServiceType.TWILIO_EMAIL: - return _twilio_email_dispatcher - return None + handler = { + MessagingServiceType.MAILGUN: _mailgun_dispatcher, + MessagingServiceType.MAILCHIMP_TRANSACTIONAL: _mailchimp_transactional_dispatcher, + MessagingServiceType.TWILIO_TEXT: _twilio_sms_dispatcher, + MessagingServiceType.TWILIO_EMAIL: _twilio_email_dispatcher, + } + return handler.get(message_service_type) # type: ignore + + +def _mailchimp_transactional_dispatcher( + messaging_config: MessagingConfig, + message: EmailForActionType, + to: Optional[str], +) -> None: + """Dispatches email using Mailchimp Transactional""" + if not to: + logger.error("Message failed to send. No email identity supplied.") + raise MessageDispatchException("No email identity supplied.") + + if not messaging_config.details or not messaging_config.secrets: + logger.error( + "Message failed to send. No Mailchimp Transactional config details or secrets supplied." + ) + raise MessageDispatchException( + "No Mailchimp Transactional config details or secrets supplied." + ) + + from_email = messaging_config.details[MessagingServiceDetails.EMAIL_FROM.value] + data = json.dumps( + { + "key": messaging_config.secrets[ + MessagingServiceSecrets.MAILCHIMP_TRANSACTIONAL_API_KEY.value + ], + "message": { + "from_email": from_email, + "subject": message.subject, + "text": message.body, + # On Mailchimp Transactional's free plan `to` must be an email of the same + # domain as `from_email` + "to": [{"email": to.strip(), "type": "to"}], + }, + } + ) + + response = requests.post( + "https://mandrillapp.com/api/1.0/messages/send", + headers={"Content-Type": "application/json"}, + data=data, + ) + if not response.ok: + logger.error("Email failed to send with status code: %s" % response.status_code) + raise MessageDispatchException( + f"Email failed to send with status code {response.status_code}" + ) + + send_data = response.json()[0] + email_rejected = send_data.get("status", "rejected") == "rejected" + if email_rejected: + reason = send_data.get("reject_reason", "Fides Error") + explanations = { + "soft-bounce": "A temporary error occured with the target inbox. For example, this inbox could be full. See https://mailchimp.com/developer/transactional/docs/reputation-rejections/#bounces for more info.", + "hard-bounce": "A permanent error occured with the target inbox. See https://mailchimp.com/developer/transactional/docs/reputation-rejections/#bounces for more info.", + "recipient-domain-mismatch": f"You are not authorised to send email to this domain from {from_email}.", + } + explanation = explanations.get(reason, "") + raise MessageDispatchException( + f"Verification email unable to send due to reason: {reason}. {explanation}" + ) def _mailgun_dispatcher( @@ -359,15 +420,17 @@ def _mailgun_dispatcher( message: EmailForActionType, to: Optional[str], ) -> None: - """Dispatches email using mailgun""" + """Dispatches email using Mailgun""" if not to: logger.error("Message failed to send. No email identity supplied.") raise MessageDispatchException("No email identity supplied.") + if not messaging_config.details or not messaging_config.secrets: logger.error( "Message failed to send. No mailgun config details or secrets supplied." ) raise MessageDispatchException("No mailgun config details or secrets supplied.") + base_url = ( "https://api.mailgun.net" if messaging_config.details[MessagingServiceDetails.IS_EU_DOMAIN.value] is False @@ -457,7 +520,6 @@ def _twilio_email_dispatcher( ) try: - sg = sendgrid.SendGridAPIClient( api_key=messaging_config.secrets[ MessagingServiceSecrets.TWILIO_API_KEY.value diff --git a/src/fides/core/config/notification_settings.py b/src/fides/core/config/notification_settings.py index 8be55c0e74..a1f4bf888b 100644 --- a/src/fides/core/config/notification_settings.py +++ b/src/fides/core/config/notification_settings.py @@ -12,7 +12,7 @@ class NotificationSettings(FidesSettings): notification_service_type: Optional[str] = Field( default=None, - description="Sets the notification service type used to send notifications. Accepts mailgun, twilio_sms, or twilio_email.", + description="Sets the notification service type used to send notifications. Accepts mailchimp_transactional, mailgun, twilio_sms, or twilio_email.", ) send_request_completion_notification: bool = Field( default=False, @@ -32,7 +32,12 @@ class NotificationSettings(FidesSettings): def validate_notification_service_type(cls, value: Optional[str]) -> Optional[str]: """Ensure the provided type is a valid value.""" if value: - valid_values = ["MAILGUN", "TWILIO_TEXT", "TWILIO_EMAIL"] + valid_values = [ + "MAILCHIMP_TRANSACTIONAL", + "MAILGUN", + "TWILIO_TEXT", + "TWILIO_EMAIL", + ] value = value.upper() # force uppercase for safety if value not in valid_values: diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 3faa6bd326..287d617463 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -329,6 +329,30 @@ def messaging_config_twilio_sms(db: Session) -> Generator: messaging_config.delete(db) +@pytest.fixture(scope="function") +def messaging_config_mailchimp_transactional(db: Session) -> Generator: + messaging_config = MessagingConfig.create( + db=db, + data={ + "name": str(uuid4()), + "key": "my_mailchimp_transactional_messaging_config", + "service_type": MessagingServiceType.MAILCHIMP_TRANSACTIONAL, + "details": { + MessagingServiceDetails.DOMAIN.value: "some.domain", + MessagingServiceDetails.EMAIL_FROM.value: "test@example.com", + }, + }, + ) + messaging_config.set_secrets( + db=db, + messaging_secrets={ + MessagingServiceSecrets.MAILCHIMP_TRANSACTIONAL_API_KEY.value: "12984r70298r" + }, + ) + yield messaging_config + messaging_config.delete(db) + + @pytest.fixture(scope="function") def https_connection_config(db: Session) -> Generator: name = str(uuid4()) @@ -655,7 +679,6 @@ def erasure_policy_string_rewrite_long( def erasure_policy_two_rules( db: Session, oauth_client: ClientDetail, erasure_policy: Policy ) -> Generator: - second_erasure_rule = Rule.create( db=db, data={ @@ -1621,7 +1644,6 @@ def authenticated_fides_client( @pytest.fixture(scope="function") def system(db: Session) -> System: - system = System.create( db=db, data={ diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 02df838028..033ffde486 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -63,6 +63,16 @@ def payload_twilio_email(self): }, } + @pytest.fixture(scope="function") + def payload_mailchimp_transactional(self): + return { + "name": "mailchimp_transactional_email", + "service_type": MessagingServiceType.MAILCHIMP_TRANSACTIONAL.value, + "details": { + MessagingServiceDetails.EMAIL_FROM.value: "user@example.com", + }, + } + @pytest.fixture(scope="function") def payload_twilio_sms(self): return { @@ -142,7 +152,7 @@ def test_post_email_config_with_not_supported_service_type( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "value is not a valid enumeration member; permitted: 'MAILGUN', 'TWILIO_TEXT', 'TWILIO_EMAIL'" + == "value is not a valid enumeration member; permitted: 'MAILCHIMP_TRANSACTIONAL', 'MAILGUN', 'TWILIO_TEXT', 'TWILIO_EMAIL'" ) def test_post_email_config_with_no_key( @@ -255,6 +265,37 @@ def test_post_email_config_service_already_exists( f"Key (service_type)=(MAILGUN) already exists" in response.json()["detail"] ) + def test_post_mailgun_transactional_config( + self, + db: Session, + api_client: TestClient, + payload_mailchimp_transactional, + url, + generate_auth_header, + ): + key = "mailchimp_transactional_messaging_config" + payload_mailchimp_transactional["key"] = key + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + + response = api_client.post( + url, + headers=auth_header, + json=payload_mailchimp_transactional, + ) + assert 200 == response.status_code + + response_body = json.loads(response.text) + email_config = db.query(MessagingConfig).filter_by(key=key)[0] + + expected_response = { + "key": key, + "name": payload_mailchimp_transactional["name"], + "service_type": MessagingServiceType.MAILCHIMP_TRANSACTIONAL.value, + "details": {MessagingServiceDetails.EMAIL_FROM.value: "user@example.com"}, + } + assert expected_response == response_body + email_config.delete(db) + def test_post_twilio_email_config( self, db: Session, @@ -543,6 +584,47 @@ def test_put_config_secrets( ) +class TestPutMessagingConfigSecretMailchimpTransactional: + @pytest.fixture(scope="function") + def url(self, messaging_config_mailchimp_transactional) -> str: + return (V1_URL_PREFIX + MESSAGING_SECRETS).format( + config_key=messaging_config_mailchimp_transactional.key + ) + + def test_put_config_secrets( + self, + db: Session, + api_client: TestClient, + url, + generate_auth_header, + messaging_config_mailchimp_transactional, + ): + key = "key-123456789" + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put( + url, + headers=auth_header, + json={ + MessagingServiceSecrets.MAILCHIMP_TRANSACTIONAL_API_KEY.value: key, + }, + ) + assert 200 == response.status_code + + db.refresh(messaging_config_mailchimp_transactional) + + assert json.loads(response.text) == { + "msg": f"Secrets updated for MessagingConfig with key: {messaging_config_mailchimp_transactional.key}.", + "test_status": None, + "failure_reason": None, + } + assert ( + messaging_config_mailchimp_transactional.secrets[ + MessagingServiceSecrets.MAILCHIMP_TRANSACTIONAL_API_KEY.value + ] + == key + ) + + class TestPutMessagingConfigSecretTwilioSms: @pytest.fixture(scope="function") def url(self, messaging_config_twilio_sms) -> str: @@ -701,7 +783,12 @@ def test_get_configs_wrong_scope( assert 403 == response.status_code def test_get_configs( - self, db, api_client: TestClient, url, generate_auth_header, messaging_config + self, + db, + api_client: TestClient, + url, + generate_auth_header, + messaging_config, ): auth_header = generate_auth_header([MESSAGING_READ]) response = api_client.get(url, headers=auth_header) @@ -757,7 +844,11 @@ def test_get_config_invalid( assert 404 == response.status_code def test_get_config( - self, url, api_client: TestClient, generate_auth_header, messaging_config + self, + url, + api_client: TestClient, + generate_auth_header, + messaging_config, ): auth_header = generate_auth_header([MESSAGING_READ]) response = api_client.get(url, headers=auth_header)