diff --git a/CHANGELOG.md b/CHANGELOG.md index 49a4d9d7c2..2f8cd57095 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The types of changes are: * Privacy-Center-Cypress workflow for CI checks of the Privacy Center. [#1722](https://github.com/ethyca/fides/pull/1722) * Privacy Center `fides-consent.js` script for accessing consent on external pages. [Details](/clients/privacy-center/packages/fides-consent/README.md) * Erasure support for Twilio Conversations API [#1673](https://github.com/ethyca/fides/pull/1673) +* Add authenticated privacy request route. [#1819](https://github.com/ethyca/fides/pull/1819) ### Changed diff --git a/src/fides/api/ctl/database/seed.py b/src/fides/api/ctl/database/seed.py index d5bbfb5e53..7fc620079a 100644 --- a/src/fides/api/ctl/database/seed.py +++ b/src/fides/api/ctl/database/seed.py @@ -6,12 +6,18 @@ from fideslang import DEFAULT_TAXONOMY from fideslib.exceptions import KeyOrNameAlreadyExists from fideslib.models.client import ClientDetail +from fideslib.models.fides_user import FidesUser +from fideslib.models.fides_user_permissions import FidesUserPermissions from fideslib.utils.text import to_snake_case from loguru import logger as log from fides.api.ctl.database.session import sync_session from fides.api.ctl.sql_models import sql_model_map # type: ignore[attr-defined] from fides.api.ctl.utils.errors import AlreadyExistsError, QueryError +from fides.api.ops.api.v1.scope_registry import ( + PRIVACY_REQUEST_CREATE, + PRIVACY_REQUEST_READ, +) from fides.api.ops.models.policy import ActionType, DrpAction, Policy, Rule, RuleTarget from fides.api.ops.models.storage import StorageConfig from fides.api.ops.schemas.storage.storage import ( @@ -34,6 +40,62 @@ DEFAULT_ERASURE_MASKING_STRATEGY = "hmac" +def create_or_update_parent_user() -> None: + with sync_session() as db_session: + if ( + not CONFIG.security.parent_server_username + and not CONFIG.security.parent_server_password + ): + return + + if ( + CONFIG.security.parent_server_username + and not CONFIG.security.parent_server_password + or CONFIG.security.parent_server_password + and not CONFIG.security.parent_server_username + ): + # Both log and raise are here because the raise message is not showing. + # It could potentially be related to https://github.com/ethyca/fides/issues/1228 + log.error( + "Both a parent_server_user and parent_server_password must be set to create a parent server user" + ) + raise ValueError( + "Both a parent_server_user and parent_server_password must be set to create a parent server user" + ) + + user = ( + FidesUser.get_by( + db_session, + field="username", + value=CONFIG.security.parent_server_username, + ) + if CONFIG.security.parent_server_username + else None + ) + + if user and CONFIG.security.parent_server_password: + if not user.credentials_valid(CONFIG.security.parent_server_password): + log.info("Updating parent user") + user.update_password(db_session, CONFIG.security.parent_server_password) + return + + log.info("Creating parent user") + user = FidesUser.create( + db=db_session, + data={ + "username": CONFIG.security.parent_server_username, + "password": CONFIG.security.parent_server_password, + }, + ) + FidesUserPermissions.create( + db=db_session, + data={ + "user_id": user.id, + "scopes": [PRIVACY_REQUEST_CREATE, PRIVACY_REQUEST_READ], + }, + ) + + def filter_data_categories( categories: List[str], excluded_categories: List[str] ) -> List[str]: diff --git a/src/fides/api/main.py b/src/fides/api/main.py index c6c2ec415d..9a892f52ef 100644 --- a/src/fides/api/main.py +++ b/src/fides/api/main.py @@ -26,6 +26,7 @@ from fides.api.ctl import view from fides.api.ctl.database.database import configure_db +from fides.api.ctl.database.seed import create_or_update_parent_user from fides.api.ctl.routes import admin, crud, datamap, generate, health, validate from fides.api.ctl.routes.util import API_PREFIX from fides.api.ctl.ui import ( @@ -218,6 +219,8 @@ async def setup_server() -> None: await configure_db(CONFIG.database.sync_database_uri) + create_or_update_parent_user() + log.info("Validating SaaS connector templates...") try: registry = load_registry(registry_file) diff --git a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py index 0515299c9a..08ea7c30f4 100644 --- a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py @@ -33,13 +33,13 @@ from fides.api.ops import common_exceptions from fides.api.ops.api import deps from fides.api.ops.api.v1 import scope_registry as scopes -from fides.api.ops.api.v1 import urn_registry as urls from fides.api.ops.api.v1.endpoints.dataset_endpoints import _get_connection_config from fides.api.ops.api.v1.endpoints.manual_webhook_endpoints import ( get_access_manual_webhook_or_404, ) from fides.api.ops.api.v1.scope_registry import ( PRIVACY_REQUEST_CALLBACK_RESUME, + PRIVACY_REQUEST_CREATE, PRIVACY_REQUEST_READ, PRIVACY_REQUEST_REVIEW, PRIVACY_REQUEST_UPLOAD_DATA, @@ -48,6 +48,7 @@ from fides.api.ops.api.v1.urn_registry import ( PRIVACY_REQUEST_ACCESS_MANUAL_WEBHOOK_INPUT, PRIVACY_REQUEST_APPROVE, + PRIVACY_REQUEST_AUTHENTICATED, PRIVACY_REQUEST_BULK_RETRY, PRIVACY_REQUEST_DENY, PRIVACY_REQUEST_MANUAL_ERASURE, @@ -56,7 +57,10 @@ PRIVACY_REQUEST_RESUME_FROM_REQUIRES_INPUT, PRIVACY_REQUEST_RETRY, PRIVACY_REQUEST_VERIFY_IDENTITY, + PRIVACY_REQUESTS, REQUEST_PREVIEW, + REQUEST_STATUS_LOGS, + V1_URL_PREFIX, ) from fides.api.ops.common_exceptions import ( FunctionalityNotConfigured, @@ -137,7 +141,7 @@ from fides.ctl.core.config import get_config logger = logging.getLogger(__name__) -router = APIRouter(tags=["Privacy Requests"], prefix=urls.V1_URL_PREFIX) +router = APIRouter(tags=["Privacy Requests"], prefix=V1_URL_PREFIX) CONFIG = get_config() EMBEDDED_EXECUTION_LOG_LIMIT = 50 @@ -160,7 +164,7 @@ def get_privacy_request_or_error( @router.post( - urls.PRIVACY_REQUESTS, + PRIVACY_REQUESTS, status_code=HTTP_200_OK, response_model=BulkPostPrivacyRequests, ) @@ -175,122 +179,29 @@ async def create_privacy_request( You cannot update privacy requests after they've been created. """ - if not CONFIG.redis.enabled: - raise FunctionalityNotConfigured( - "Application redis cache required, but it is currently disabled! Please update your application configuration to enable integration with a redis cache." - ) - - created = [] - failed = [] - # Optional fields to validate here are those that are both nullable in the DB, and exist - # on the Pydantic schema - - logger.info("Starting creation for %s privacy requests", len(data)) - - optional_fields = ["external_id", "started_processing_at", "finished_processing_at"] - for privacy_request_data in data: - if not any(privacy_request_data.identity.dict().values()): - logger.warning( - "Create failed for privacy request with no identity provided" - ) - failure = { - "message": "You must provide at least one identity to process", - "data": privacy_request_data, - } - failed.append(failure) - continue - - logger.info("Finding policy with key '%s'", privacy_request_data.policy_key) - policy: Optional[Policy] = Policy.get_by( - db=db, - field="key", - value=privacy_request_data.policy_key, - ) - if policy is None: - logger.warning( - "Create failed for privacy request with invalid policy key %s'", - privacy_request_data.policy_key, - ) + return _create_privacy_request(db, data, False) - failure = { - "message": f"Policy with key {privacy_request_data.policy_key} does not exist", - "data": privacy_request_data, - } - failed.append(failure) - continue - kwargs = build_required_privacy_request_kwargs( - privacy_request_data.requested_at, policy.id - ) - for field in optional_fields: - attr = getattr(privacy_request_data, field) - if attr is not None: - kwargs[field] = attr - - try: - privacy_request: PrivacyRequest = PrivacyRequest.create(db=db, data=kwargs) - privacy_request.persist_identity( - db=db, identity=privacy_request_data.identity - ) - - cache_data( - privacy_request, - policy, - privacy_request_data.identity, - privacy_request_data.encryption_key, - None, - ) +@router.post( + PRIVACY_REQUEST_AUTHENTICATED, + status_code=HTTP_200_OK, + dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_CREATE])], + response_model=BulkPostPrivacyRequests, +) +async def create_privacy_request_authenticated( + *, + db: Session = Depends(deps.get_db), + data: conlist(PrivacyRequestCreate, max_items=50) = Body(...), # type: ignore +) -> BulkPostPrivacyRequests: + """ + Given a list of privacy request data elements, create corresponding PrivacyRequest objects + or report failure and execute them within the Fidesops system. - if CONFIG.execution.subject_identity_verification_required: - send_verification_code_to_user( - db, privacy_request, privacy_request_data.identity - ) - created.append(privacy_request) - continue # Skip further processing for this privacy request - if CONFIG.notifications.send_request_receipt_notification: - _send_privacy_request_receipt_message_to_user( - policy, privacy_request_data.identity - ) - if not CONFIG.execution.require_manual_request_approval: - AuditLog.create( - db=db, - data={ - "user_id": "system", - "privacy_request_id": privacy_request.id, - "action": AuditLogAction.approved, - "message": "", - }, - ) - queue_privacy_request(privacy_request.id) - except MessageDispatchException as exc: - kwargs["privacy_request_id"] = privacy_request.id - logger.error("MessageDispatchException: %s", exc) - failure = { - "message": "Verification message could not be sent.", - "data": kwargs, - } - failed.append(failure) - except common_exceptions.RedisConnectionError as exc: - logger.error("RedisConnectionError: %s", Pii(str(exc))) - # Thrown when cache.ping() fails on cache connection retrieval - raise HTTPException( - status_code=HTTP_424_FAILED_DEPENDENCY, - detail=exc.args[0], - ) - except Exception as exc: - logger.error("Exception: %s", Pii(str(exc))) - failure = { - "message": "This record could not be added", - "data": kwargs, - } - failed.append(failure) - else: - created.append(privacy_request) + You cannot update privacy requests after they've been created. - return BulkPostPrivacyRequests( - succeeded=created, - failed=failed, - ) + This route requires authentication instead of using verification codes. + """ + return _create_privacy_request(db, data, True) def _send_privacy_request_receipt_message_to_user( @@ -601,7 +512,7 @@ def attach_resume_instructions(privacy_request: PrivacyRequest) -> None: @router.get( - urls.PRIVACY_REQUESTS, + PRIVACY_REQUESTS, dependencies=[Security(verify_oauth_client, scopes=[scopes.PRIVACY_REQUEST_READ])], response_model=Page[ Union[ @@ -693,7 +604,7 @@ def get_request_status( @router.get( - urls.REQUEST_STATUS_LOGS, + REQUEST_STATUS_LOGS, dependencies=[Security(verify_oauth_client, scopes=[scopes.PRIVACY_REQUEST_READ])], response_model=Page[ExecutionLogDetailResponse], ) @@ -1487,6 +1398,139 @@ async def resume_privacy_request_from_requires_input( return privacy_request +def _create_privacy_request( + db: Session, + data: conlist(PrivacyRequestCreate), # type: ignore + authenticated: bool = False, +) -> BulkPostPrivacyRequests: + """Creates privacy requests. + + If authenticated is True the identity verification step is bypassed. + """ + if not CONFIG.redis.enabled: + raise FunctionalityNotConfigured( + "Application redis cache required, but it is currently disabled! Please update your application configuration to enable integration with a redis cache." + ) + + created = [] + failed = [] + # Optional fields to validate here are those that are both nullable in the DB, and exist + # on the Pydantic schema + + logger.info("Starting creation for %s privacy requests", len(data)) + + optional_fields = ["external_id", "started_processing_at", "finished_processing_at"] + for privacy_request_data in data: + if not any(privacy_request_data.identity.dict().values()): + logger.warning( + "Create failed for privacy request with no identity provided" + ) + failure = { + "message": "You must provide at least one identity to process", + "data": privacy_request_data, + } + failed.append(failure) + continue + + logger.info("Finding policy with key '%s'", privacy_request_data.policy_key) + policy: Optional[Policy] = Policy.get_by( + db=db, + field="key", + value=privacy_request_data.policy_key, + ) + if policy is None: + logger.warning( + "Create failed for privacy request with invalid policy key %s'", + privacy_request_data.policy_key, + ) + + failure = { + "message": f"Policy with key {privacy_request_data.policy_key} does not exist", + "data": privacy_request_data, + } + failed.append(failure) + continue + + kwargs = build_required_privacy_request_kwargs( + privacy_request_data.requested_at, policy.id + ) + for field in optional_fields: + attr = getattr(privacy_request_data, field) + if attr is not None: + kwargs[field] = attr + + try: + privacy_request: PrivacyRequest = PrivacyRequest.create(db=db, data=kwargs) + privacy_request.persist_identity( + db=db, identity=privacy_request_data.identity + ) + + cache_data( + privacy_request, + policy, + privacy_request_data.identity, + privacy_request_data.encryption_key, + None, + ) + + if ( + not authenticated + and CONFIG.execution.subject_identity_verification_required + ): + send_verification_code_to_user( + db, privacy_request, privacy_request_data.identity + ) + created.append(privacy_request) + continue # Skip further processing for this privacy request + if ( + not authenticated + and CONFIG.notifications.send_request_receipt_notification + ): + _send_privacy_request_receipt_message_to_user( + policy, privacy_request_data.identity + ) + if not CONFIG.execution.require_manual_request_approval: + AuditLog.create( + db=db, + data={ + "user_id": "system", + "privacy_request_id": privacy_request.id, + "action": AuditLogAction.approved, + "message": "", + }, + ) + queue_privacy_request(privacy_request.id) + except MessageDispatchException as exc: + kwargs["privacy_request_id"] = privacy_request.id + logger.error("MessageDispatchException: %s", exc) + failure = { + "message": "Verification message could not be sent.", + "data": kwargs, + } + failed.append(failure) + except common_exceptions.RedisConnectionError as exc: + logger.error("RedisConnectionError: %s", Pii(str(exc))) + # Thrown when cache.ping() fails on cache connection retrieval + raise HTTPException( + status_code=HTTP_424_FAILED_DEPENDENCY, + detail=exc.args[0], + ) + except Exception as exc: + logger.error("Exception: %s", Pii(str(exc))) + failure = { + "message": "This record could not be added", + "data": kwargs, + } + failed.append(failure) + else: + created.append(privacy_request) + + return BulkPostPrivacyRequests( + succeeded=created, + failed=failed, + ) + + def _process_privacy_request_restart( privacy_request: PrivacyRequest, failed_step: CurrentStep, diff --git a/src/fides/api/ops/api/v1/scope_registry.py b/src/fides/api/ops/api/v1/scope_registry.py index c13199c27a..767e503be1 100644 --- a/src/fides/api/ops/api/v1/scope_registry.py +++ b/src/fides/api/ops/api/v1/scope_registry.py @@ -19,6 +19,7 @@ CONSENT_READ = "consent:read" +PRIVACY_REQUEST_CREATE = "privacy-request:create" PRIVACY_REQUEST_READ = "privacy-request:read" PRIVACY_REQUEST_DELETE = "privacy-request:delete" PRIVACY_REQUEST_CALLBACK_RESUME = ( @@ -86,6 +87,7 @@ POLICY_CREATE_OR_UPDATE, POLICY_READ, POLICY_DELETE, + PRIVACY_REQUEST_CREATE, PRIVACY_REQUEST_REVIEW, PRIVACY_REQUEST_READ, PRIVACY_REQUEST_DELETE, diff --git a/src/fides/api/ops/api/v1/urn_registry.py b/src/fides/api/ops/api/v1/urn_registry.py index f3624da52a..bd2fe1e311 100644 --- a/src/fides/api/ops/api/v1/urn_registry.py +++ b/src/fides/api/ops/api/v1/urn_registry.py @@ -49,6 +49,7 @@ # Privacy request URLs PRIVACY_REQUESTS = "/privacy-request" PRIVACY_REQUEST_APPROVE = "/privacy-request/administrate/approve" +PRIVACY_REQUEST_AUTHENTICATED = "/privacy-request/authenticated" PRIVACY_REQUEST_BULK_RETRY = "/privacy-request/bulk/retry" PRIVACY_REQUEST_DENY = "/privacy-request/administrate/deny" REQUEST_STATUS_LOGS = "/privacy-request/{privacy_request_id}/log" diff --git a/src/fides/ctl/core/config/security_settings.py b/src/fides/ctl/core/config/security_settings.py index c8fb742ecb..f660ff3304 100644 --- a/src/fides/ctl/core/config/security_settings.py +++ b/src/fides/ctl/core/config/security_settings.py @@ -27,6 +27,8 @@ class SecuritySettings(FidesSettings): drp_jwt_secret: Optional[str] = None root_username: Optional[str] = None root_password: Optional[str] = None + parent_server_username: Optional[str] = None + parent_server_password: Optional[str] = None identity_verification_attempt_limit: int = 3 # 3 attempts encoding: str = "UTF-8" diff --git a/tests/ctl/api/test_seed.py b/tests/ctl/api/test_seed.py index 776dde4a9a..7a2c87419f 100644 --- a/tests/ctl/api/test_seed.py +++ b/tests/ctl/api/test_seed.py @@ -2,10 +2,13 @@ import pytest from fideslang import DEFAULT_TAXONOMY, DataCategory +from fideslib.models.fides_user import FidesUser from fides.api.ctl.database import seed from fides.ctl.core import api as _api -from fides.ctl.core.config import FidesConfig +from fides.ctl.core.config import FidesConfig, get_config + +CONFIG = get_config() @pytest.fixture(scope="function", name="data_category") @@ -24,6 +27,50 @@ def fixture_data_category(test_config: FidesConfig) -> Generator: ) +@pytest.fixture +def parent_server_config(): + original_username = CONFIG.security.parent_server_username + original_password = CONFIG.security.parent_server_password + CONFIG.security.parent_server_username = "test_user" + CONFIG.security.parent_server_password = "Atestpassword1!" + yield + CONFIG.security.parent_server_username = original_username + CONFIG.security.parent_server_password = original_password + + +@pytest.fixture +def parent_server_config_none(): + original_username = CONFIG.security.parent_server_username + original_password = CONFIG.security.parent_server_password + CONFIG.security.parent_server_username = None + CONFIG.security.parent_server_password = None + yield + CONFIG.security.parent_server_username = original_username + CONFIG.security.parent_server_password = original_password + + +@pytest.fixture +def parent_server_config_username_only(): + original_username = CONFIG.security.parent_server_username + original_password = CONFIG.security.parent_server_password + CONFIG.security.parent_server_username = "test_user" + CONFIG.security.parent_server_password = None + yield + CONFIG.security.parent_server_username = original_username + CONFIG.security.parent_server_password = original_password + + +@pytest.fixture +def parent_server_config_password_only(): + original_username = CONFIG.security.parent_server_username + original_password = CONFIG.security.parent_server_password + CONFIG.security.parent_server_username = None + CONFIG.security.parent_server_password = "Atestpassword1!" + yield + CONFIG.security.parent_server_username = original_username + CONFIG.security.parent_server_password = original_password + + @pytest.mark.unit class TestFilterDataCategories: def test_filter_data_categories_excluded(self) -> None: @@ -218,3 +265,52 @@ async def test_does_not_remove_user_added_taxonomies( headers=test_config.user.request_headers, ) assert result.status_code == 200 + + +@pytest.mark.usefixtures("parent_server_config") +def test_create_or_update_parent_user(db): + seed.create_or_update_parent_user() + user = FidesUser.get_by( + db, field="username", value=CONFIG.security.parent_server_username + ) + + assert user is not None + user.delete(db) + + +@pytest.mark.usefixtures("parent_server_config") +def test_create_or_update_parent_user_change_password(db): + user = FidesUser.create( + db=db, + data={ + "username": CONFIG.security.parent_server_username, + "password": "Somepassword1!", + }, + ) + + seed.create_or_update_parent_user() + db.refresh(user) + + assert user.password_reset_at is not None + assert user.credentials_valid(CONFIG.security.parent_server_password) is True + user.delete(db) + + +@pytest.mark.usefixtures("parent_server_config_none") +def test_create_or_update_parent_user_no_settings(db): + seed.create_or_update_parent_user() + user = FidesUser.all(db) + + assert user == [] + + +@pytest.mark.usefixtures("parent_server_config_username_only") +def test_create_or_update_parent_user_username_only(): + with pytest.raises(ValueError): + seed.create_or_update_parent_user() + + +@pytest.mark.usefixtures("parent_server_config_password_only") +def test_create_or_update_parent_user_password_only(): + with pytest.raises(ValueError): + seed.create_or_update_parent_user() diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 62c28e0937..bb88308865 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -28,6 +28,7 @@ from fides.api.ops.api.v1.scope_registry import ( DATASET_CREATE_OR_UPDATE, PRIVACY_REQUEST_CALLBACK_RESUME, + PRIVACY_REQUEST_CREATE, PRIVACY_REQUEST_READ, PRIVACY_REQUEST_REVIEW, PRIVACY_REQUEST_UPLOAD_DATA, @@ -38,6 +39,7 @@ DATASETS, PRIVACY_REQUEST_ACCESS_MANUAL_WEBHOOK_INPUT, PRIVACY_REQUEST_APPROVE, + PRIVACY_REQUEST_AUTHENTICATED, PRIVACY_REQUEST_BULK_RETRY, PRIVACY_REQUEST_DENY, PRIVACY_REQUEST_MANUAL_ERASURE, @@ -3806,3 +3808,463 @@ def test_create_privacy_request_with_email_config( assert queue == MESSAGING_QUEUE_NAME pr.delete(db=db) + + +class TestCreatePrivacyRequestAuthenticated: + @pytest.fixture(scope="function") + def url(self) -> str: + return f"{V1_URL_PREFIX}{PRIVACY_REQUEST_AUTHENTICATED}" + + @pytest.fixture + def verification_config(self): + original = CONFIG.execution.subject_identity_verification_required + CONFIG.execution.subject_identity_verification_required = True + yield + CONFIG.execution.subject_identity_verification_required = original + + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request( + self, + run_access_request_mock, + url, + generate_auth_header, + api_client: TestClient, + policy, + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + assert run_access_request_mock.called + + @pytest.mark.usefixtures("verification_config") + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_bypass_verification( + self, + run_access_request_mock, + url, + generate_auth_header, + api_client: TestClient, + policy, + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + assert run_access_request_mock.called + + def test_create_privacy_requests_unauthenticated( + self, api_client: TestClient, url, policy + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + } + ] + response = api_client.post(url, json=data, headers={}) + assert 401 == response.status_code + + def test_create_privacy_requests_wrong_scope( + self, api_client: TestClient, generate_auth_header, url, policy + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + } + ] + auth_header = generate_auth_header(scopes=[STORAGE_CREATE_OR_UPDATE]) + response = api_client.post(url, json=data, headers=auth_header) + assert 403 == response.status_code + + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_stores_identities( + self, + run_access_request_mock, + url, + db, + generate_auth_header, + api_client: TestClient, + policy, + ): + TEST_EMAIL = "test@example.com" + TEST_PHONE_NUMBER = "+12345678910" + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": { + "email": TEST_EMAIL, + "phone_number": TEST_PHONE_NUMBER, + }, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) + persisted_identity = pr.get_persisted_identity() + assert persisted_identity.email == TEST_EMAIL + assert persisted_identity.phone_number == TEST_PHONE_NUMBER + assert run_access_request_mock.called + + @pytest.mark.usefixtures("require_manual_request_approval") + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_require_manual_approval( + self, + run_access_request_mock, + url, + generate_auth_header, + api_client: TestClient, + policy, + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + assert response_data[0]["status"] == "pending" + assert not run_access_request_mock.called + + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_with_masking_configuration( + self, + run_access_request_mock, + url, + generate_auth_header, + api_client: TestClient, + erasure_policy_string_rewrite, + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": erasure_policy_string_rewrite.key, + "identity": {"email": "test@example.com"}, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_access_request" + ) + def test_create_privacy_request_limit_exceeded( + self, + _, + url, + generate_auth_header, + api_client: TestClient, + policy, + ): + payload = [] + for _ in range(0, 51): + payload.append( + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "ftest{i}@example.com"}, + }, + ) + + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + response = api_client.post(url, json=payload, headers=auth_header) + + assert 422 == response.status_code + assert ( + json.loads(response.text)["detail"][0]["msg"] + == "ensure this value has at most 50 items" + ) + + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_starts_processing( + self, + run_privacy_request_mock, + url, + generate_auth_header, + api_client: TestClient, + policy, + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert run_privacy_request_mock.called + assert resp.status_code == 200 + + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_with_external_id( + self, + run_access_request_mock, + url, + db, + generate_auth_header, + api_client: TestClient, + policy, + ): + external_id = "ext_some-uuid-here-1234" + data = [ + { + "external_id": external_id, + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + assert response_data[0]["external_id"] == external_id + pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) + assert pr.external_id == external_id + assert run_access_request_mock.called + + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_caches_identity( + self, + run_access_request_mock, + url, + db, + generate_auth_header, + api_client: TestClient, + policy, + cache, + ): + identity = {"email": "test@example.com"} + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": identity, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) + key = get_identity_cache_key( + privacy_request_id=pr.id, + identity_attribute=list(identity.keys())[0], + ) + assert cache.get(key) == list(identity.values())[0] + assert run_access_request_mock.called + + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_caches_masking_secrets( + self, + run_erasure_request_mock, + url, + db, + generate_auth_header, + api_client: TestClient, + erasure_policy_aes, + cache, + ): + identity = {"email": "test@example.com"} + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": erasure_policy_aes.key, + "identity": identity, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) + secret_key = get_masking_secret_cache_key( + privacy_request_id=pr.id, + masking_strategy="aes_encrypt", + secret_type=SecretType.key, + ) + assert cache.get_encoded_by_key(secret_key) is not None + assert run_erasure_request_mock.called + + def test_create_privacy_request_invalid_encryption_values( + self, url, generate_auth_header, api_client: TestClient, policy # , cache + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + "encryption_key": "test", + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 422 + assert resp.json()["detail"][0]["msg"] == "Encryption key must be 16 bytes long" + + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_caches_encryption_keys( + self, + run_access_request_mock, + url, + db, + generate_auth_header, + api_client: TestClient, + policy, + cache, + ): + identity = {"email": "test@example.com"} + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": identity, + "encryption_key": "test--encryption", + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) + encryption_key = get_encryption_cache_key( + privacy_request_id=pr.id, + encryption_attr="key", + ) + assert cache.get(encryption_key) == "test--encryption" + assert run_access_request_mock.called + + def test_create_privacy_request_no_identities( + self, + url, + generate_auth_header, + api_client: TestClient, + policy, + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {}, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 0 + response_data = resp.json()["failed"] + assert len(response_data) == 1 + + def test_create_privacy_request_registers_async_task( + self, + db, + url, + generate_auth_header, + api_client, + policy, + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) + assert pr.get_cached_task_id() is not None + assert pr.get_async_execution_task() is not None + + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_creates_system_audit_log( + self, + run_access_request_mock, + url, + db, + generate_auth_header, + api_client: TestClient, + policy, + ): + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + } + ] + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_CREATE]) + resp = api_client.post(url, json=data, headers=auth_header) + response_data = resp.json()["succeeded"][0] + approval_audit_log: AuditLog = AuditLog.filter( + db=db, + conditions=( + (AuditLog.privacy_request_id == response_data["id"]) + & (AuditLog.action == AuditLogAction.approved) + ), + ).first() + assert approval_audit_log is not None + assert approval_audit_log.user_id == "system" + assert run_access_request_mock.called