diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 03ce0f111c..77ca779efb 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -1358,6 +1358,8 @@ dataset: data_categories: [user.contact, user.unique_id] - name: field_name data_categories: [system.operations] + - name: field_label + data_categories: [system.operations] - name: hashed_value data_categories: [user.contact, user.unique_id] - name: id diff --git a/CHANGELOG.md b/CHANGELOG.md index cd56bc25cf..bf02fb0be7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The types of changes are: - Added `getModalLinkLabel` method to global fides object [#4766](https://github.com/ethyca/fides/pull/4766) - Added language switcher to fides overlay modal [#4773](https://github.com/ethyca/fides/pull/4773) - Added modal link label to experience translation model [#4767](https://github.com/ethyca/fides/pull/4767) +- Added support for custom identities [#4764](https://github.com/ethyca/fides/pull/4764) ### Changed - Changed the Stripe integration for `Cards` to delete instead of update due to possible issues of a past expiration date [#4768](https://github.com/ethyca/fides/pull/4768) diff --git a/clients/admin-ui/cypress/fixtures/privacy-requests/list.json b/clients/admin-ui/cypress/fixtures/privacy-requests/list.json index 92088a5ec3..ceca22cdd2 100644 --- a/clients/admin-ui/cypress/fixtures/privacy-requests/list.json +++ b/clients/admin-ui/cypress/fixtures/privacy-requests/list.json @@ -12,7 +12,10 @@ "paused_at": null, "status": "pending", "external_id": null, - "identity": { "phone_number": null, "email": "cypress-user@ethyca.com" }, + "identity": { + "phone_number": { "label": "Phone number", "value": null }, + "email": { "label": "Email", "value": "cypress-user@ethyca.com" } + }, "custom_privacy_request_fields": { "account_id": { "label": "Account ID", "value": 1 }, "user_id": { "label": "User ID", "value": "123" }, @@ -61,7 +64,10 @@ "paused_at": null, "status": "pending", "external_id": null, - "identity": { "phone_number": null, "email": "cypress-user@ethyca.com" }, + "identity": { + "phone_number": { "label": "Phone number", "value": null }, + "email": { "label": "Email", "value": "cypress-user@ethyca.com" } + }, "policy": { "name": "default_access_policy", "key": "default_access_policy", @@ -104,7 +110,10 @@ "paused_at": null, "status": "pending", "external_id": null, - "identity": { "phone_number": null, "email": "horse@example.com" }, + "identity": { + "phone_number": { "label": "Phone number", "value": null }, + "email": { "label": "Email", "value": "horse@example.com" } + }, "policy": { "name": "default_access_policy", "key": "default_access_policy", @@ -147,7 +156,10 @@ "paused_at": null, "status": "pending", "external_id": null, - "identity": { "phone_number": null, "email": "cypress-user@ethyca.com" }, + "identity": { + "phone_number": { "label": "Phone number", "value": null }, + "email": { "label": "Email", "value": "cypress-user@ethyca.com" } + }, "policy": { "name": "default_access_policy", "key": "default_access_policy", @@ -190,7 +202,10 @@ "paused_at": null, "status": "complete", "external_id": null, - "identity": { "phone_number": null, "email": "cypress-user@ethyca.com" }, + "identity": { + "phone_number": { "label": "Phone number", "value": null }, + "email": { "label": "Email", "value": "cypress-user@ethyca.com" } + }, "policy": { "name": "default_access_policy", "key": "default_access_policy", @@ -233,7 +248,10 @@ "paused_at": null, "status": "complete", "external_id": null, - "identity": { "phone_number": null, "email": "cypress-user@ethyca.com" }, + "identity": { + "phone_number": { "label": "Phone number", "value": null }, + "email": { "label": "Email", "value": "cypress-user@ethyca.com" } + }, "policy": { "name": "default_access_policy", "key": "default_access_policy", @@ -276,7 +294,10 @@ "paused_at": null, "status": "complete", "external_id": null, - "identity": { "phone_number": null, "email": "cypress-user@ethyca.com" }, + "identity": { + "phone_number": { "label": "Phone number", "value": null }, + "email": { "label": "Email", "value": "cypress-user@ethyca.com" } + }, "policy": { "name": "default_access_policy", "key": "default_access_policy", @@ -320,8 +341,8 @@ "status": "error", "external_id": null, "identity": { - "phone_number": "+14155551234", - "email": "cypress-user@ethyca.com" + "phone_number": { "label": "Phone number", "value": "+14155551234" }, + "email": { "label": "Email", "value": "cypress-user@ethyca.com" } }, "policy": { "name": "default_access_policy", diff --git a/clients/admin-ui/src/features/privacy-requests/ApprovePrivacyRequestModal.tsx b/clients/admin-ui/src/features/privacy-requests/ApprovePrivacyRequestModal.tsx index ff4ef3aa30..8f8c6c32ac 100644 --- a/clients/admin-ui/src/features/privacy-requests/ApprovePrivacyRequestModal.tsx +++ b/clients/admin-ui/src/features/privacy-requests/ApprovePrivacyRequestModal.tsx @@ -52,32 +52,31 @@ const ApprovePrivacyRequestModal = ({ Are you sure you want to approve this privacy request? - {identity.email && ( - - - - Email: - - - {identity.email} ( - {identityVerifiedAt ? "Verified" : "Unverified"}) - - - - )} - {identity.phone_number && ( - - - - Phone Number: - - - {identity.phone_number} ( - {identityVerifiedAt ? "Verified" : "Unverified"}) - - - - )} + {Object.entries(identity) + .filter(([, { value }]) => value !== null) + .map(([key, { value, label }]) => ( + + + + {label}: + + + {value} + + ({identityVerifiedAt ? "Verified" : "Unverified"}) + + + ))} {customPrivacyRequestFields && Object.entries(customPrivacyRequestFields) .filter(([, item]) => item.value) @@ -101,8 +100,8 @@ const ApprovePrivacyRequestModal = ({ {Array.isArray(item.value) ? item.value.join(", ") : item.value}{" "} - (Unverified) + (Unverified) ))} diff --git a/clients/admin-ui/src/features/privacy-requests/RequestDetails.tsx b/clients/admin-ui/src/features/privacy-requests/RequestDetails.tsx index 78b4fee219..51aee3047c 100644 --- a/clients/admin-ui/src/features/privacy-requests/RequestDetails.tsx +++ b/clients/admin-ui/src/features/privacy-requests/RequestDetails.tsx @@ -29,7 +29,13 @@ const RequestDetails = ({ subjectRequest }: RequestDetailsProps) => { return ( <> - + Request details diff --git a/clients/admin-ui/src/features/privacy-requests/RequestRow.tsx b/clients/admin-ui/src/features/privacy-requests/RequestRow.tsx index 7fc402de8b..e28a88e51f 100644 --- a/clients/admin-ui/src/features/privacy-requests/RequestRow.tsx +++ b/clients/admin-ui/src/features/privacy-requests/RequestRow.tsx @@ -162,7 +162,9 @@ const RequestRow = ({ { - - - {identity.email && ( - - - Email: - - - - - - {identityVerifiedAt ? "Verified" : "Unverified"} - - - )} - {identity.phone_number && ( - - - Mobile: - - - - - - {identityVerifiedAt ? "Verified" : "Unverified"} - - - )} + + {Object.entries(identity) + .filter(([, { value }]) => value !== null) + .map(([key, { value, label }]) => ( + + + {label}: + + + + + + {identityVerifiedAt ? "Verified" : "Unverified"} + + + ))} {customPrivacyRequestFields && Object.keys(customPrivacyRequestFields).length > 0 && ( <> - Custom privacy request fields + Custom request fields + {Object.entries(customPrivacyRequestFields) .filter(([, item]) => item.value) .map(([key, item]) => ( diff --git a/clients/admin-ui/src/features/privacy-requests/types.ts b/clients/admin-ui/src/features/privacy-requests/types.ts index e3df855757..c74685c115 100644 --- a/clients/admin-ui/src/features/privacy-requests/types.ts +++ b/clients/admin-ui/src/features/privacy-requests/types.ts @@ -71,8 +71,7 @@ export interface PrivacyRequestEntity { status: PrivacyRequestStatus; results?: PrivacyRequestResults; identity: { - email?: string; - phone_number?: string; + [key: string]: { label: string; value: any }; }; identity_verified_at?: string; custom_privacy_request_fields?: { diff --git a/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts b/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts index e61503a6aa..a1045b6755 100644 --- a/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts +++ b/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts @@ -19,6 +19,7 @@ describe("Smoke test", () => { cy.getByTestId("card").contains("Access your data").click(); cy.getByTestId("privacy-request-form").within(() => { cy.get("input#email").type("jenny@example.com"); + cy.get("input#loyalty_id").type("CH-1"); cy.get("input#first_name").type("Jenny"); cy.get("input#color").clear().type("blue"); cy.get("button").contains("Continue").click(); @@ -30,17 +31,13 @@ describe("Smoke test", () => { { identity: { email: "jenny@example.com", - phone_number: "", + loyalty_id: { label: "Loyalty ID", value: "CH-1" }, }, custom_privacy_request_fields: { first_name: { label: "First name", value: "Jenny", }, - last_name: { - label: "Last name", - value: "", - }, color: { label: "Color", value: "blue", diff --git a/clients/privacy-center/components/modals/privacy-request-modal/PrivacyRequestForm.tsx b/clients/privacy-center/components/modals/privacy-request-modal/PrivacyRequestForm.tsx index ff6dd4b0a0..9a082aa680 100644 --- a/clients/privacy-center/components/modals/privacy-request-modal/PrivacyRequestForm.tsx +++ b/clients/privacy-center/components/modals/privacy-request-modal/PrivacyRequestForm.tsx @@ -19,29 +19,30 @@ import { Headers } from "headers-polyfill"; import { addCommonHeaders } from "~/common/CommonHeaders"; import { ErrorToastOptions, SuccessToastOptions } from "~/common/toast-options"; import { PrivacyRequestStatus } from "~/types"; -import { PrivacyRequestOption } from "~/types/config"; +import { CustomIdentity, PrivacyRequestOption } from "~/types/config"; import { defaultIdentityInput } from "~/constants"; import { PhoneInput } from "~/components/phone-input"; import { ModalViews } from "~/components/modals/types"; import { FormErrorMessage } from "~/components/FormErrorMessage"; import { emailValidation, - nameValidation, phoneValidation, } from "~/components/modals/validation"; import { useConfig } from "~/features/common/config.slice"; import { useSettings } from "~/features/common/settings.slice"; -type KnownKeys = { - name: string; - email: string; - phone: string; -}; - -type FormValues = KnownKeys & { +type FormValues = { [key: string]: any; }; +/** + * + * @param value + * @returns Default to null if the value is undefined or an empty string + */ +const fallbackNull = (value: any) => + value === undefined || value === "" ? null : value; + const usePrivacyRequestForm = ({ onClose, action, @@ -62,9 +63,9 @@ const usePrivacyRequestForm = ({ const toast = useToast(); const formik = useFormik({ initialValues: { - name: "", - email: "", - phone: "", + ...Object.fromEntries( + Object.entries(identityInputs).map(([key]) => [key, ""]) + ), ...Object.fromEntries( Object.entries(customPrivacyRequestFields) .filter(([, field]) => !field.hidden) @@ -77,32 +78,39 @@ const usePrivacyRequestForm = ({ return; } - const { email, phone, name, ...customPrivacyRequestFieldValues } = values; + // extract identity input values + const identityInputValues = Object.fromEntries( + Object.entries(action.identity_inputs ?? {}).map(([key, field]) => { + const value = fallbackNull(values[key]); + if (typeof field === "string") { + return [key, value]; + } + return [key, { label: field.label, value }]; + }) + ); - // populate the values from the form or from the field's default value - const transformedCustomPrivacyRequestFields = Object.fromEntries( - Object.entries(action.custom_privacy_request_fields ?? {}).map( - ([key, field]) => [ + // extract custom privacy request field values + const customPrivacyRequestFieldValues = Object.fromEntries( + Object.entries(action.custom_privacy_request_fields ?? {}) + .map(([key, field]) => [ key, { label: field.label, value: field.hidden ? field.default_value - : customPrivacyRequestFieldValues[key] || "", + : fallbackNull(values[key]), }, - ] - ) + ]) + // @ts-ignore + .filter(([, { value }]) => value !== null) ); const body = [ { - identity: { - email, - phone_number: phone, - // enable this when name field is supported on the server - // name: values.name - }, - custom_privacy_request_fields: transformedCustomPrivacyRequestFields, + identity: identityInputValues, + ...(Object.keys(customPrivacyRequestFieldValues).length > 0 && { + custom_privacy_request_fields: customPrivacyRequestFieldValues, + }), policy_key: action.policy_key, }, ]; @@ -175,7 +183,6 @@ const usePrivacyRequestForm = ({ } }, validationSchema: Yup.object().shape({ - name: nameValidation(identityInputs?.name), email: emailValidation(identityInputs?.email).test( "one of email or phone entered", "You must enter either email or phone", @@ -202,6 +209,17 @@ const usePrivacyRequestForm = ({ return true; } ), + ...Object.fromEntries( + Object.entries(identityInputs) + .filter(([key]) => key !== "email" && key !== "phone") + .map(([key, value]) => { + const customIdentity = value as CustomIdentity; + return [ + key, + Yup.string().required(`${customIdentity.label} is required`), + ]; + }) + ), ...Object.fromEntries( Object.entries(customPrivacyRequestFields) .filter(([, field]) => !field.hidden) @@ -288,70 +306,41 @@ const PrivacyRequestForm: React.FC = ({ ))} - {identityInputs.name ? ( - - Name - - {errors.name} - - ) : null} - {identityInputs.email ? ( + {Object.entries(identityInputs).map(([key, item]) => ( - Email - - {errors.email} - - ) : null} - {identityInputs.phone ? ( - - Phone - { - setFieldValue("phone", value, true); - }} - onBlur={handleBlur} - value={values.phone} - isDisabled={Boolean( - typeof values.email !== "undefined" && values.email - )} - /> - {errors.phone} + + {typeof item === "string" + ? key[0].toUpperCase() + key.slice(1) + : item.label} + + {key === "phone" ? ( + { + setFieldValue(key, value, true); + }} + onBlur={handleBlur} + value={values[key]} + /> + ) : ( + + )} + {errors[key]} - ) : null} + ))} {Object.entries(customPrivacyRequestFields) .filter(([, field]) => !field.hidden) .map(([key, item]) => ( diff --git a/clients/privacy-center/types/config.ts b/clients/privacy-center/types/config.ts index 8806b63e31..0f9f0d5ac3 100644 --- a/clients/privacy-center/types/config.ts +++ b/clients/privacy-center/types/config.ts @@ -1,11 +1,17 @@ import { ConsentValue } from "fides-js"; -export type IdentityInputs = { - name?: string; +type DefaultIdentities = { email?: string; phone?: string; }; +export type CustomIdentity = { + label: string; +}; + +export type IdentityInputs = DefaultIdentities & + (Record | {}); + export type CustomPrivacyRequestFields = Record< string, { diff --git a/data/dataset/postgres_example_test_extended_dataset.yml b/data/dataset/postgres_example_test_extended_dataset.yml new file mode 100644 index 0000000000..9a396bd2a4 --- /dev/null +++ b/data/dataset/postgres_example_test_extended_dataset.yml @@ -0,0 +1,23 @@ +dataset: + - fides_key: postgres_example_test_extended_dataset + name: Postgres Example Test Extended Dataset + description: Contains a reference to a collection that can only be reached by the custom `loyalty_id` identity (for testing purposes) + collections: + - name: loyalty + fields: + - name: id + data_categories: [user.unique_id] + fides_meta: + identity: loyalty_id + - name: name + data_categories: [user.name] + fides_meta: + data_type: string + - name: points + data_categories: [user.content] + fides_meta: + data_type: integer + - name: tier + data_categories: [user.content] + fides_meta: + data_type: string diff --git a/requirements.txt b/requirements.txt index a54d1e20b7..3352f70dc4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ GitPython==3.1.41 httpx==0.23.1 hvac==0.11.2 iab-tcf==0.2.2 +immutables==0.20 importlib_resources==5.12.0 Jinja2==3.1.3 loguru==0.6.0 @@ -58,4 +59,4 @@ toml==0.10.2 twilio==7.15.0 typing_extensions==4.5.0 # pinned to work around https://github.com/pydantic/pydantic/issues/5821 validators==0.20.0 -versioneer==0.19 +versioneer==0.19 \ No newline at end of file diff --git a/src/fides/api/alembic/migrations/versions/d49a767eb49d_convert_field_name_to_string_type.py b/src/fides/api/alembic/migrations/versions/d49a767eb49d_convert_field_name_to_string_type.py new file mode 100644 index 0000000000..0942b09ed9 --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/d49a767eb49d_convert_field_name_to_string_type.py @@ -0,0 +1,44 @@ +"""convert field_name to string type + +Revision ID: d49a767eb49d +Revises: e4023342ebbb +Create Date: 2024-03-26 04:35:22.358246 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d49a767eb49d" +down_revision = "e4023342ebbb" +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column( + "providedidentity", + "field_name", + existing_type=sa.Enum("providedidentitytype"), + type_=sa.String(), + existing_nullable=False, + ) + op.execute("DROP TYPE providedidentitytype") + op.add_column( + "providedidentity", sa.Column("field_label", sa.String(), nullable=True) + ) + + +def downgrade(): + op.drop_column("providedidentity", "field_label") + op.execute( + "CREATE TYPE providedidentitytype AS ENUM('email', 'phone_number', 'ga_client_id', 'ljt_readerID', 'fides_user_device_id')" + ) + op.alter_column( + "providedidentity", + "field_name", + existing_type=sa.String(), + type_=sa.Enum("providedidentitytype"), + existing_nullable=False, + ) diff --git a/src/fides/api/api/v1/endpoints/consent_request_endpoints.py b/src/fides/api/api/v1/endpoints/consent_request_endpoints.py index ec939e3348..24cb46df9a 100644 --- a/src/fides/api/api/v1/endpoints/consent_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/consent_request_endpoints.py @@ -375,7 +375,7 @@ def queue_privacy_request_to_propagate_consent_old_workflow( identity = browser_identity if browser_identity else Identity() setattr( identity, - provided_identity.field_name.value, # type:ignore[attr-defined] + provided_identity.field_name, # type:ignore[attr-defined] provided_identity.encrypted_value["value"], # type:ignore[index] ) # Pull the information on the ProvidedIdentity for the ConsentRequest to pass along to create a PrivacyRequest @@ -508,7 +508,7 @@ def _get_or_create_provided_identity( identity = ProvidedIdentity.filter( db=db, conditions=( - (ProvidedIdentity.field_name == ProvidedIdentityType.email) + (ProvidedIdentity.field_name == ProvidedIdentityType.email.value) & ( ProvidedIdentity.hashed_value == ProvidedIdentity.hash_value(identity_data.email) @@ -533,7 +533,7 @@ def _get_or_create_provided_identity( identity = ProvidedIdentity.filter( db=db, conditions=( - (ProvidedIdentity.field_name == ProvidedIdentityType.phone_number) + (ProvidedIdentity.field_name == ProvidedIdentityType.phone_number.value) & ( ProvidedIdentity.hashed_value == ProvidedIdentity.hash_value(identity_data.phone_number) diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index a64a9c49fb..d7dd17eadd 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -609,7 +609,9 @@ def get_request_status( for item in paginated.items: # type: ignore if include_identities: - item.identity = item.get_persisted_identity().dict() + item.identity = item.get_persisted_identity().labeled_dict( + include_default_labels=True + ) if include_custom_privacy_request_fields: item.custom_privacy_request_fields = ( diff --git a/src/fides/api/db/samples.py b/src/fides/api/db/samples.py index 60054e32a8..46cc3a12f2 100644 --- a/src/fides/api/db/samples.py +++ b/src/fides/api/db/samples.py @@ -4,6 +4,7 @@ See load_samples() in seed.py for usage. """ + from typing import Dict, List, Optional, TextIO import yaml @@ -106,20 +107,19 @@ def load_sample_connections_from_project() -> List[SampleConnection]: [SampleConnection.parse_obj(e) for e in connections] ) - # Disable any connections whose "secrets" dict has empty values + # Exclude any connections whose "secrets" dict has empty values + valid_connections = [] for connection in sample_connections: - # If there are no secrets at all, disable! + # If there are no secrets at all, skip this connection if not connection.secrets: - connection.disabled = True continue - # If any of the secret values are missing, disable! - for _, value in dict(connection.secrets).items(): - if not value or value == "": - connection.disabled = True + # Check if all secret values are present and non-empty + if all(value and value != "" for value in connection.secrets.values()): # type: ignore + valid_connections.append(connection) - # Exclude any disabled connections from the final results - return [e for e in sample_connections if not e.disabled] + # Exclude any invalid connections from the final results + return valid_connections def load_sample_yaml_file(file: TextIO, expand_vars: bool = True) -> Dict: diff --git a/src/fides/api/models/privacy_request.py b/src/fides/api/models/privacy_request.py index a071f4770a..707702938c 100644 --- a/src/fides/api/models/privacy_request.py +++ b/src/fides/api/models/privacy_request.py @@ -61,9 +61,10 @@ CustomPrivacyRequestField as CustomPrivacyRequestFieldSchema, ) from fides.api.schemas.redis_cache import ( - CustomPrivacyRequestFieldValue, Identity, IdentityBase, + LabeledIdentity, + MultiValue, ) from fides.api.tasks import celery_app from fides.api.util.cache import ( @@ -342,12 +343,17 @@ def delete(self, db: Session) -> None: def cache_identity(self, identity: Identity) -> None: """Sets the identity's values at their specific locations in the Fides app cache""" cache: FidesopsRedis = get_cache() - identity_dict: Dict[str, Any] = dict(identity) + + if isinstance(identity, dict): + identity = Identity(**identity) + + identity_dict: Dict[str, Any] = identity.labeled_dict() + for key, value in identity_dict.items(): if value is not None: cache.set_with_autoexpire( get_identity_cache_key(self.id, key), - value, + FidesopsRedis.encode_obj(value), ) def cache_custom_privacy_request_fields( @@ -382,19 +388,39 @@ def persist_identity(self, db: Session, identity: Identity) -> None: Stores the identity provided with the privacy request in a secure way, compatible with blind indexing for later searching and audit purposes. """ - identity_dict: Dict[str, Any] = dict(identity) + + if isinstance(identity, dict): + identity = Identity(**identity) + + identity_dict = identity.labeled_dict() for key, value in identity_dict.items(): - if value: + if value is not None: + if isinstance(value, dict): + if "label" in value and "value" in value: + label = value["label"] + value = value["value"] + else: + raise RuntimeError( + f"Programming error: unexpected dict value '{value}' found in an Identity's `labeled_dict()`!" + ) + else: + label = None + hashed_value = ProvidedIdentity.hash_value(value) + provided_identity_data = { + "privacy_request_id": self.id, + "field_name": key, + # We don't need to manually encrypt this field, it's done at the ORM level + "encrypted_value": {"value": value}, + "hashed_value": hashed_value, + } + + if label is not None: + provided_identity_data["field_label"] = label + ProvidedIdentity.create( db=db, - data={ - "privacy_request_id": self.id, - "field_name": key, - # We don't need to manually encrypt this field, it's done at the ORM level - "encrypted_value": {"value": value}, - "hashed_value": hashed_value, - }, + data=provided_identity_data, ) def persist_custom_privacy_request_fields( @@ -429,14 +455,13 @@ def get_persisted_identity(self) -> Identity: """ Retrieves persisted identity fields from the DB. """ - schema = Identity() + schema_dict = {} for field in self.provided_identities: # type: ignore[attr-defined] - setattr( - schema, - field.field_name.value, - field.encrypted_value["value"], - ) - return schema + value = field.encrypted_value.get("value") + if field.field_label: + value = LabeledIdentity(label=field.field_label, value=value) + schema_dict[field.field_name] = value + return Identity(**schema_dict) def get_persisted_custom_privacy_request_fields(self) -> Dict[str, Any]: return { @@ -531,7 +556,12 @@ def get_cached_identity_data(self) -> Dict[str, Any]: prefix = f"id-{self.id}-identity-*" cache: FidesopsRedis = get_cache() keys = cache.keys(prefix) - return {key.split("-")[-1]: cache.get(key) for key in keys} + result = {} + for key in keys: + value = cache.get(key) + if value: + result[key.split("-")[-1]] = json.loads(value) + return result def get_cached_custom_privacy_request_fields(self) -> Dict[str, Any]: """Retrieves any custom fields pertaining to this request from the cache""" @@ -1045,10 +1075,15 @@ class ProvidedIdentity(Base): # pylint: disable=R0904 ) # Which privacy request this identity belongs to field_name = Column( - EnumColumn(ProvidedIdentityType), + String, index=False, nullable=False, ) + field_label = Column( + String, + index=False, + nullable=True, + ) hashed_value = Column( String, index=True, @@ -1078,34 +1113,35 @@ class ProvidedIdentity(Base): # pylint: disable=R0904 @classmethod def hash_value( cls, - value: str, + value: MultiValue, encoding: str = "UTF-8", ) -> str: """Utility function to hash the value with a generated salt""" SALT = "$2b$12$UErimNtlsE6qgYf2BrI1Du" + value_str = str(value) hashed_value = hash_with_salt( - value.encode(encoding), + value_str.encode(encoding), SALT.encode(encoding), ) return hashed_value def as_identity_schema(self) -> Identity: """Creates an Identity schema from a ProvidedIdentity record in the application DB.""" - identity = Identity() + + identity_dict = {} if any( [ not self.field_name, not self.encrypted_value, ] ): - return identity + return Identity() - setattr( - identity, - self.field_name.value, # type:ignore - self.encrypted_value.get("value"), # type:ignore - ) - return identity + value = self.encrypted_value.get("value") # type:ignore + if self.field_label: + value = LabeledIdentity(label=self.field_label, value=value) + identity_dict[self.field_name] = value + return Identity(**identity_dict) class CustomPrivacyRequestField(Base): @@ -1156,7 +1192,7 @@ def __tablename__(self) -> str: @classmethod def hash_value( cls, - value: CustomPrivacyRequestFieldValue, + value: MultiValue, encoding: str = "UTF-8", ) -> Union[str, List[str]]: """Utility function to hash the value(s) with a generated salt""" diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py index 6172180b7c..c1ba35dfd9 100644 --- a/src/fides/api/schemas/privacy_request.py +++ b/src/fides/api/schemas/privacy_request.py @@ -207,7 +207,7 @@ class PrivacyRequestResponse(FidesSchema): # as it is an API response field, and we don't want to reveal any more # about our PII structure than is explicitly stored in the cache on request # creation. - identity: Optional[Dict[str, Optional[str]]] + identity: Optional[Dict[str, Union[Optional[str], Dict[str, Any]]]] custom_privacy_request_fields: Optional[Dict[str, Any]] policy: PolicySchema action_required_details: Optional[CheckpointActionRequiredDetails] = None diff --git a/src/fides/api/schemas/redis_cache.py b/src/fides/api/schemas/redis_cache.py index 2a8f017b99..3d87bb41f1 100644 --- a/src/fides/api/schemas/redis_cache.py +++ b/src/fides/api/schemas/redis_cache.py @@ -1,11 +1,13 @@ import uuid -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Union -from pydantic import EmailStr, Extra, StrictInt, StrictStr, validator +from pydantic import EmailStr, Extra, Field, StrictInt, StrictStr, validator from fides.api.custom_types import PhoneNumber from fides.api.schemas.base_class import FidesSchema +MultiValue = Union[StrictInt, StrictStr, List[Union[StrictInt, StrictStr]]] + class IdentityBase(FidesSchema): """The minimum fields required to represent an identity.""" @@ -19,20 +21,43 @@ class Config: extra = Extra.forbid +class LabeledIdentity(FidesSchema): + """ + An identity value with its accompanying UI label + """ + + label: str + value: MultiValue + + class Identity(IdentityBase): """Some PII grouping pertaining to a human""" # These are repeated so we can continue to forbid extra fields - phone_number: Optional[PhoneNumber] = None - email: Optional[EmailStr] = None - ga_client_id: Optional[str] = None - ljt_readerID: Optional[str] = None - fides_user_device_id: Optional[str] = None + phone_number: Optional[PhoneNumber] = Field(None, title="Phone number") + email: Optional[EmailStr] = Field(None, title="Email") + ga_client_id: Optional[str] = Field(None, title="GA client ID") + ljt_readerID: Optional[str] = Field(None, title="LJT reader ID") + fides_user_device_id: Optional[str] = Field(None, title="Fides user device ID") class Config: - """Only allow phone_number, and email.""" - - extra = Extra.forbid + """Allows extra fields to be provided but they must have a value of type LabeledIdentity.""" + + extra = Extra.allow + + def __init__(self, **data: Any): + for field, value in data.items(): + if field not in self.__fields__: + if isinstance(value, LabeledIdentity): + data[field] = value + elif isinstance(value, dict) and "label" in value and "value" in value: + data[field] = LabeledIdentity(**value) + else: + raise ValueError( + f'Custom identity "{field}" must be an instance of LabeledIdentity ' + '(e.g. {"label": "Field label", "value": "123"})' + ) + super().__init__(**data) @validator("fides_user_device_id") @classmethod @@ -43,10 +68,38 @@ def validate_fides_user_device_id(cls, v: Optional[str]) -> Optional[str]: uuid.UUID(v, version=4) return v - -CustomPrivacyRequestFieldValue = Union[ - Union[StrictInt, StrictStr], List[Union[StrictInt, StrictStr]] -] + def dict(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Returns a dictionary with LabeledIdentity values returned as simple values. + """ + d = super().dict(*args, **kwargs) + for key, value in self.__dict__.items(): + if isinstance(value, LabeledIdentity): + d[key] = value.value + else: + d[key] = value + return d + + def labeled_dict( + self, include_default_labels: Optional[bool] = False + ) -> Dict[str, Any]: + """Returns a dictionary that preserves the labels for all custom/labeled identities.""" + d = {} + for key, value in self.__dict__.items(): + if key in self.__fields__: + if include_default_labels: + d[key] = { + "label": self.__fields__[key].field_info.title, + "value": value, + } + else: + d[key] = value + else: + if isinstance(value, LabeledIdentity): + d[key] = value.dict() + else: + d[key] = value + return d class CustomPrivacyRequestField(FidesSchema): @@ -54,4 +107,4 @@ class CustomPrivacyRequestField(FidesSchema): label: str # use StrictInt and StrictStr to avoid type coercion and maintain the original types - value: CustomPrivacyRequestFieldValue + value: MultiValue diff --git a/src/fides/api/service/connectors/fides/fides_client.py b/src/fides/api/service/connectors/fides/fides_client.py index a2f65c9bf2..9b3c9a0e56 100644 --- a/src/fides/api/service/connectors/fides/fides_client.py +++ b/src/fides/api/service/connectors/fides/fides_client.py @@ -220,9 +220,9 @@ def request_status(self, privacy_request_id: str = "") -> List[Dict[str, Any]]: request: Request = self.authenticated_request( method="GET", path=urls.V1_URL_PREFIX + urls.PRIVACY_REQUESTS, - query_params={"request_id": privacy_request_id} - if privacy_request_id - else None, + query_params=( + {"request_id": privacy_request_id} if privacy_request_id else None + ), ) response = self.session.send(request) diff --git a/src/fides/api/service/connectors/fides_connector.py b/src/fides/api/service/connectors/fides_connector.py index 5e443ba3a2..e709bb5b7f 100644 --- a/src/fides/api/service/connectors/fides_connector.py +++ b/src/fides/api/service/connectors/fides_connector.py @@ -88,7 +88,10 @@ def retrieve_data( input_data: Dict[str, List[Any]], ) -> List[Row]: """Execute access request and fetch access data from remote Fides""" - identity_data = privacy_request.get_cached_identity_data() + identity_data = { + **privacy_request.get_persisted_identity().labeled_dict(), + **privacy_request.get_cached_identity_data(), + } if not identity_data: raise FidesError( f"No identity data found for privacy request {privacy_request.id}, cannot execute Fides connector!" @@ -136,7 +139,10 @@ def mask_data( input_data: Dict[str, List[Any]], ) -> int: """Execute an erasure request on remote fides""" - identity_data = privacy_request.get_cached_identity_data() + identity_data = { + **privacy_request.get_persisted_identity().labeled_dict(), + **privacy_request.get_cached_identity_data(), + } if not identity_data: raise FidesError( f"No identity data found for privacy request {privacy_request.id}, cannot execute Fides connector!" diff --git a/src/fides/api/service/connectors/saas_connector.py b/src/fides/api/service/connectors/saas_connector.py index 195f97b73f..a1272869d7 100644 --- a/src/fides/api/service/connectors/saas_connector.py +++ b/src/fides/api/service/connectors/saas_connector.py @@ -192,7 +192,7 @@ def retrieve_data( # 1) If a collection can be retrieved by using different identities such as email or phone number # 2) The complete set of results for a collection is made up of subsets. For example, to retrieve all tickets # we must change a 'status' query param from 'active' to 'pending' and finally 'closed' - read_requests: List[SaaSRequest] = query_config.get_read_requests_by_identity() + read_requests: List[SaaSRequest] = query_config.get_read_requests() delete_request: Optional[ SaaSRequest ] = query_config.get_erasure_request_by_action("delete") diff --git a/src/fides/api/service/connectors/saas_query_config.py b/src/fides/api/service/connectors/saas_query_config.py index 535cb35a55..438161a438 100644 --- a/src/fides/api/service/connectors/saas_query_config.py +++ b/src/fides/api/service/connectors/saas_query_config.py @@ -27,7 +27,6 @@ MASKED_OBJECT_FIELDS, PRIVACY_REQUEST_ID, UUID, - get_identity, unflatten_dict, ) from fides.config import CONFIG @@ -55,9 +54,9 @@ def __init__( self.action: Optional[str] = None self.current_request: Optional[SaaSRequest] = None - def get_read_requests_by_identity(self) -> List[SaaSRequest]: + def get_read_requests(self) -> List[SaaSRequest]: """ - Returns the appropriate request configs based on the current collection and identity + Returns the appropriate request configs based on the current collection """ collection_name = self.node.address.collection @@ -70,27 +69,7 @@ def get_read_requests_by_identity(self) -> List[SaaSRequest]: if not requests.read: return [] - read_requests = ( - requests.read if isinstance(requests.read, list) else [requests.read] - ) - filtered_requests = self._requests_using_identity(read_requests) - # return all the requests if none contained an identity reference - return read_requests if not filtered_requests else filtered_requests - - def _requests_using_identity( - self, requests: List[SaaSRequest] - ) -> List[SaaSRequest]: - """Filters for the requests using the provided identity""" - - return [ - request - for request in requests - if any( - param_value - for param_value in request.param_values or [] - if param_value.identity == get_identity(self.privacy_request) - ) - ] + return requests.read if isinstance(requests.read, list) else [requests.read] def get_erasure_request_by_action( self, action: Literal["update", "delete"] diff --git a/src/fides/api/service/privacy_request/dsr_package/dsr_report_builder.py b/src/fides/api/service/privacy_request/dsr_package/dsr_report_builder.py index 2a48f356be..1fcf12f5ff 100644 --- a/src/fides/api/service/privacy_request/dsr_package/dsr_report_builder.py +++ b/src/fides/api/service/privacy_request/dsr_package/dsr_report_builder.py @@ -11,7 +11,6 @@ from fides.api.models.privacy_request import PrivacyRequest from fides.api.schemas.policy import ActionType -from fides.api.schemas.redis_cache import Identity from fides.api.util.storage_util import storage_json_encoder DSR_DIRECTORY = Path(__file__).parent.resolve() @@ -186,14 +185,21 @@ def generate(self) -> BytesIO: def _map_privacy_request(privacy_request: PrivacyRequest) -> Dict[str, Any]: """Creates a map with a subset of values from the privacy request""" - request_data = {} + request_data: Dict[str, Any] = {} request_data["id"] = privacy_request.id + action_type: Optional[ActionType] = privacy_request.policy.get_action_type() if action_type: request_data["type"] = action_type.value - identity: Identity = privacy_request.get_persisted_identity() - if identity.email: - request_data["email"] = identity.email + + request_data["identity"] = { + key: value + for key, value in privacy_request.get_persisted_identity() + .labeled_dict(include_default_labels=True) + .items() + if value["value"] is not None + } + if privacy_request.requested_at: request_data["requested_at"] = privacy_request.requested_at.strftime( "%m/%d/%Y %H:%M %Z" diff --git a/src/fides/api/service/privacy_request/dsr_package/templates/welcome.html b/src/fides/api/service/privacy_request/dsr_package/templates/welcome.html index c90daabadf..77ea0a2215 100644 --- a/src/fides/api/service/privacy_request/dsr_package/templates/welcome.html +++ b/src/fides/api/service/privacy_request/dsr_package/templates/welcome.html @@ -37,8 +37,10 @@

Your requested data

{{ request.id }}
Request type:
{{ request.type }}
-
Email:
-
{{ request.email }}
+ {% for identity_type, identity_data in request.identity.items() %} +
{{ identity_data.label }}:
+
{{ identity_data.value }}
+ {% endfor %}
Requested at:
{{ request.requested_at }}
diff --git a/src/fides/api/service/privacy_request/request_runner_service.py b/src/fides/api/service/privacy_request/request_runner_service.py index 59ddd87264..0f8121de40 100644 --- a/src/fides/api/service/privacy_request/request_runner_service.py +++ b/src/fides/api/service/privacy_request/request_runner_service.py @@ -379,7 +379,10 @@ async def run_privacy_request( datasets = DatasetConfig.all(db=session) dataset_graphs = [dataset_config.get_graph() for dataset_config in datasets] dataset_graph = DatasetGraph(*dataset_graphs) - identity_data = privacy_request.get_cached_identity_data() + identity_data = { + key: value["value"] if isinstance(value, dict) else value + for key, value in privacy_request.get_cached_identity_data().items() + } connection_configs = ConnectionConfig.all(db=session) fides_connector_datasets: Set[str] = filter_fides_connector_datasets( connection_configs diff --git a/src/fides/api/task/graph_task.py b/src/fides/api/task/graph_task.py index 14eea60222..09b36fdab4 100644 --- a/src/fides/api/task/graph_task.py +++ b/src/fides/api/task/graph_task.py @@ -11,6 +11,7 @@ from dask.core import getcycle from dask.threaded import get from loguru import logger +from ordered_set import OrderedSet from sqlalchemy.orm import Session from fides.api.common_exceptions import ( @@ -51,8 +52,10 @@ from fides.api.util.collection_util import ( NodeInput, Row, - append, + append_unique, extract_key_for_address, + make_immutable, + make_mutable, partition, ) from fides.api.util.consent_util import add_errored_system_status_for_consent_reporting @@ -305,8 +308,8 @@ def pre_process_input_data( table1: [{x:1, y:A}, {x:2, y:B}], table2: [{x:3},{x:4}], table3: [{z: {a: C}, "y": [4, 5]}] where table1.x => self.id, - table1.y=> self.name, - table2.x=>self.id + table1.y => self.name, + table2.x => self.id table3.z.a => self.contact.address table3.y => self.contact.email becomes @@ -314,6 +317,9 @@ def pre_process_input_data( If there are dependent fields from one collection into another, they are separated out as follows: {fidesops_grouped_inputs: [{"organization_id": 1, "project_id": "math}, {"organization_id": 5, "project_id": "science"}] + + The output dictionary is constructed with deduplicated values for each key, ensuring that the value lists + and the fides_grouped_input list contain only unique elements. """ if not len(data) == len(self.input_keys): logger.warning( @@ -323,7 +329,8 @@ def pre_process_input_data( len(data), ) - output: Dict[str, List[Any]] = {FIDESOPS_GROUPED_INPUTS: []} + # the ordered set is just to have a consistent output for testing, the order is not needed otherwise + output: Dict[str, OrderedSet] = {FIDESOPS_GROUPED_INPUTS: OrderedSet()} ( independent_field_mappings, @@ -355,7 +362,7 @@ def pre_process_input_data( row=row, target_path=foreign_field_path ) if new_values: - append(output, local_field_path.string_path, new_values) + append_unique(output, local_field_path.string_path, new_values) # Separately group together dependent inputs if applicable if dependent_field_mappings[collection_address]: @@ -376,8 +383,9 @@ def pre_process_input_data( dependent_field_mappings=dependent_field_mappings, ) - output[FIDESOPS_GROUPED_INPUTS].append(grouped_data) - return output + output[FIDESOPS_GROUPED_INPUTS].add(make_immutable(grouped_data)) + + return make_mutable(output) def update_status( self, diff --git a/src/fides/api/util/collection_util.py b/src/fides/api/util/collection_util.py index acde73e0cd..6484a1f169 100644 --- a/src/fides/api/util/collection_util.py +++ b/src/fides/api/util/collection_util.py @@ -1,5 +1,8 @@ from functools import reduce -from typing import Any, Callable, Dict, Iterable, List, Optional, TypeVar +from typing import Any, Callable, Dict, Iterable, List, Optional, TypeVar, Union + +import immutables +from ordered_set import OrderedSet T = TypeVar("T") U = TypeVar("U") @@ -9,6 +12,34 @@ FIDESOPS_DO_NOT_MASK_INDEX = "FIDESOPS_DO_NOT_MASK" +def make_immutable(obj: Any) -> Any: + """ + Recursively converts a mutable object into an immutable version. + Dictionaries are converted to immutable `Map`s from the `immutables` library, + lists are converted to tuples, and other objects are returned unchanged. + """ + if isinstance(obj, dict): + return immutables.Map( + {key: make_immutable(value) for key, value in obj.items()} + ) + if isinstance(obj, list): + return tuple(make_immutable(item) for item in obj) + return obj + + +def make_mutable(obj: Any) -> Any: + """ + Recursively converts an immutable object into a mutable version. + `Map`s from the `immutables` library and dictionaries are converted to mutable dictionaries, + tuples and `OrderedSet`s are converted to lists, and other objects are returned unchanged. + """ + if isinstance(obj, (dict, immutables.Map)): + return {key: make_mutable(value) for key, value in obj.items()} + if isinstance(obj, (tuple, OrderedSet)): + return [make_mutable(item) for item in obj] + return obj + + def merge_dicts(*dicts: Dict[T, U]) -> Dict[T, U]: """Merge any number of dictionaries. @@ -38,6 +69,24 @@ def append(d: Dict[T, List[U]], key: T, value: U) -> None: d[key] = value if isinstance(value, list) else [value] +def append_unique(d: Dict[T, OrderedSet[U]], key: T, value: Union[U, List[U]]) -> None: + """Append to values stored under a dictionary key. + + append_unique({}, "A", 1) sets dict to {"A": {1}} + append_unique({"A": {1}}, "A", 2) sets dict to {"A": {1, 2}} + append_unique({"A": {1}}, "A", [2, 3, 4]) sets dict to {"A": {1, 2, 3, 4}} + append_unique({"A": {1, 2}}, "A", [2, 3, 4]) sets dict to {"A": {1, 2, 3, 4}} + """ + if value: + if key not in d: + d[key] = OrderedSet() + + if isinstance(value, list): + d[key].update(make_immutable(value)) + else: + d[key].add(make_immutable(value)) + + def partition(_iterable: Iterable[T], extractor: Callable[[T], U]) -> Dict[U, List[T]]: """partition a collection by the output of an arbitrary extractor function""" out: Dict[U, List[T]] = {} diff --git a/src/fides/api/util/consent_util.py b/src/fides/api/util/consent_util.py index a67815153e..59c03c4511 100644 --- a/src/fides/api/util/consent_util.py +++ b/src/fides/api/util/consent_util.py @@ -201,7 +201,10 @@ def get_fides_user_device_id_provided_identity( return ProvidedIdentity.filter( db=db, conditions=( - (ProvidedIdentity.field_name == ProvidedIdentityType.fides_user_device_id) + ( + ProvidedIdentity.field_name + == ProvidedIdentityType.fides_user_device_id.value + ) & ( ProvidedIdentity.hashed_value == ProvidedIdentity.hash_value(fides_user_device_id) diff --git a/src/fides/data/sample_project/postgres_sample.sql b/src/fides/data/sample_project/postgres_sample.sql index 6156c45a91..94f156941a 100644 --- a/src/fides/data/sample_project/postgres_sample.sql +++ b/src/fides/data/sample_project/postgres_sample.sql @@ -102,6 +102,13 @@ CREATE TABLE public.type_link_test ( name CHARACTER VARYING(100) ); +CREATE TABLE public.loyalty ( + id CHARACTER VARYING(100) PRIMARY KEY, + name CHARACTER VARYING(100), + points INT, + tier CHARACTER VARYING(100) +); + INSERT INTO public.composite_pk_test VALUES (1,10,'linked to customer 1',1), (1,11,'linked to customer 2',2), @@ -189,6 +196,10 @@ INSERT INTO public.type_link_test VALUES ('1', 'name1'), ('2', 'name2'); +INSERT INTO public.loyalty VALUES +('CH-1', 'Jane Customer', 100, 'Cookie Rookie'), +('CH-2', 'John Customer', 200, 'Cookie Connoisseur'); + CREATE SCHEMA backup_schema; CREATE TABLE backup_schema.product (LIKE public.product INCLUDING ALL); diff --git a/src/fides/data/sample_project/privacy_center/config/config.json b/src/fides/data/sample_project/privacy_center/config/config.json index 7a8eaaac6e..57ca6b7ff8 100644 --- a/src/fides/data/sample_project/privacy_center/config/config.json +++ b/src/fides/data/sample_project/privacy_center/config/config.json @@ -15,7 +15,10 @@ "description": "We will provide you a report of all your personal data.", "description_subtext": [], "identity_inputs": { - "email": "required" + "email": "required", + "loyalty_id": { + "label": "Loyalty ID" + } }, "custom_privacy_request_fields": { "first_name": { diff --git a/src/fides/data/sample_project/sample_connections/sample_connections.yml b/src/fides/data/sample_project/sample_connections/sample_connections.yml index 11d3737c18..c7d9993a93 100644 --- a/src/fides/data/sample_project/sample_connections/sample_connections.yml +++ b/src/fides/data/sample_project/sample_connections/sample_connections.yml @@ -11,6 +11,19 @@ connection: dbname: $FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME username: $FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME password: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD + - key: cookie_house_loyalty_database + name: Postgres Connector (Loyalty) + connection_type: postgres + access: write + dataset: postgres_example_test_extended_dataset + system_key: cookie_house_loyalty_database + secrets: + host: $FIDES_DEPLOY__CONNECTORS__POSTGRES__HOST + port: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT + dbname: $FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME + username: $FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME + password: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD + disabled: True - key: cookie_house_customer_database_mongodb name: MongoDB Connector connection_type: mongodb diff --git a/src/fides/data/sample_project/sample_resources/postgres_example_test_extended_dataset.yml b/src/fides/data/sample_project/sample_resources/postgres_example_test_extended_dataset.yml new file mode 100644 index 0000000000..9a396bd2a4 --- /dev/null +++ b/src/fides/data/sample_project/sample_resources/postgres_example_test_extended_dataset.yml @@ -0,0 +1,23 @@ +dataset: + - fides_key: postgres_example_test_extended_dataset + name: Postgres Example Test Extended Dataset + description: Contains a reference to a collection that can only be reached by the custom `loyalty_id` identity (for testing purposes) + collections: + - name: loyalty + fields: + - name: id + data_categories: [user.unique_id] + fides_meta: + identity: loyalty_id + - name: name + data_categories: [user.name] + fides_meta: + data_type: string + - name: points + data_categories: [user.content] + fides_meta: + data_type: integer + - name: tier + data_categories: [user.content] + fides_meta: + data_type: string diff --git a/src/fides/data/sample_project/sample_resources/sample_systems.yml b/src/fides/data/sample_project/sample_resources/sample_systems.yml index d1d5c380f3..3a95f23f6b 100644 --- a/src/fides/data/sample_project/sample_resources/sample_systems.yml +++ b/src/fides/data/sample_project/sample_resources/sample_systems.yml @@ -12,6 +12,23 @@ system: data_subjects: - customer + - fides_key: cookie_house_loyalty_database + name: Cookie House Loyalty Program + description: Secondary database for Cookie House's loyalty program. + system_type: Database + administrating_department: Engineering + egress: + - fides_key: cookie_house + type: system + privacy_declarations: + - data_categories: + - user + data_use: essential.service + data_subjects: + - customer + dataset_references: + - postgres_example_test_extended_dataset + - fides_key: cookie_house_postgresql_database name: Cookie House PostgreSQL Database description: Primary database for Cookie House orders. diff --git a/tests/ctl/api/test_seed.py b/tests/ctl/api/test_seed.py index 25e2bb2b0e..0fc1080fb6 100644 --- a/tests/ctl/api/test_seed.py +++ b/tests/ctl/api/test_seed.py @@ -488,21 +488,23 @@ async def test_load_samples( dataset_configs = ( (await async_session.execute(select(DatasetConfig))).scalars().all() ) - assert len(systems) == 4 - assert len(datasets) == 3 + assert len(systems) == 5 + assert len(datasets) == 4 assert len(policies) == 1 - assert len(connections) == 2 - assert len(dataset_configs) == 2 + assert len(connections) == 3 + assert len(dataset_configs) == 3 assert sorted([e.fides_key for e in systems]) == [ "cookie_house", "cookie_house_customer_database", + "cookie_house_loyalty_database", "cookie_house_marketing_system", "cookie_house_postgresql_database", ] assert sorted([e.fides_key for e in datasets]) == [ "mongo_test", "postgres_example_test_dataset", + "postgres_example_test_extended_dataset", "stripe_connector", ] assert sorted([e.fides_key for e in policies]) == ["sample_policy"] @@ -511,11 +513,13 @@ async def test_load_samples( # expected to exist; the others defined in the sample_connections.yml # will be ignored since they are missing secrets! assert sorted([e.key for e in connections]) == [ + "cookie_house_loyalty_database", "cookie_house_postgresql_database", "stripe_connector", ] assert sorted([e.fides_key for e in dataset_configs]) == [ "postgres_example_test_dataset", + "postgres_example_test_extended_dataset", "stripe_connector", ] @@ -584,8 +588,9 @@ async def test_load_sample_connections(self): assert False, error_message # Assert that only the connections with all their secrets are returned - assert len(connections) == 2 + assert len(connections) == 3 assert sorted([e.key for e in connections]) == [ + "cookie_house_loyalty_database", "cookie_house_postgresql_database", "stripe_connector", ] diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 027446ad58..f07db250c2 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -1920,6 +1920,7 @@ def example_datasets() -> List[Dict]: "data/dataset/email_dataset.yml", "data/dataset/remote_fides_example_test_dataset.yml", "data/dataset/dynamodb_example_test_dataset.yml", + "data/dataset/postgres_example_test_extended_dataset.yml", ] for filename in example_filenames: example_datasets += load_dataset(filename) @@ -2123,6 +2124,23 @@ def empty_provided_identity(db): provided_identity.delete(db) +@pytest.fixture(scope="function") +def custom_provided_identity(db): + provided_identity_data = { + "privacy_request_id": None, + "field_name": "customer_id", + "field_label": "Customer ID", + "hashed_value": ProvidedIdentity.hash_value("123"), + "encrypted_value": {"value": "123"}, + } + provided_identity = ProvidedIdentity.create( + db, + data=provided_identity_data, + ) + yield provided_identity + provided_identity.delete(db=db) + + @pytest.fixture(scope="function") def provided_identity_value(): return "test@email.com" diff --git a/tests/fixtures/postgres_fixtures.py b/tests/fixtures/postgres_fixtures.py index cee178e49b..9a793b77d1 100644 --- a/tests/fixtures/postgres_fixtures.py +++ b/tests/fixtures/postgres_fixtures.py @@ -60,6 +60,33 @@ def postgres_example_test_dataset_config( ctl_dataset.delete(db=db) +@pytest.fixture +def postgres_example_test_extended_dataset_config( + connection_config: ConnectionConfig, + db: Session, + example_datasets: List[Dict], +) -> Generator: + postgres_dataset = example_datasets[12] + fides_key = postgres_dataset["fides_key"] + connection_config.name = fides_key + connection_config.key = fides_key + connection_config.save(db=db) + + ctl_dataset = CtlDataset.create_from_dataset_dict(db, postgres_dataset) + + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": connection_config.id, + "fides_key": fides_key, + "ctl_dataset_id": ctl_dataset.id, + }, + ) + yield dataset + dataset.delete(db=db) + ctl_dataset.delete(db=db) + + @pytest.fixture def postgres_example_test_dataset_config_read_access( read_connection_config: ConnectionConfig, diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py index ae761e127c..e7f658d896 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py @@ -1003,7 +1003,7 @@ def test_patch_datasets_bulk_create( ) assert response.status_code == 200 response_body = json.loads(response.text) - assert len(response_body["succeeded"]) == 12 + assert len(response_body["succeeded"]) == 13 assert len(response_body["failed"]) == 0 # Confirm that postgres dataset matches the values we provided @@ -1146,7 +1146,7 @@ def test_patch_datasets_bulk_update( assert response.status_code == 200 response_body = json.loads(response.text) - assert len(response_body["succeeded"]) == 12 + assert len(response_body["succeeded"]) == 13 assert len(response_body["failed"]) == 0 # test postgres @@ -1395,7 +1395,7 @@ def test_patch_datasets_failed_response( assert response.status_code == 200 # Returns 200 regardless response_body = json.loads(response.text) assert len(response_body["succeeded"]) == 0 - assert len(response_body["failed"]) == 12 + assert len(response_body["failed"]) == 13 for failed_response in response_body["failed"]: assert "Dataset create/update failed" in failed_response["message"] diff --git a/tests/ops/api/v1/endpoints/test_drp_endpoints.py b/tests/ops/api/v1/endpoints/test_drp_endpoints.py index 322e68e3cd..d6544ffcf9 100644 --- a/tests/ops/api/v1/endpoints/test_drp_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_drp_endpoints.py @@ -94,11 +94,8 @@ def test_create_drp_privacy_request( identity_attribute="identity", ) assert cache.get(identity_key) == encoded_identity - fidesops_identity_key = get_identity_cache_key( - privacy_request_id=pr.id, - identity_attribute="email", - ) - assert cache.get(fidesops_identity_key) == identity["email"] + assert pr.get_cached_identity_data()["email"] == identity["email"] + persisted_identity = pr.get_persisted_identity() assert persisted_identity.email == TEST_EMAIL assert persisted_identity.phone_number == TEST_PHONE_NUMBER @@ -157,16 +154,9 @@ def test_create_drp_privacy_request_unsupported_identity_props( identity_attribute="identity", ) assert cache.get(identity_key) == encoded_identity - fidesops_identity_key = get_identity_cache_key( - privacy_request_id=pr.id, - identity_attribute="email", - ) - assert cache.get(fidesops_identity_key) == identity["email"] - fidesops_identity_key_address = get_identity_cache_key( - privacy_request_id=pr.id, - identity_attribute="address", - ) - assert cache.get(fidesops_identity_key_address) is None + assert pr.get_cached_identity_data()["email"] == identity["email"] + assert "address" not in pr.get_cached_identity_data().keys() + pr.delete(db=db) assert run_access_request_mock.called @@ -337,11 +327,8 @@ def test_create_drp_privacy_request_error_notification( identity_attribute="identity", ) assert cache.get(identity_key) == encoded_identity - fidesops_identity_key = get_identity_cache_key( - privacy_request_id=pr.id, - identity_attribute="email", - ) - assert cache.get(fidesops_identity_key) == identity["email"] + assert pr.get_cached_identity_data()["email"] == identity["email"] + persisted_identity = pr.get_persisted_identity() assert persisted_identity.email == TEST_EMAIL assert persisted_identity.phone_number == TEST_PHONE_NUMBER diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 159e43576a..09d724857a 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -53,7 +53,7 @@ SubjectIdentityVerificationBodyParams, ) from fides.api.schemas.policy import ActionType, PolicyResponse -from fides.api.schemas.redis_cache import Identity +from fides.api.schemas.redis_cache import Identity, LabeledIdentity from fides.api.task import graph_task from fides.api.tasks import MESSAGING_QUEUE_NAME from fides.api.util.cache import ( @@ -174,6 +174,43 @@ def test_create_privacy_request_stores_identities( pr.delete(db=db) assert run_access_request_mock.called + @mock.patch( + "fides.api.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_create_privacy_request_stores_custom_identities( + self, + run_access_request_mock, + url, + db, + api_client: TestClient, + policy, + ): + TEST_EMAIL = "test@example.com" + TEST_PHONE_NUMBER = "+12345678910" + data = [ + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": { + "email": TEST_EMAIL, + "phone_number": TEST_PHONE_NUMBER, + "loyalty_id": {"label": "Loyalty ID", "value": "CH-1"}, + }, + } + ] + resp = api_client.post(url, json=data) + assert resp.status_code == 200 + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) + assert pr.get_persisted_identity() == Identity( + email=TEST_EMAIL, + phone_number=TEST_PHONE_NUMBER, + loyalty_id=LabeledIdentity(label="Loyalty ID", value="CH-1"), + ) + pr.delete(db=db) + assert run_access_request_mock.called + @mock.patch( "fides.api.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @@ -443,7 +480,6 @@ def test_create_privacy_request_caches_identity( db, api_client: TestClient, policy, - cache, ): identity = {"email": "test@example.com"} data = [ @@ -458,11 +494,7 @@ def test_create_privacy_request_caches_identity( response_data = resp.json()["succeeded"] assert len(response_data) == 1 pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) - key = get_identity_cache_key( - privacy_request_id=pr.id, - identity_attribute=list(identity.keys())[0], - ) - assert cache.get(key) == list(identity.values())[0] + assert pr.get_cached_identity_data() == identity pr.delete(db=db) assert run_access_request_mock.called @@ -925,9 +957,10 @@ def test_get_privacy_requests_with_identity( resp = response.json() assert len(resp["items"]) == 1 assert resp["items"][0]["id"] == succeeded_privacy_request.id - assert ( - resp["items"][0]["identity"] - == succeeded_privacy_request.get_persisted_identity() + assert resp["items"][0][ + "identity" + ] == succeeded_privacy_request.get_persisted_identity().labeled_dict( + include_default_labels=True ) assert resp["items"][0]["policy"]["key"] == privacy_request.policy.key @@ -4865,11 +4898,7 @@ def test_create_privacy_request_caches_identity( response_data = resp.json()["succeeded"] assert len(response_data) == 1 pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) - key = get_identity_cache_key( - privacy_request_id=pr.id, - identity_attribute=list(identity.keys())[0], - ) - assert cache.get(key) == list(identity.values())[0] + assert pr.get_cached_identity_data() == identity assert run_access_request_mock.called @mock.patch( diff --git a/tests/ops/integration_tests/saas/test_square_task.py b/tests/ops/integration_tests/saas/test_square_task.py index 0f146c625a..a9e73a9b6c 100644 --- a/tests/ops/integration_tests/saas/test_square_task.py +++ b/tests/ops/integration_tests/saas/test_square_task.py @@ -161,6 +161,71 @@ async def test_square_access_request_task_by_phone_number( @pytest.mark.integration_saas @pytest.mark.asyncio +async def test_square_access_request_task_with_multiple_identities( + db, + policy, + square_connection_config, + square_dataset_config, + square_identity_email, + square_identity_phone_number, +) -> None: + """Full access request based on the Square SaaS config""" + + privacy_request = PrivacyRequest( + id=f"test_square_access_request_task_{random.randint(0, 1000)}" + ) + identity = Identity( + **{"email": square_identity_email, "phone_number": square_identity_phone_number} + ) + privacy_request.cache_identity(identity) + + dataset_name = square_connection_config.get_saas_config().fides_key + merged_graph = square_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + + v = await graph_task.run_access_request( + privacy_request, + policy, + graph, + [square_connection_config], + {"email": square_identity_email, "phone_number": square_identity_phone_number}, + db, + ) + + assert_rows_match( + v[f"{dataset_name}:customer"], + min_size=1, + keys=[ + "id", + "created_at", + "updated_at", + "given_name", + "family_name", + "nickname", + "email_address", + "address", + "phone_number", + "company_name", + "preferences", + "creation_source", + "birthday", + "segment_ids", + "version", + ], + ) + + # verify we only returned data for our identity phone number + for customer in v[f"{dataset_name}:customer"]: + assert customer["phone_number"] == square_identity_phone_number + assert customer["email_address"] == square_identity_email + + # verify orders aren't duplicated since we are looking up the customer by two different identities + assert len(v[f"{dataset_name}:orders"]) == 2 + + +@pytest.mark.integration_saas +@pytest.mark.integration_square +@pytest.mark.asyncio async def test_square_erasure_request_task( db, policy, diff --git a/tests/ops/models/test_privacy_request.py b/tests/ops/models/test_privacy_request.py index c4737d3be7..9ce9747ce7 100644 --- a/tests/ops/models/test_privacy_request.py +++ b/tests/ops/models/test_privacy_request.py @@ -27,7 +27,7 @@ can_run_checkpoint, ) from fides.api.schemas.privacy_request import CustomPrivacyRequestField -from fides.api.schemas.redis_cache import Identity +from fides.api.schemas.redis_cache import Identity, LabeledIdentity from fides.api.service.connectors.manual_connector import ManualAction from fides.api.util.cache import FidesopsRedis, get_identity_cache_key from fides.api.util.constants import API_DATE_FORMAT @@ -51,6 +51,16 @@ def test_blank_provided_identity_to_identity( assert identity.email is None +def test_custom_provided_identity_to_identity( + custom_provided_identity: ProvidedIdentity, +) -> None: + identity = custom_provided_identity.as_identity_schema() + assert identity.customer_id == LabeledIdentity( + label=custom_provided_identity.field_label, + value=custom_provided_identity.encrypted_value.get("value"), + ) + + def test_privacy_request( db: Session, policy: Policy, @@ -246,7 +256,9 @@ def test_delete_privacy_request_removes_cached_data( privacy_request_id=privacy_request.id, identity_attribute=identity_attribute, ) - assert cache.get(key) == identity_value + assert ( + privacy_request.get_cached_identity_data()[identity_attribute] == identity_value + ) privacy_request.delete(db) from_db = PrivacyRequest.get(db=db, object_id=privacy_request.id) assert from_db is None @@ -1186,3 +1198,33 @@ def test_persist_custom_privacy_request_fields_collection_disabled( }, ) assert consent_request.get_persisted_custom_privacy_request_fields() == {} + + +class TestPrivacyRequestCustomIdentities: + def test_cache_custom_identities(self, privacy_request): + privacy_request.cache_identity( + identity={ + "customer_id": LabeledIdentity(label="Custom ID", value=123), + "account_id": LabeledIdentity(label="Account ID", value="456"), + }, + ) + assert privacy_request.get_cached_identity_data() == { + "email": "test@example.com", + "customer_id": {"label": "Custom ID", "value": 123}, + "account_id": {"label": "Account ID", "value": "456"}, + } + + def test_persist_custom_identities(self, db, privacy_request): + privacy_request.persist_identity( + db=db, + identity={ + "customer_id": LabeledIdentity(label="Custom ID", value=123), + "account_id": LabeledIdentity(label="Account ID", value="456"), + }, + ) + assert privacy_request.get_persisted_identity() == Identity( + email="test@example.com", + phone_number="+12345678910", + customer_id=LabeledIdentity(label="Custom ID", value=123), + account_id=LabeledIdentity(label="Account ID", value="456"), + ) diff --git a/tests/ops/schemas/test_identity_schema.py b/tests/ops/schemas/test_identity_schema.py new file mode 100644 index 0000000000..8631656b3b --- /dev/null +++ b/tests/ops/schemas/test_identity_schema.py @@ -0,0 +1,245 @@ +import pytest +from pydantic import ValidationError + +from fides.api.schemas.redis_cache import Identity + + +class TestIdentitySchema: + def test_email_identity(self): + Identity(email="user@example.com") + + def test_valid_custom_identity(self): + Identity(customer_id={"label": "Customer ID", "value": "123"}) + + def test_mixed_identities(self): + Identity( + email="user@example.com", + customer_id={"label": "Customer ID", "value": "123"}, + ) + + def test_multiple_custom_identities(self): + Identity( + customer_id={"label": "Customer ID", "value": "123"}, + account_id={"label": "Account ID", "value": 123}, + ) + + def test_none_custom_identity(self): + with pytest.raises(ValidationError) as exc: + Identity( + customer_id={"label": "Customer ID", "value": None}, + ) + assert "none is not an allowed value" in str(exc.value) + + def test_invalid_custom_identity(self): + with pytest.raises(ValueError) as exc: + Identity(customer_id="123") + assert ( + str(exc.value) + == 'Custom identity "customer_id" must be an instance of LabeledIdentity (e.g. {"label": "Field label", "value": "123"})' + ) + + @pytest.mark.parametrize( + "identity_data, expected_dict", + [ + ( + { + "email": "user@example.com", + }, + { + "phone_number": None, + "email": "user@example.com", + "ga_client_id": None, + "ljt_readerID": None, + "fides_user_device_id": None, + }, + ), + ( + { + "customer_id": {"label": "Customer ID", "value": "123"}, + }, + { + "phone_number": None, + "email": None, + "ga_client_id": None, + "ljt_readerID": None, + "fides_user_device_id": None, + "customer_id": "123", + }, + ), + ( + { + "email": "user@example.com", + "customer_id": {"label": "Customer ID", "value": "123"}, + }, + { + "phone_number": None, + "email": "user@example.com", + "ga_client_id": None, + "ljt_readerID": None, + "fides_user_device_id": None, + "customer_id": "123", + }, + ), + ( + { + "email": "user@example.com", + "phone_number": "+15558675309", + "ga_client_id": "GA123", + "ljt_readerID": "LJT456", + "fides_user_device_id": "4fbb6edf-34f6-4717-a6f1-541fd1e5d585", + }, + { + "phone_number": "+15558675309", + "email": "user@example.com", + "ga_client_id": "GA123", + "ljt_readerID": "LJT456", + "fides_user_device_id": "4fbb6edf-34f6-4717-a6f1-541fd1e5d585", + }, + ), + ], + ) + def test_identity_dict(self, identity_data, expected_dict): + identity = Identity(**identity_data) + assert identity.dict() == expected_dict + + @pytest.mark.parametrize( + "identity_data, expected_dict", + [ + ( + { + "email": "user@example.com", + }, + { + "phone_number": None, + "email": "user@example.com", + "ga_client_id": None, + "ljt_readerID": None, + "fides_user_device_id": None, + }, + ), + ( + { + "customer_id": {"label": "Customer ID", "value": "123"}, + }, + { + "phone_number": None, + "email": None, + "ga_client_id": None, + "ljt_readerID": None, + "fides_user_device_id": None, + "customer_id": {"label": "Customer ID", "value": "123"}, + }, + ), + ( + { + "email": "user@example.com", + "customer_id": {"label": "Customer ID", "value": "123"}, + }, + { + "phone_number": None, + "email": "user@example.com", + "ga_client_id": None, + "ljt_readerID": None, + "fides_user_device_id": None, + "customer_id": {"label": "Customer ID", "value": "123"}, + }, + ), + ( + { + "email": "user@example.com", + "phone_number": "+15558675309", + "ga_client_id": "GA123", + "ljt_readerID": "LJT456", + "fides_user_device_id": "4fbb6edf-34f6-4717-a6f1-541fd1e5d585", + }, + { + "phone_number": "+15558675309", + "email": "user@example.com", + "ga_client_id": "GA123", + "ljt_readerID": "LJT456", + "fides_user_device_id": "4fbb6edf-34f6-4717-a6f1-541fd1e5d585", + }, + ), + ], + ) + def test_identity_labeled_dict(self, identity_data, expected_dict): + identity = Identity(**identity_data) + assert identity.labeled_dict() == expected_dict + + @pytest.mark.parametrize( + "identity_data, expected_dict", + [ + ( + { + "email": "user@example.com", + }, + { + "phone_number": {"label": "Phone number", "value": None}, + "email": {"label": "Email", "value": "user@example.com"}, + "ga_client_id": {"label": "GA client ID", "value": None}, + "ljt_readerID": {"label": "LJT reader ID", "value": None}, + "fides_user_device_id": { + "label": "Fides user device ID", + "value": None, + }, + }, + ), + ( + { + "customer_id": {"label": "Customer ID", "value": "123"}, + }, + { + "phone_number": {"label": "Phone number", "value": None}, + "email": {"label": "Email", "value": None}, + "ga_client_id": {"label": "GA client ID", "value": None}, + "ljt_readerID": {"label": "LJT reader ID", "value": None}, + "fides_user_device_id": { + "label": "Fides user device ID", + "value": None, + }, + "customer_id": {"label": "Customer ID", "value": "123"}, + }, + ), + ( + { + "email": "user@example.com", + "customer_id": {"label": "Customer ID", "value": "123"}, + }, + { + "phone_number": {"label": "Phone number", "value": None}, + "email": {"label": "Email", "value": "user@example.com"}, + "ga_client_id": {"label": "GA client ID", "value": None}, + "ljt_readerID": {"label": "LJT reader ID", "value": None}, + "fides_user_device_id": { + "label": "Fides user device ID", + "value": None, + }, + "customer_id": {"label": "Customer ID", "value": "123"}, + }, + ), + ( + { + "email": "user@example.com", + "phone_number": "+15558675309", + "ga_client_id": "GA123", + "ljt_readerID": "LJT456", + "fides_user_device_id": "4fbb6edf-34f6-4717-a6f1-541fd1e5d585", + }, + { + "phone_number": {"label": "Phone number", "value": "+15558675309"}, + "email": {"label": "Email", "value": "user@example.com"}, + "ga_client_id": {"label": "GA client ID", "value": "GA123"}, + "ljt_readerID": {"label": "LJT reader ID", "value": "LJT456"}, + "fides_user_device_id": { + "label": "Fides user device ID", + "value": "4fbb6edf-34f6-4717-a6f1-541fd1e5d585", + }, + }, + ), + ], + ) + def test_identity_labeled_dict_include_default_labels( + self, identity_data, expected_dict + ): + identity = Identity(**identity_data) + assert identity.labeled_dict(include_default_labels=True) == expected_dict diff --git a/tests/ops/service/connectors/test_saas_queryconfig.py b/tests/ops/service/connectors/test_saas_queryconfig.py index 692792a261..041ba42729 100644 --- a/tests/ops/service/connectors/test_saas_queryconfig.py +++ b/tests/ops/service/connectors/test_saas_queryconfig.py @@ -355,51 +355,6 @@ def test_generate_update_stmt_with_url_encoded_body( assert prepared_request.query_params == {} assert prepared_request.body == "name%5Bfirst%5D=MASKED&name%5Blast%5D=MASKED" - @mock.patch( - "fides.api.models.privacy_request.PrivacyRequest.get_cached_identity_data" - ) - def test_get_read_requests_by_identity( - self, - mock_identity_data: Mock, - combined_traversal, - saas_example_connection_config, - ): - mock_identity_data.return_value = {"email": "test@example.com"} - - saas_config: Optional[ - SaaSConfig - ] = saas_example_connection_config.get_saas_config() - endpoints = saas_config.top_level_endpoint_dict - - member = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "member") - ] - tickets = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "tickets") - ] - - query_config = SaaSQueryConfig( - member, endpoints, {}, privacy_request=PrivacyRequest(id="123") - ) - saas_requests = query_config.get_read_requests_by_identity() - assert len(saas_requests) == 1 - assert saas_requests[0].param_values[0].identity == "email" - - mock_identity_data.return_value = {"phone": "+951555555"} - - query_config = SaaSQueryConfig( - member, endpoints, {}, privacy_request=PrivacyRequest(id="123") - ) - saas_requests = query_config.get_read_requests_by_identity() - assert len(saas_requests) == 1 - assert saas_requests[0].param_values[0].identity == "phone" - - query_config = SaaSQueryConfig( - tickets, endpoints, {}, privacy_request=PrivacyRequest(id="123") - ) - saas_requests = query_config.get_read_requests_by_identity() - assert len(saas_requests) == 2 - def test_get_masking_request( self, combined_traversal, saas_example_connection_config ): diff --git a/tests/ops/service/privacy_request/test_request_runner_service.py b/tests/ops/service/privacy_request/test_request_runner_service.py index 9ba0888538..a3ee24ccce 100644 --- a/tests/ops/service/privacy_request/test_request_runner_service.py +++ b/tests/ops/service/privacy_request/test_request_runner_service.py @@ -515,6 +515,86 @@ def test_create_and_process_access_request_postgres( assert ExecutionLog.get(db, object_id=log_id).privacy_request_id == pr_id +@pytest.mark.integration_postgres +@pytest.mark.integration +@mock.patch("fides.api.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +def test_create_and_process_access_request_with_custom_identities_postgres( + trigger_webhook_mock, + postgres_example_test_dataset_config_read_access, + postgres_example_test_extended_dataset_config, + postgres_integration_db, + db, + cache, + policy, + policy_pre_execution_webhooks, + policy_post_execution_webhooks, + run_privacy_request_task, +): + customer_email = "customer-1@example.com" + loyalty_id = "CH-1" + data = { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": { + "email": customer_email, + "loyalty_id": {"label": "Loyalty ID", "value": loyalty_id}, + }, + } + + pr = get_privacy_request_results( + db, + policy, + run_privacy_request_task, + data, + ) + + results = pr.get_results() + assert len(results.keys()) == 12 + + for key in results.keys(): + assert results[key] is not None + assert results[key] != {} + + result_key_prefix = f"EN_{pr.id}__access_request__postgres_example_test_dataset:" + customer_key = result_key_prefix + "customer" + assert results[customer_key][0]["email"] == customer_email + + visit_key = result_key_prefix + "visit" + assert results[visit_key][0]["email"] == customer_email + + loyalty_key = ( + f"EN_{pr.id}__access_request__postgres_example_test_extended_dataset:loyalty" + ) + assert results[loyalty_key][0]["id"] == loyalty_id + + log_id = pr.execution_logs[0].id + pr_id = pr.id + + finished_audit_log: AuditLog = AuditLog.filter( + db=db, + conditions=( + (AuditLog.privacy_request_id == pr_id) + & (AuditLog.action == AuditLogAction.finished) + ), + ).first() + + assert finished_audit_log is not None + + # Both pre-execution webhooks and both post-execution webhooks were called + assert trigger_webhook_mock.call_count == 4 + + for webhook in policy_pre_execution_webhooks: + webhook.delete(db=db) + + for webhook in policy_post_execution_webhooks: + webhook.delete(db=db) + + policy.delete(db=db) + pr.delete(db=db) + assert not pr in db # Check that `pr` has been expunged from the session + assert ExecutionLog.get(db, object_id=log_id).privacy_request_id == pr_id + + @pytest.mark.integration_postgres @pytest.mark.integration @pytest.mark.usefixtures( @@ -2041,7 +2121,7 @@ def test_email_complete_send_access_no_email_identity( data = { "requested_at": "2021-08-30T16:09:37.359Z", "policy_key": policy.key, - "identity": {"phone_number": "1231231233"}, + "identity": {"phone_number": "+1231231233"}, } pr = get_privacy_request_results( diff --git a/tests/ops/task/test_graph_task.py b/tests/ops/task/test_graph_task.py index 0aada504de..adebf83aae 100644 --- a/tests/ops/task/test_graph_task.py +++ b/tests/ops/task/test_graph_task.py @@ -36,7 +36,6 @@ build_affected_field_logs, collect_queries, filter_by_enabled_actions, - start_function, update_erasure_mapping_from_cache, ) from fides.api.task.task_resources import Connections @@ -187,17 +186,18 @@ def test_pre_process_input_flights_collection( truncated_customer_details_output = [ { "_id": ObjectId("61f422e0ddc2559e0c300e95"), - "travel_identifiers": ["A111-11111", "B111-11111"], + "travel_identifiers": ["A111-11111", "B111-11111", "D111-11111"], }, { "_id": ObjectId("61f422e0ddc2559e0c300e95"), - "travel_identifiers": ["C111-11111"], + "travel_identifiers": ["C111-11111", "D111-11111"], }, ] assert task.pre_process_input_data(truncated_customer_details_output) == { "passenger_information.passenger_ids": [ "A111-11111", "B111-11111", + "D111-11111", "C111-11111", ], "fidesops_grouped_inputs": [], @@ -312,7 +312,7 @@ def test_pre_process_input_data_group_dependent_fields(self, db): assert task.pre_process_input_data(identity_output, project_output) == { "email": ["email@gmail.com"], "project": ["abcde", "fghij", "klmno"], - "organization": ["12345", "54321", "54321"], + "organization": ["12345", "54321"], "fidesops_grouped_inputs": [], } diff --git a/tests/ops/util/test_collection_util.py b/tests/ops/util/test_collection_util.py index f377f711e7..678552eb24 100644 --- a/tests/ops/util/test_collection_util.py +++ b/tests/ops/util/test_collection_util.py @@ -1,8 +1,13 @@ from typing import Dict, List +from immutables import Map +from ordered_set import OrderedSet + from fides.api.util.collection_util import ( append, filter_nonempty_values, + make_immutable, + make_mutable, merge_dicts, partition, ) @@ -47,3 +52,49 @@ def test_filter_nonempty_values() -> None: assert filter_nonempty_values({"B": None}) == {} assert filter_nonempty_values({}) == {} assert filter_nonempty_values(None) == {} + + +class TestMutabilityConversion: + def test_make_immutable_with_dict(self): + mutable_dict = {"a": 1, "b": {"c": 2}} + immutable_obj = make_immutable(mutable_dict) + assert isinstance(immutable_obj, Map) + assert immutable_obj == Map({"a": 1, "b": Map({"c": 2})}) + assert hash(immutable_obj) is not None + + def test_make_immutable_with_list(self): + mutable_list = [1, [2, 3], {"a": 4}] + immutable_obj = make_immutable(mutable_list) + assert isinstance(immutable_obj, tuple) + assert immutable_obj == (1, (2, 3), Map({"a": 4})) + assert hash(immutable_obj) is not None + + def test_make_immutable_with_other_types(self): + int_obj = 42 + str_obj = "hello" + assert make_immutable(int_obj) == 42 + assert make_immutable(str_obj) == "hello" + + def test_make_mutable_with_immutable_map(self): + immutable_map = Map({"a": 1, "b": Map({"c": 2})}) + mutable_obj = make_mutable(immutable_map) + assert isinstance(mutable_obj, dict) + assert mutable_obj == {"a": 1, "b": {"c": 2}} + + def test_make_mutable_with_tuple(self): + immutable_tuple = (1, (2, 3), Map({"a": 4})) + mutable_obj = make_mutable(immutable_tuple) + assert isinstance(mutable_obj, list) + assert mutable_obj == [1, [2, 3], {"a": 4}] + + def test_make_mutable_with_ordered_set(self): + ordered_set = OrderedSet([1, 2, 3]) + mutable_obj = make_mutable(ordered_set) + assert isinstance(mutable_obj, list) + assert mutable_obj == [1, 2, 3] + + def test_make_mutable_with_other_types(self): + int_obj = 42 + str_obj = "hello" + assert make_mutable(int_obj) == 42 + assert make_mutable(str_obj) == "hello"