diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index a498fa8003..754429c33c 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -832,6 +832,8 @@ dataset: data_categories: [system.operations] - name: property_id data_categories: [system.operations] + - name: source + data_categories: [system.operations] - name: custom_connector_template description: 'A table used to hold custom connector templates which include a SaaS config, dataset, and an optional icon and functions' data_categories: [] @@ -1293,6 +1295,10 @@ dataset: data_categories: [ system.operations ] - name: custom_privacy_request_fields_approved_by data_categories: [ system.operations ] + - name: source + data_categories: [ system.operations ] + - name: submitted_by + data_categories: [ system.operations ] - name: privacyrequesterror data_categories: [] fields: diff --git a/CHANGELOG.md b/CHANGELOG.md index 659a75b8c3..a84f73c7ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The types of changes are: ### Added - Added Gzip Middleware for responses [#5225](https://github.com/ethyca/fides/pull/5225) +- Adding source and submitted_by fields to privacy requests (Fidesplus) [#5206](https://github.com/ethyca/fides/pull/5206) ### Changed - Removed unused `username` parameter from the Delighted integration configuration [#5220](https://github.com/ethyca/fides/pull/5220) diff --git a/clients/admin-ui/cypress/fixtures/privacy-requests/list.json b/clients/admin-ui/cypress/fixtures/privacy-requests/list.json index ceca22cdd2..1ac27a97c0 100644 --- a/clients/admin-ui/cypress/fixtures/privacy-requests/list.json +++ b/clients/admin-ui/cypress/fixtures/privacy-requests/list.json @@ -50,7 +50,8 @@ }, "action_required_details": null, "resume_endpoint": null, - "days_left": 45 + "days_left": 45, + "source": "Privacy Center" }, { "id": "pri_6411a2ea-72d2-4111-aad3-9170ba5e5934", @@ -96,7 +97,8 @@ }, "action_required_details": null, "resume_endpoint": null, - "days_left": 45 + "days_left": 45, + "source": "Privacy Center" }, { "id": "pri_34650722-960c-4abd-b6a6-6dba4461dfbe", @@ -142,7 +144,8 @@ }, "action_required_details": null, "resume_endpoint": null, - "days_left": 45 + "days_left": 45, + "source": "Privacy Center" }, { "id": "pri_8750c782-3fad-4ae6-bbcf-219f70f537ee", @@ -188,7 +191,8 @@ }, "action_required_details": null, "resume_endpoint": null, - "days_left": 45 + "days_left": 45, + "source": "Privacy Center" }, { "id": "pri_8f719d4a-848d-42b9-8aaa-e7ac442ebba0", @@ -234,7 +238,8 @@ }, "action_required_details": null, "resume_endpoint": null, - "days_left": 45 + "days_left": 45, + "source": "Privacy Center" }, { "id": "pri_a0fe994d-f1ee-40d9-bbcb-dedb76a08efe", @@ -280,7 +285,8 @@ }, "action_required_details": null, "resume_endpoint": null, - "days_left": 45 + "days_left": 45, + "source": "Privacy Center" }, { "id": "pri_741784e9-1d75-4a6c-bdf7-66c9c814f1c1", @@ -326,7 +332,8 @@ }, "action_required_details": null, "resume_endpoint": null, - "days_left": 45 + "days_left": 45, + "source": "Privacy Center" }, { "id": "pri_4f87b8b0-f97e-45e7-8561-300a6a932d04", @@ -376,7 +383,8 @@ "action_needed": null }, "resume_endpoint": "/privacy-request/pri_4f87b8b0-f97e-45e7-8561-300a6a932d04/retry", - "days_left": 45 + "days_left": 45, + "source": "Privacy Center" } ], "total": 8, diff --git a/clients/admin-ui/src/features/privacy-requests/RequestDetails.tsx b/clients/admin-ui/src/features/privacy-requests/RequestDetails.tsx index 90e1a32922..abc055edc9 100644 --- a/clients/admin-ui/src/features/privacy-requests/RequestDetails.tsx +++ b/clients/admin-ui/src/features/privacy-requests/RequestDetails.tsx @@ -11,6 +11,7 @@ import { import ClipboardButton from "~/features/common/ClipboardButton"; import DaysLeftTag from "~/features/common/DaysLeftTag"; +import { useFeatures } from "~/features/common/features"; import RequestStatusBadge from "~/features/common/RequestStatusBadge"; import RequestType from "~/features/common/RequestType"; import { PrivacyRequestEntity } from "~/features/privacy-requests/types"; @@ -25,6 +26,7 @@ type RequestDetailsProps = { }; const RequestDetails = ({ subjectRequest }: RequestDetailsProps) => { + const { plus: hasPlus } = useFeatures(); const { id, status, policy } = subjectRequest; return ( @@ -55,7 +57,23 @@ const RequestDetails = ({ subjectRequest }: RequestDetailsProps) => { - + {hasPlus && subjectRequest.source && ( + + + Source: + + + + {subjectRequest.source} + + + + )} Request type: diff --git a/clients/admin-ui/src/features/privacy-requests/RequestTable.tsx b/clients/admin-ui/src/features/privacy-requests/RequestTable.tsx index 3a47998305..0ff4c3fc3f 100644 --- a/clients/admin-ui/src/features/privacy-requests/RequestTable.tsx +++ b/clients/admin-ui/src/features/privacy-requests/RequestTable.tsx @@ -20,6 +20,7 @@ import { useMemo, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; import { selectToken } from "~/features/auth"; +import { useFeatures } from "~/features/common/features"; import { DownloadLightIcon } from "~/features/common/Icon"; import { FidesTableV2, @@ -43,6 +44,7 @@ import { RequestTableFilterModal } from "~/features/privacy-requests/RequestTabl import { PrivacyRequestEntity } from "~/features/privacy-requests/types"; export const RequestTable = ({ ...props }: BoxProps): JSX.Element => { + const { plus: hasPlus } = useFeatures(); const [requestIdFilter, setRequestIdFilter] = useState(); const [revealPII, setRevealPII] = useState(false); const filters = useSelector(selectPrivacyRequestFilters); @@ -124,7 +126,10 @@ export const RequestTable = ({ ...props }: BoxProps): JSX.Element => { const tableInstance = useReactTable({ getCoreRowModel: getCoreRowModel(), data: requests, - columns: useMemo(() => getRequestTableColumns(revealPII), [revealPII]), + columns: useMemo( + () => getRequestTableColumns(revealPII, hasPlus), + [revealPII, hasPlus], + ), getRowId: (row) => `${row.status}-${row.id}`, manualPagination: true, }); diff --git a/clients/admin-ui/src/features/privacy-requests/RequestTableColumns.tsx b/clients/admin-ui/src/features/privacy-requests/RequestTableColumns.tsx index 500cc4bcb5..80945fde7f 100644 --- a/clients/admin-ui/src/features/privacy-requests/RequestTableColumns.tsx +++ b/clients/admin-ui/src/features/privacy-requests/RequestTableColumns.tsx @@ -1,6 +1,10 @@ import { createColumnHelper } from "@tanstack/react-table"; -import { DefaultCell, DefaultHeaderCell } from "~/features/common/table/v2"; +import { + BadgeCell, + DefaultCell, + DefaultHeaderCell, +} from "~/features/common/table/v2"; import { formatDate, getPII } from "~/features/common/utils"; import { RequestActionTypeCell, @@ -14,9 +18,10 @@ import { PrivacyRequestEntity } from "~/features/privacy-requests/types"; enum COLUMN_IDS { STATUS = "status", DAYS_LEFT = "due_date", + SOURCE = "source", REQUEST_TYPE = "request_type", SUBJECT_IDENTITY = "subject_identity", - TIME_RECIEVED = "created_at", + TIME_RECEIVED = "created_at", CREATED_BY = "created_by", REVIEWER = "reviewer", ID = "id", @@ -25,7 +30,7 @@ enum COLUMN_IDS { const columnHelper = createColumnHelper(); -export const getRequestTableColumns = (revealPII = false) => [ +export const getRequestTableColumns = (revealPII = false, hasPlus = false) => [ columnHelper.accessor((row) => row.status, { id: COLUMN_IDS.STATUS, cell: ({ getValue }) => , @@ -42,6 +47,21 @@ export const getRequestTableColumns = (revealPII = false) => [ ), header: (props) => , }), + ...(hasPlus + ? [ + columnHelper.accessor((row) => row.source, { + id: COLUMN_IDS.SOURCE, + cell: (props) => + props.getValue() ? ( + + ) : ( + + ), + header: (props) => , + enableSorting: false, + }), + ] + : []), columnHelper.accessor((row) => row.policy.rules, { id: COLUMN_IDS.REQUEST_TYPE, cell: ({ getValue }) => , @@ -63,7 +83,7 @@ export const getRequestTableColumns = (revealPII = false) => [ }, ), columnHelper.accessor((row) => row.created_at, { - id: COLUMN_IDS.TIME_RECIEVED, + id: COLUMN_IDS.TIME_RECEIVED, cell: ({ getValue }) => , header: (props) => , }), diff --git a/clients/admin-ui/src/features/privacy-requests/SubjectIdentities.tsx b/clients/admin-ui/src/features/privacy-requests/SubjectIdentities.tsx index 9e201538c8..8f1fee8ebe 100644 --- a/clients/admin-ui/src/features/privacy-requests/SubjectIdentities.tsx +++ b/clients/admin-ui/src/features/privacy-requests/SubjectIdentities.tsx @@ -20,7 +20,13 @@ const SubjectIdentities = ({ subjectRequest }: SubjectIdentitiesProps) => { return ( <> - + Subject identities diff --git a/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts b/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts index aca894cef9..f20fb6993a 100644 --- a/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts +++ b/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts @@ -12,6 +12,7 @@ import { PrivacyRequestStatus, SecurityApplicationConfig, } from "~/types/api"; +import { PrivacyRequestSource } from "~/types/api/models/PrivacyRequestSource"; import type { RootState } from "../../app/store"; import { BASE_URL } from "../../constants"; @@ -317,7 +318,10 @@ export const privacyRequestApi = baseApi.injectEndpoints({ query: (payload) => ({ url: `privacy-request/authenticated`, method: "POST", - body: payload, + body: payload.map((item) => ({ + ...item, + source: PrivacyRequestSource.REQUEST_MANAGER, + })), }), invalidatesTags: () => ["Request"], }), diff --git a/clients/admin-ui/src/features/privacy-requests/types.ts b/clients/admin-ui/src/features/privacy-requests/types.ts index 0ff6604461..48cea3c207 100644 --- a/clients/admin-ui/src/features/privacy-requests/types.ts +++ b/clients/admin-ui/src/features/privacy-requests/types.ts @@ -76,6 +76,7 @@ export interface PrivacyRequestEntity { reviewed_by: string; id: string; days_left?: number; + source?: string; } export interface PrivacyRequestResponse { diff --git a/clients/admin-ui/src/types/api/index.ts b/clients/admin-ui/src/types/api/index.ts index 5794c45649..2090976441 100644 --- a/clients/admin-ui/src/types/api/index.ts +++ b/clients/admin-ui/src/types/api/index.ts @@ -334,6 +334,7 @@ export type { PrivacyRequestOption } from "./models/PrivacyRequestOption"; export type { PrivacyRequestResponse } from "./models/PrivacyRequestResponse"; export type { PrivacyRequestResumeFormat } from "./models/PrivacyRequestResumeFormat"; export type { PrivacyRequestReviewer } from "./models/PrivacyRequestReviewer"; +export { PrivacyRequestSource } from "./models/PrivacyRequestSource"; export { PrivacyRequestStatus } from "./models/PrivacyRequestStatus"; export type { PrivacyRequestTaskSchema } from "./models/PrivacyRequestTaskSchema"; export type { PrivacyRequestVerboseResponse } from "./models/PrivacyRequestVerboseResponse"; diff --git a/clients/admin-ui/src/types/api/models/ConsentRequestCreateExtended.ts b/clients/admin-ui/src/types/api/models/ConsentRequestCreateExtended.ts index e41f217e8a..71e3a616b7 100644 --- a/clients/admin-ui/src/types/api/models/ConsentRequestCreateExtended.ts +++ b/clients/admin-ui/src/types/api/models/ConsentRequestCreateExtended.ts @@ -4,6 +4,7 @@ import type { fides__api__schemas__redis_cache__CustomPrivacyRequestField } from "./fides__api__schemas__redis_cache__CustomPrivacyRequestField"; import type { Identity } from "./Identity"; +import type { PrivacyRequestSource } from "./PrivacyRequestSource"; /** * An extension of the base fides model with the addition of plus-only fields @@ -15,4 +16,5 @@ export type ConsentRequestCreateExtended = { fides__api__schemas__redis_cache__CustomPrivacyRequestField > | null; property_id?: string | null; + source?: PrivacyRequestSource; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestCreate.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestCreate.ts index 65ff3e1794..30ab8bf4ed 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestCreate.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestCreate.ts @@ -5,6 +5,7 @@ import type { Consent } from "./Consent"; import type { fides__api__schemas__redis_cache__CustomPrivacyRequestField } from "./fides__api__schemas__redis_cache__CustomPrivacyRequestField"; import type { Identity } from "./Identity"; +import type { PrivacyRequestSource } from "./PrivacyRequestSource"; /** * Data required to create a PrivacyRequest diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestResponse.ts index f7da59430d..6dd2e41364 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestResponse.ts @@ -5,6 +5,7 @@ import type { CheckpointActionRequiredDetails } from "./CheckpointActionRequiredDetails"; import type { PolicyResponse } from "./PolicyResponse"; import type { PrivacyRequestReviewer } from "./PrivacyRequestReviewer"; +import type { PrivacyRequestSource } from "./PrivacyRequestSource"; import type { PrivacyRequestStatus } from "./PrivacyRequestStatus"; /** diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestSource.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestSource.ts new file mode 100644 index 0000000000..6d1e633ee1 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestSource.ts @@ -0,0 +1,13 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * The source where the privacy request originated from + */ +export enum PrivacyRequestSource { + PRIVACY_CENTER = "Privacy Center", + REQUEST_MANAGER = "Request Manager", + CONSENT_WEBHOOK = "Consent Webhook", + FIDES_JS = "Fides.js", +} diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestVerboseResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestVerboseResponse.ts index 7c0e59dd10..0d423b8f6b 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestVerboseResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestVerboseResponse.ts @@ -6,6 +6,7 @@ import type { CheckpointActionRequiredDetails } from "./CheckpointActionRequired import type { ExecutionAndAuditLogResponse } from "./ExecutionAndAuditLogResponse"; import type { PolicyResponse } from "./PolicyResponse"; import type { PrivacyRequestReviewer } from "./PrivacyRequestReviewer"; +import type { PrivacyRequestSource } from "./PrivacyRequestSource"; import type { PrivacyRequestStatus } from "./PrivacyRequestStatus"; /** diff --git a/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts b/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts index 0562a96aac..997569670d 100644 --- a/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts +++ b/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts @@ -47,6 +47,7 @@ describe("Smoke test", () => { }, policy_key: "default_access_policy", property_id: null, + source: "Privacy Center", }, ]); }); diff --git a/clients/fides-js/src/lib/consent-types.ts b/clients/fides-js/src/lib/consent-types.ts index 574edd8fc6..92bed9188f 100644 --- a/clients/fides-js/src/lib/consent-types.ts +++ b/clients/fides-js/src/lib/consent-types.ts @@ -749,6 +749,7 @@ export type PrivacyPreferencesRequest = { user_geography?: string; method?: ConsentMethod; served_notice_history_id?: string; + source?: string; }; export type ConsentOptionCreate = { diff --git a/clients/fides-js/src/services/api.ts b/clients/fides-js/src/services/api.ts index bffd74eb65..b15c549331 100644 --- a/clients/fides-js/src/services/api.ts +++ b/clients/fides-js/src/services/api.ts @@ -150,6 +150,9 @@ const PATCH_FETCH_OPTIONS: RequestInit = { }, }; +// See: PrivacyRequestSource enum in Fides +export const REQUEST_SOURCE = "Fides.js"; + /** * Sends user consent preference downstream to Fides or custom API */ @@ -183,7 +186,7 @@ export const patchUserPreference = async ( debugLog(options.debug, "Calling Fides save preferences API"); const fetchOptions: RequestInit = { ...PATCH_FETCH_OPTIONS, - body: JSON.stringify(preferences), + body: JSON.stringify({ ...preferences, source: REQUEST_SOURCE }), }; const response = await fetch( `${options.fidesApiUrl}${FidesEndpointPaths.PRIVACY_PREFERENCES}`, diff --git a/clients/privacy-center/components/modals/consent-request-modal/ConsentRequestForm.tsx b/clients/privacy-center/components/modals/consent-request-modal/ConsentRequestForm.tsx index a9bc4e1809..e91b689a06 100644 --- a/clients/privacy-center/components/modals/consent-request-modal/ConsentRequestForm.tsx +++ b/clients/privacy-center/components/modals/consent-request-modal/ConsentRequestForm.tsx @@ -29,6 +29,7 @@ import { PhoneInput } from "~/components/phone-input"; import { defaultIdentityInput } from "~/constants"; import { useConfig } from "~/features/common/config.slice"; import { useSettings } from "~/features/common/settings.slice"; +import { PrivacyRequestSource } from "~/types/api/models/PrivacyRequestSource"; type KnownKeys = { email: string; @@ -95,6 +96,7 @@ const useConsentRequestForm = ({ fides_user_device_id: cookie.identity.fides_user_device_id, }, custom_privacy_request_fields: transformedCustomPrivacyRequestFields, + source: PrivacyRequestSource.PRIVACY_CENTER, }; const handleError = ({ title, diff --git a/clients/privacy-center/components/modals/privacy-request-modal/PrivacyRequestForm.tsx b/clients/privacy-center/components/modals/privacy-request-modal/PrivacyRequestForm.tsx index b0f196bf35..8694bd56b5 100644 --- a/clients/privacy-center/components/modals/privacy-request-modal/PrivacyRequestForm.tsx +++ b/clients/privacy-center/components/modals/privacy-request-modal/PrivacyRequestForm.tsx @@ -32,6 +32,7 @@ import { useConfig } from "~/features/common/config.slice"; import { useProperty } from "~/features/common/property.slice"; import { useSettings } from "~/features/common/settings.slice"; import { PrivacyRequestStatus } from "~/types"; +import { PrivacyRequestSource } from "~/types/api/models/PrivacyRequestSource"; import { CustomIdentity, PrivacyRequestOption } from "~/types/config"; type FormValues = { @@ -157,6 +158,7 @@ const usePrivacyRequestForm = ({ }), policy_key: action.policy_key, property_id: property?.id || null, + source: PrivacyRequestSource.PRIVACY_CENTER, }, ]; diff --git a/clients/privacy-center/cypress/e2e/consent-banner.cy.ts b/clients/privacy-center/cypress/e2e/consent-banner.cy.ts index 97a22629df..85af75842f 100644 --- a/clients/privacy-center/cypress/e2e/consent-banner.cy.ts +++ b/clients/privacy-center/cypress/e2e/consent-banner.cy.ts @@ -7,6 +7,7 @@ import { FidesInitOptions, PrivacyNotice, RecordConsentServedRequest, + REQUEST_SOURCE, UserConsentPreference, } from "fides-js"; @@ -542,6 +543,7 @@ describe("Consent overlay", () => { method: ConsentMethod.SAVE, served_notice_history_id: body.served_notice_history_id, + source: REQUEST_SOURCE, }; expect(body).to.eql(expected); expect(body.served_notice_history_id).to.be.a("string"); @@ -643,6 +645,7 @@ describe("Consent overlay", () => { user_geography: "us_ca", method: ConsentMethod.SAVE, served_notice_history_id: body.served_notice_history_id, + source: REQUEST_SOURCE, }; expect(body).to.eql(expected); }); diff --git a/clients/privacy-center/types/api/models/PrivacyPreferencesRequest.ts b/clients/privacy-center/types/api/models/PrivacyPreferencesRequest.ts index aa94067aeb..14e062f776 100644 --- a/clients/privacy-center/types/api/models/PrivacyPreferencesRequest.ts +++ b/clients/privacy-center/types/api/models/PrivacyPreferencesRequest.ts @@ -2,6 +2,7 @@ /* tslint:disable */ /* eslint-disable */ +import type { PrivacyRequestSource } from "./PrivacyRequestSource"; import type { ConsentMethod } from "./ConsentMethod"; import type { ConsentOptionCreate } from "./ConsentOptionCreate"; import type { Identity } from "./Identity"; @@ -45,4 +46,5 @@ export type PrivacyPreferencesRequest = { method?: ConsentMethod; served_notice_history_id?: string; property_id?: string; + source?: PrivacyRequestSource; }; diff --git a/clients/privacy-center/types/api/models/PrivacyRequestSource.ts b/clients/privacy-center/types/api/models/PrivacyRequestSource.ts new file mode 100644 index 0000000000..6d1e633ee1 --- /dev/null +++ b/clients/privacy-center/types/api/models/PrivacyRequestSource.ts @@ -0,0 +1,13 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * The source where the privacy request originated from + */ +export enum PrivacyRequestSource { + PRIVACY_CENTER = "Privacy Center", + REQUEST_MANAGER = "Request Manager", + CONSENT_WEBHOOK = "Consent Webhook", + FIDES_JS = "Fides.js", +} diff --git a/src/fides/api/alembic/migrations/versions/896ea3803770_update_privacy_request_with_source_and_submitted_by.py b/src/fides/api/alembic/migrations/versions/896ea3803770_update_privacy_request_with_source_and_submitted_by.py new file mode 100644 index 0000000000..798432640f --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/896ea3803770_update_privacy_request_with_source_and_submitted_by.py @@ -0,0 +1,41 @@ +"""update privacy request with source and submitted_by + +Revision ID: 896ea3803770 +Revises: ffee79245c9a +Create Date: 2024-08-15 23:08:00.169034 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "896ea3803770" +down_revision = "ffee79245c9a" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column("consentrequest", sa.Column("source", sa.String(), nullable=True)) + op.add_column( + "privacyrequest", sa.Column("submitted_by", sa.String(), nullable=True) + ) + op.add_column("privacyrequest", sa.Column("source", sa.String(), nullable=True)) + op.create_foreign_key( + "privacyrequest_submitted_by_fkey", + "privacyrequest", + "fidesuser", + ["submitted_by"], + ["id"], + ondelete="SET NULL", + ) + + +def downgrade(): + op.drop_constraint( + "privacyrequest_submitted_by_fkey", "privacyrequest", type_="foreignkey" + ) + op.drop_column("privacyrequest", "source") + op.drop_column("privacyrequest", "submitted_by") + op.drop_column("consentrequest", "source") diff --git a/src/fides/api/api/v1/endpoints/consent_request_endpoints.py b/src/fides/api/api/v1/endpoints/consent_request_endpoints.py index 72186be3f4..2964da29d6 100644 --- a/src/fides/api/api/v1/endpoints/consent_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/consent_request_endpoints.py @@ -221,6 +221,7 @@ def create_consent_request( consent_request_data = { "provided_identity_id": provided_identity.id, "property_id": getattr(data, "property_id", None), + "source": getattr(data, "source", None), } consent_request = ConsentRequest.create(db, data=consent_request_data) @@ -427,6 +428,7 @@ def queue_privacy_request_to_propagate_consent_old_workflow( consent_preferences=executable_consent_preferences, consent_request_id=consent_request.id, custom_privacy_request_fields=consent_request.get_persisted_custom_privacy_request_fields(), + source=consent_request.source, ) ], authenticated=True, diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index 8deb516e31..97d0bf4c48 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -79,6 +79,7 @@ ExecutionLogStatus, PrivacyRequest, PrivacyRequestNotifications, + PrivacyRequestSource, PrivacyRequestStatus, ProvidedIdentity, ProvidedIdentityType, @@ -218,19 +219,22 @@ def create_privacy_request( or report failure and execute them within the Fidesops system. You cannot update privacy requests after they've been created. """ - return create_privacy_request_func(db, config_proxy, data, False) + return create_privacy_request_func(db, config_proxy, data, authenticated=False) @router.post( PRIVACY_REQUEST_AUTHENTICATED, status_code=HTTP_200_OK, - dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_CREATE])], response_model=BulkPostPrivacyRequests, ) def create_privacy_request_authenticated( *, db: Session = Depends(deps.get_db), config_proxy: ConfigProxy = Depends(deps.get_config_proxy), + client: ClientDetail = Security( + verify_oauth_client, + scopes=[PRIVACY_REQUEST_CREATE], + ), data: Annotated[List[PrivacyRequestCreate], Field(max_length=50)], # type: ignore ) -> BulkPostPrivacyRequests: """ @@ -239,7 +243,10 @@ def create_privacy_request_authenticated( You cannot update privacy requests after they've been created. This route requires authentication instead of using verification codes. """ - return create_privacy_request_func(db, config_proxy, data, True) + + return create_privacy_request_func( + db, config_proxy, data, authenticated=True, user_id=client.user_id + ) def _send_privacy_request_receipt_message_to_user( @@ -424,6 +431,7 @@ def _filter_privacy_request_queryset( errored_gt: Optional[datetime] = None, external_id: Optional[str] = None, action_type: Optional[ActionType] = None, + include_consent_webhook_requests: Optional[bool] = False, ) -> Query: """ Utility method to apply filters to our privacy request query. @@ -546,6 +554,14 @@ def _filter_privacy_request_queryset( ) query = query.filter(PrivacyRequest.policy_id.in_(policy_ids_for_action_type)) + if not include_consent_webhook_requests: + query = query.filter( + or_( + PrivacyRequest.source != PrivacyRequestSource.consent_webhook, + PrivacyRequest.source.is_(None), + ) + ) + return query @@ -1988,10 +2004,12 @@ def create_privacy_request_func( db: Session, config_proxy: ConfigProxy, data: Annotated[List[PrivacyRequestCreate], Field()], # type: ignore + *, authenticated: bool = False, - privacy_preferences: List[ - PrivacyPreferenceHistory - ] = [], # For consent requests only + privacy_preferences: Optional[ + List[PrivacyPreferenceHistory] + ] = None, # For consent requests only + user_id: Optional[str] = None, ) -> BulkPostPrivacyRequests: """Creates privacy requests. @@ -2003,6 +2021,8 @@ def create_privacy_request_func( "Application redis cache required, but it is currently disabled! Please update your application configuration to enable integration with a redis cache." ) + privacy_preferences = privacy_preferences or [] + created = [] failed = [] # Optional fields to validate here are those that are both nullable in the DB, and exist @@ -2016,6 +2036,7 @@ def create_privacy_request_func( "finished_processing_at", "consent_preferences", "property_id", + "source", ] for privacy_request_data in data: if not any(privacy_request_data.identity.model_dump(mode="json").values()): @@ -2077,6 +2098,14 @@ def create_privacy_request_func( kwargs[field] = attr + # if the privacy request originated from the request manager (Admin UI) + # then add the user_id as the submitted_by user + if ( + getattr(privacy_request_data, "source") + == PrivacyRequestSource.request_manager + ): + kwargs["submitted_by"] = user_id + try: privacy_request: PrivacyRequest = PrivacyRequest.create(db=db, data=kwargs) privacy_request.persist_identity( diff --git a/src/fides/api/models/privacy_request.py b/src/fides/api/models/privacy_request.py index 945b1aa666..a7ffa67ab3 100644 --- a/src/fides/api/models/privacy_request.py +++ b/src/fides/api/models/privacy_request.py @@ -167,6 +167,22 @@ class PrivacyRequestStatus(str, EnumType): error = "error" +class PrivacyRequestSource(str, EnumType): + """ + The source where the privacy request originated from + + - Privacy Center: Request created from the Privacy Center + - Request Manager: Request submitted from the Admin UI's Request manager page + - Consent Webhook: Request created as a side-effect of a consent webhook request (bidirectional consent) + - Fides.js: Request created as a side-effect of a privacy preference update from Fides.js + """ + + privacy_center = "Privacy Center" + request_manager = "Request Manager" + consent_webhook = "Consent Webhook" + fides_js = "Fides.js" + + class CallbackType(EnumType): """We currently have three types of Webhooks: pre-approval, pre (-execution), post (-execution)""" @@ -266,6 +282,11 @@ class PrivacyRequest( ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"), nullable=True, ) + submitted_by = Column( + String, + ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"), + nullable=True, + ) custom_privacy_request_fields_approved_by = Column( String, ForeignKey(FidesUser.id_field_path, ondelete="SET NULL"), @@ -294,6 +315,7 @@ class PrivacyRequest( cancel_reason = Column(String(200)) canceled_at = Column(DateTime(timezone=True), nullable=True) consent_preferences = Column(MutableList.as_mutable(JSONB), nullable=True) + source = Column(EnumColumn(PrivacyRequestSource), nullable=True) # passive_deletes="all" prevents execution logs from having their privacy_request_id set to null when # a privacy_request is deleted. We want to retain for record-keeping. @@ -1523,6 +1545,8 @@ class ConsentRequest(IdentityVerificationMixin, Base): nullable=True, ) + source = Column(EnumColumn(PrivacyRequestSource), nullable=True) + privacy_request_id = Column(String, ForeignKey(PrivacyRequest.id), nullable=True) privacy_request = relationship(PrivacyRequest) diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py index a9036db8e9..3abbc280e2 100644 --- a/src/fides/api/schemas/privacy_request.py +++ b/src/fides/api/schemas/privacy_request.py @@ -10,6 +10,7 @@ from fides.api.models.privacy_request import ( CheckpointActionRequired, ExecutionLogStatus, + PrivacyRequestSource, PrivacyRequestStatus, ) from fides.api.schemas.api import BulkResponse, BulkUpdateFailed @@ -85,6 +86,7 @@ class PrivacyRequestCreate(FidesSchema): encryption_key: Optional[str] = None property_id: Optional[str] = None consent_preferences: Optional[List[Consent]] = None # TODO Slated for deprecation + source: Optional[PrivacyRequestSource] = None @field_validator("encryption_key") @classmethod @@ -103,6 +105,7 @@ class ConsentRequestCreate(FidesSchema): identity: Identity custom_privacy_request_fields: Optional[Dict[str, CustomPrivacyRequestField]] = None property_id: Optional[str] = None + source: Optional[PrivacyRequestSource] = None class FieldsAffectedResponse(FidesSchema): @@ -241,6 +244,7 @@ class PrivacyRequestResponse(FidesSchema): days_left: Optional[int] = None custom_privacy_request_fields_approved_by: Optional[str] = None custom_privacy_request_fields_approved_at: Optional[datetime] = None + source: Optional[PrivacyRequestSource] = None model_config = ConfigDict(from_attributes=True, use_enum_values=True) diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index f520b7bbe7..ec5549b7d7 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -61,6 +61,7 @@ Consent, ConsentRequest, PrivacyRequest, + PrivacyRequestSource, PrivacyRequestStatus, ProvidedIdentity, RequestTask, @@ -2745,6 +2746,7 @@ def provided_identity_and_consent_request( consent_request_data = { "provided_identity_id": provided_identity.id, + "source": PrivacyRequestSource.privacy_center, } consent_request = ConsentRequest.create(db, data=consent_request_data) diff --git a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py index befbb7c370..e20771dbec 100644 --- a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py @@ -13,6 +13,7 @@ Consent, ConsentRequest, CustomPrivacyRequestField, + PrivacyRequestSource, PrivacyRequestStatus, ProvidedIdentity, ) @@ -367,6 +368,32 @@ def test_consent_request_email_and_phone_default_to_email( ).first() assert provided_identity is not None + @pytest.mark.usefixtures( + "messaging_config", + "subject_identity_verification_required", + ) + @patch("fides.api.service._verification.dispatch_message") + def test_consent_request_with_source( + self, + mock_dispatch_message, + db, + api_client, + url, + ): + data = { + "identity": {"email": "test@example.com"}, + "source": PrivacyRequestSource.privacy_center, + } + response = api_client.post(url, json=data) + assert response.status_code == 200 + assert mock_dispatch_message.called + + consent_request_id = response.json()["consent_request_id"] + consent_request = ConsentRequest.get_by_key_or_id( + db=db, data={"id": consent_request_id} + ) + assert consent_request.source == PrivacyRequestSource.privacy_center + @pytest.mark.usefixtures( "messaging_config", "sovrn_email_connection_config", diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 77c46feb10..6c4d2897b8 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -38,11 +38,12 @@ PrivacyRequest, PrivacyRequestError, PrivacyRequestNotifications, + PrivacyRequestSource, PrivacyRequestStatus, generate_request_task_callback_jwe, ) from fides.api.oauth.jwt import generate_jwe -from fides.api.oauth.roles import APPROVER, VIEWER +from fides.api.oauth.roles import APPROVER, OWNER, VIEWER from fides.api.schemas.dataset import DryRunDatasetResponse from fides.api.schemas.masking.masking_secrets import SecretType from fides.api.schemas.messaging.messaging import ( @@ -95,6 +96,7 @@ V1_URL_PREFIX, ) from fides.config import CONFIG +from tests.conftest import generate_auth_header_for_user, generate_role_header_for_user page_size = Params().size @@ -934,6 +936,7 @@ def test_get_privacy_requests_by_id( "reviewed_by": None, "paused_at": None, "reviewer": None, + "source": None, "policy": { "drp_action": None, "execution_timeframe": 7, @@ -998,6 +1001,7 @@ def test_get_privacy_requests_by_partial_id( "reviewed_by": None, "paused_at": None, "reviewer": None, + "source": None, "policy": { "execution_timeframe": 7, "drp_action": None, @@ -1429,6 +1433,7 @@ def test_verbose_privacy_requests( "reviewed_by": None, "paused_at": None, "reviewer": None, + "source": None, "policy": { "execution_timeframe": 7, "drp_action": None, @@ -1945,6 +1950,7 @@ def test_privacy_request_search_by_id( "reviewed_by": None, "paused_at": None, "reviewer": None, + "source": None, "policy": { "drp_action": None, "execution_timeframe": 7, @@ -2009,6 +2015,7 @@ def test_privacy_request_search_by_partial_id( "reviewed_by": None, "paused_at": None, "reviewer": None, + "source": None, "policy": { "execution_timeframe": 7, "drp_action": None, @@ -2519,6 +2526,7 @@ def test_verbose_privacy_requests( "reviewed_by": None, "paused_at": None, "reviewer": None, + "source": None, "policy": { "execution_timeframe": 7, "drp_action": None, @@ -4390,6 +4398,7 @@ def test_resume_privacy_request( "reviewed_at": None, "reviewed_by": None, "reviewer": None, + "source": None, "paused_at": None, "policy": { "execution_timeframe": 7, @@ -6129,6 +6138,54 @@ def test_create_privacy_request( assert len(response_data) == 1 assert run_access_request_mock.called + @pytest.mark.parametrize( + "source, expected_submitted_by", + [ + (PrivacyRequestSource.request_manager, lambda user: user.client.user_id), + (PrivacyRequestSource.privacy_center, lambda _: None), + (None, lambda _: None), + ], + ) + @mock.patch( + "fides.api.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_request_manager_privacy_request_stores_submitted_by( + self, + run_access_request_mock, + db, + url, + api_client, + owner_user, + policy, + source, + expected_submitted_by, + ): + auth_header = generate_role_header_for_user( + owner_user, roles=owner_user.permissions.roles + ) + data = [ + { + "policy_key": policy.key, + "identity": {"email": "test@example.com"}, + "source": source, + } + ] + resp = api_client.post( + url, + headers=auth_header, + json=data, + ) + assert resp.status_code == 200 + + response_data = resp.json()["succeeded"] + assert len(response_data) == 1 + + pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) + assert pr.submitted_by == expected_submitted_by(owner_user) + + pr.delete(db=db) + assert run_access_request_mock.called + @pytest.mark.usefixtures("verification_config") @mock.patch( "fides.api.service.privacy_request.request_runner_service.run_privacy_request.delay" diff --git a/tests/ops/models/test_consent_request.py b/tests/ops/models/test_consent_request.py index 253f3f5617..327b5b3036 100644 --- a/tests/ops/models/test_consent_request.py +++ b/tests/ops/models/test_consent_request.py @@ -1,6 +1,8 @@ from unittest import mock from unittest.mock import MagicMock +import pytest + from fides.api.api.v1.endpoints.consent_request_endpoints import ( queue_privacy_request_to_propagate_consent_old_workflow, ) @@ -19,7 +21,7 @@ ConsentWithExecutableStatus, PrivacyRequestResponse, ) -from fides.api.schemas.redis_cache import Identity +from fides.api.schemas.redis_cache import CustomPrivacyRequestField, Identity paused_location = CollectionAddress("test_dataset", "test_collection") @@ -118,18 +120,35 @@ def test_consent_request(db): class TestQueuePrivacyRequestToPropagateConsentHelper: + + @pytest.mark.usefixtures("allow_custom_privacy_request_field_collection_enabled") @mock.patch( "fides.api.api.v1.endpoints.consent_request_endpoints.create_privacy_request_func" ) def test_queue_privacy_request_to_propagate_consent( self, mock_create_privacy_request, db, consent_policy ): - custom_fields = {"first_name": {"label": "First name", "value": "John"}} - mock_consent_request = MagicMock(spec=ConsentRequest) - mock_consent_request.id = "123" - mock_consent_request.get_persisted_custom_privacy_request_fields.return_value = ( - custom_fields + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "encrypted_value": {"value": "test@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + + consent_request = ConsentRequest.create( + db=db, + data={ + "provided_identity_id": provided_identity.id, + }, ) + + custom_fields = { + "first_name": CustomPrivacyRequestField(label="First name", value="John") + } + consent_request.persist_custom_privacy_request_fields( + db=db, custom_privacy_request_fields=custom_fields + ) + mock_create_privacy_request.return_value = BulkPostPrivacyRequests( succeeded=[ PrivacyRequestResponse( @@ -140,12 +159,6 @@ def test_queue_privacy_request_to_propagate_consent( ], failed=[], ) - provided_identity_data = { - "privacy_request_id": None, - "field_name": "email", - "encrypted_value": {"value": "test@email.com"}, - } - provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) consent_preferences = ConsentPreferences( consent=[{"data_use": "marketing.advertising", "opt_in": False}] @@ -161,7 +174,7 @@ def test_queue_privacy_request_to_propagate_consent( provided_identity=provided_identity, policy=DEFAULT_CONSENT_POLICY, consent_preferences=consent_preferences, - consent_request=mock_consent_request, + consent_request=consent_request, executable_consents=executable_consents, ) assert mock_create_privacy_request.called @@ -186,7 +199,9 @@ def test_queue_privacy_request_to_propagate_consent( value.model_dump(mode="json") ) - assert call_kwargs["data"][0].custom_privacy_request_fields == custom_fields + assert call_kwargs["data"][0].custom_privacy_request_fields == { + "first_name": {"label": "First name", "value": "John"} + } provided_identity.delete(db) @@ -196,29 +211,24 @@ def test_queue_privacy_request_to_propagate_consent( def test_do_not_queue_privacy_request_if_no_executable_preferences( self, mock_create_privacy_request, db, consent_policy ): - custom_fields = {"first_name": {"label": "First name", "value": "John"}} - mock_consent_request = MagicMock(spec=ConsentRequest) - mock_consent_request.id = "123" - mock_consent_request.get_persisted_custom_privacy_request_fields.return_value = ( - custom_fields - ) - mock_create_privacy_request.return_value = BulkPostPrivacyRequests( - succeeded=[ - PrivacyRequestResponse( - id="fake_privacy_request_id", - status=PrivacyRequestStatus.pending, - policy=PolicyResponse.model_validate(consent_policy), - ) - ], - failed=[], - ) provided_identity_data = { "privacy_request_id": None, "field_name": "email", "encrypted_value": {"value": "test@email.com"}, } provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) - + consent_request = ConsentRequest.create( + db=db, + data={ + "provided_identity_id": provided_identity.id, + }, + ) + custom_fields = { + "first_name": CustomPrivacyRequestField(label="First name", value="John") + } + consent_request.persist_custom_privacy_request_fields( + db=db, custom_privacy_request_fields=custom_fields + ) consent_preferences = ConsentPreferences( consent=[{"data_use": "marketing.advertising", "opt_in": False}] ) @@ -228,7 +238,7 @@ def test_do_not_queue_privacy_request_if_no_executable_preferences( provided_identity=provided_identity, policy=DEFAULT_CONSENT_POLICY, consent_preferences=consent_preferences, - consent_request=mock_consent_request, + consent_request=consent_request, executable_consents=[ ConsentWithExecutableStatus( data_use="marketing.advertising", executable=False @@ -238,17 +248,30 @@ def test_do_not_queue_privacy_request_if_no_executable_preferences( assert not mock_create_privacy_request.called + @pytest.mark.usefixtures("allow_custom_privacy_request_field_collection_enabled") @mock.patch( "fides.api.api.v1.endpoints.consent_request_endpoints.create_privacy_request_func" ) def test_merge_in_browser_identity_with_provided_identity( self, mock_create_privacy_request, db, consent_policy ): - custom_fields = {"first_name": {"label": "First name", "value": "John"}} - mock_consent_request = MagicMock(spec=ConsentRequest) - mock_consent_request.id = "123" - mock_consent_request.get_persisted_custom_privacy_request_fields.return_value = ( - custom_fields + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "encrypted_value": {"value": "test@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + consent_request = ConsentRequest.create( + db=db, + data={ + "provided_identity_id": provided_identity.id, + }, + ) + custom_fields = { + "first_name": CustomPrivacyRequestField(label="First name", value="John") + } + consent_request.persist_custom_privacy_request_fields( + db=db, custom_privacy_request_fields=custom_fields ) mock_create_privacy_request.return_value = BulkPostPrivacyRequests( succeeded=[ @@ -260,12 +283,6 @@ def test_merge_in_browser_identity_with_provided_identity( ], failed=[], ) - provided_identity_data = { - "privacy_request_id": None, - "field_name": "email", - "encrypted_value": {"value": "test@email.com"}, - } - provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) browser_identity = Identity(ga_client_id="user_id_from_browser") consent_preferences = ConsentPreferences( @@ -277,7 +294,7 @@ def test_merge_in_browser_identity_with_provided_identity( provided_identity=provided_identity, policy=DEFAULT_CONSENT_POLICY, consent_preferences=consent_preferences, - consent_request=mock_consent_request, + consent_request=consent_request, executable_consents=[ ConsentWithExecutableStatus( data_use="marketing.advertising", executable=True @@ -298,5 +315,8 @@ def test_merge_in_browser_identity_with_provided_identity( call_kwargs["data"][0].custom_privacy_request_fields[label] = ( value.model_dump(mode="json") ) + assert call_kwargs["data"][0].custom_privacy_request_fields == { + "first_name": {"label": "First name", "value": "John"} + } provided_identity.delete(db) diff --git a/tests/ops/schemas/test_privacy_request.py b/tests/ops/schemas/test_privacy_request.py index c3135b7783..cbad1f9ee1 100644 --- a/tests/ops/schemas/test_privacy_request.py +++ b/tests/ops/schemas/test_privacy_request.py @@ -1,8 +1,13 @@ import pytest from pydantic import ValidationError +from fides.api.db.seed import DEFAULT_ACCESS_POLICY from fides.api.models.privacy_request import PrivacyRequestStatus -from fides.api.schemas.privacy_request import PrivacyRequestFilter +from fides.api.schemas.privacy_request import ( + PrivacyRequestCreate, + PrivacyRequestFilter, + PrivacyRequestSource, +) class TestPrivacyRequestFilter: @@ -26,3 +31,24 @@ def test_none_status(self): def test_invalid_status(self): with pytest.raises(ValidationError): PrivacyRequestFilter(status="invalid_status") + + +class TestPrivacyRequestCreate: + def test_valid_source(self): + PrivacyRequestCreate( + **{ + "identity": {"email": "user@example.com"}, + "policy_key": DEFAULT_ACCESS_POLICY, + "source": PrivacyRequestSource.privacy_center, + } + ) + + def test_invalid_source(self): + with pytest.raises(ValidationError): + PrivacyRequestCreate( + **{ + "identity": {"email": "user@example.com"}, + "policy_key": DEFAULT_ACCESS_POLICY, + "source": "Email", + } + )