diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml
index 315aeeb780..ebe9c7a8fa 100644
--- a/.fides/db_dataset.yml
+++ b/.fides/db_dataset.yml
@@ -1025,6 +1025,12 @@ dataset:
- name: opt_in
data_categories: [system.operations]
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+ - name: has_gpc_flag
+ data_categories: [ system.operations ]
+ data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+ - name: conflicts_with_gpc
+ data_categories: [ system.operations ]
+ data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
- name: provided_identity_id
data_categories: [system.operations]
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
diff --git a/.fides/fides.toml b/.fides/fides.toml
index 794da4f5b5..bb21d7f107 100644
--- a/.fides/fides.toml
+++ b/.fides/fides.toml
@@ -22,7 +22,7 @@ analytics_opt_out = false
[redis]
host = "redis"
-password = "testpassword"
+password = "redispassword"
port = 6379
charset = "utf8"
default_ttl_seconds = 604800
diff --git a/CHANGELOG.md b/CHANGELOG.md
index da91e36971..fdf28c155f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+<<<<<<< HEAD
+=======
+# Changelog
+
+>>>>>>> main
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/)
@@ -16,16 +21,34 @@ The types of changes are:
## [Unreleased](https://github.com/ethyca/fides/compare/2.7.0...main)
### Added
+
* Add API support for messaging config properties [#2551](https://github.com/ethyca/fides/pull/2551)
+* Access and erasure support for Kustomer [#2520](https://github.com/ethyca/fides/pull/2520)
+* Added the `erase_after` field on collections to be able to set the order for erasures [#2619](https://github.com/ethyca/fides/pull/2619)
### Changed
-- Admin UI
- - Add flow for selecting system types when manually creating a system [#2530](https://github.com/ethyca/fides/pull/2530)
- - Updated forms for privacy declarations [#2648](https://github.com/ethyca/fides/pull/2648)
- - Delete flow for privacy declarations [#2664](https://github.com/ethyca/fides/pull/2664)
+* Admin UI
+ * Add flow for selecting system types when manually creating a system [#2530](https://github.com/ethyca/fides/pull/2530)
+ * Updated forms for privacy declarations [#2648](https://github.com/ethyca/fides/pull/2648)
+ * Delete flow for privacy declarations [#2664](https://github.com/ethyca/fides/pull/2664)
+* Convert all config values to Pydantic `Field` objects [#2613](https://github.com/ethyca/fides/pull/2613)
+* Add warning to 'fides deploy' when installed outside of a virtual environment [#2641](https://github.com/ethyca/fides/pull/2641)
+* Change how config creation/import is handled across the application [#2622](https://github.com/ethyca/fides/pull/2622)
-- Add warning to 'fides deploy' when installed outside of a virtual environment [#2641](https://github.com/ethyca/fides/pull/2641)
+### Developer Experience
+
+* Set the security environment of the fides dev setup to `prod` instead of `dev` [#2588](https://github.com/ethyca/fides/pull/2588)
+* Removed unexpected default Redis password [#2666](https://github.com/ethyca/fides/pull/2666)
+* Privacy Center
+ * Typechecking and validation of the `config.json` will be checked for backwards-compatibility. [#2661](https://github.com/ethyca/fides/pull/2661)
+
+### Fixed
+
+* Fix support for "redis.user" setting when authenticating to the Redis cache [#2666](https://github.com/ethyca/fides/pull/2666)
+* Fix error with the classify dataset feature flag not writing the dataset to the server [#2675](https://github.com/ethyca/fides/pull/2675)
+* Admin UI
+ * Remove Identifiability (Data Qualifier) from taxonomy editor [2684](https://github.com/ethyca/fides/pull/2684)
## [2.7.0](https://github.com/ethyca/fides/compare/2.6.6...2.7.0)
@@ -33,7 +56,7 @@ The types of changes are:
* Access and erasure support for Braintree [#2223](https://github.com/ethyca/fides/pull/2223)
* Added route to send a test message [#2585](https://github.com/ethyca/fides/pull/2585)
* Add default storage configuration functionality and associated APIs [#2438](https://github.com/ethyca/fides/pull/2438)
-
+
* Admin UI
* Custom Metadata [#2536](https://github.com/ethyca/fides/pull/2536)
* Create Custom Lists
@@ -49,9 +72,15 @@ The types of changes are:
### Added
+<<<<<<< HEAD
- Added new Wunderkind Consent Saas Connector [#2600](https://github.com/ethyca/fides/pull/2600)
- Added new Sovrn Email Consent Connector [#2543](https://github.com/ethyca/fides/pull/2543/)
- Log Fides version at startup [#2566](https://github.com/ethyca/fides/pull/2566)
+=======
+* Added new Wunderkind Consent Saas Connector [#2600](https://github.com/ethyca/fides/pull/2600)
+* Added new Sovrn Email Consent Connector [#2543](https://github.com/ethyca/fides/pull/2543/)
+* Log Fides version at startup [#2566](https://github.com/ethyca/fides/pull/2566)
+>>>>>>> main
### Changed
@@ -73,11 +102,18 @@ The types of changes are:
### Developer Experience
+<<<<<<< HEAD
- Added new Cypress E2E smoke tests [#2241](https://github.com/ethyca/fides/pull/2241)
- Set the security environment of the fides dev setup to `prod` instead of `dev` [#2588](https://github.com/ethyca/fides/pull/2588)
- New command `nox -s e2e_test` which will spin up the test environment and run true E2E Cypress tests against it [#2417](https://github.com/ethyca/fides/pull/2417)
- Cypress E2E tests now run in CI and are reported to Cypress Cloud [#2417](https://github.com/ethyca/fides/pull/2417)
- Change from `randomint` to `uuid` in mongodb tests to reduce flakiness. [#2591](https://github.com/ethyca/fides/pull/2591)
+=======
+* Added new Cypress E2E smoke tests [#2241](https://github.com/ethyca/fides/pull/2241)
+* New command `nox -s e2e_test` which will spin up the test environment and run true E2E Cypress tests against it [#2417](https://github.com/ethyca/fides/pull/2417)
+* Cypress E2E tests now run in CI and are reported to Cypress Cloud [#2417](https://github.com/ethyca/fides/pull/2417)
+* Change from `randomint` to `uuid` in mongodb tests to reduce flakiness. [#2591](https://github.com/ethyca/fides/pull/2591)
+>>>>>>> main
### Removed
@@ -133,6 +169,7 @@ The types of changes are:
### Added
+<<<<<<< HEAD
- Added the `env` option to the `security` configuration options to allow for users to completely secure the API endpoints [#2267](https://github.com/ethyca/fides/pull/2267)
- Unified Fides Resources
- Added a dataset dropdown selector when configuring a connector to link an existing dataset to the connector configuration. [#2162](https://github.com/ethyca/fides/pull/2162)
@@ -152,6 +189,28 @@ The types of changes are:
- Patch Google Analytics Consent Connector to delete by client_id [#2355](https://github.com/ethyca/fides/pull/2355)
- Add a "skip_param_values option" to optionally skip when we are missing param values in the body [#2384](https://github.com/ethyca/fides/pull/2384)
- Adds a new Universal Analytics Connector that works with the UA Tracking Id
+=======
+* Added the `env` option to the `security` configuration options to allow for users to completely secure the API endpoints [#2267](https://github.com/ethyca/fides/pull/2267)
+* Unified Fides Resources
+ * Added a dataset dropdown selector when configuring a connector to link an existing dataset to the connector configuration. [#2162](https://github.com/ethyca/fides/pull/2162)
+ * Added new datasetconfig.ctl_dataset_id field to unify fides dataset resources [#2046](https://github.com/ethyca/fides/pull/2046)
+* Add new connection config routes that couple them with systems [#2249](https://github.com/ethyca/fides/pull/2249)
+* Add new select/deselect all permissions buttons [#2437](https://github.com/ethyca/fides/pull/2437)
+* Endpoints to allow a user with the `user:password-reset` scope to reset users' passwords. In addition, users no longer require a scope to edit their own passwords. [#2373](https://github.com/ethyca/fides/pull/2373)
+* New form to reset a user's password without knowing an old password [#2390](https://github.com/ethyca/fides/pull/2390)
+* Approve & deny buttons on the "Request details" page. [#2473](https://github.com/ethyca/fides/pull/2473)
+* Consent Propagation
+ * Add the ability to execute Consent Requests via the Privacy Request Execution layer [#2125](https://github.com/ethyca/fides/pull/2125)
+ * Add a Mailchimp Transactional Consent Connector [#2194](https://github.com/ethyca/fides/pull/2194)
+ * Allow defining a list of opt-in and/or opt-out requests in consent connectors [#2315](https://github.com/ethyca/fides/pull/2315)
+ * Add a Google Analytics Consent Connector for GA4 properties [#2302](https://github.com/ethyca/fides/pull/2302)
+ * Pass the GA Cookie from the Privacy Center [#2337](https://github.com/ethyca/fides/pull/2337)
+ * Rename "user_id" to more specific "ga_client_id" [#2356](https://github.com/ethyca/fides/pull/2356)
+ * Patch Google Analytics Consent Connector to delete by client_id [#2355](https://github.com/ethyca/fides/pull/2355)
+ * Add a "skip_param_values option" to optionally skip when we are missing param values in the body [#2384](https://github.com/ethyca/fides/pull/2384)
+ * Adds a new Universal Analytics Connector that works with the UA Tracking Id
+* Adds intake and storage of Global Privacy Control Signal props for Consent [#2599](https://github.com/ethyca/fides/pull/2599)
+>>>>>>> main
### Changed
@@ -173,9 +232,15 @@ The types of changes are:
### Developer Experience
+<<<<<<< HEAD
- `nox -s test_env` has been replaced with `nox -s "fides_env(dev)"`
- New command `nox -s "fides_env(test)"` creates a complete test environment with seed data (similar to `fides_env(dev)`) but with the production fides image so the built UI can be accessed at `localhost:8080` [#2399](https://github.com/ethyca/fides/pull/2399)
- Change from code climate to codecov for coverage reporting [#2402](https://github.com/ethyca/fides/pull/2402)
+=======
+* `nox -s test_env` has been replaced with `nox -s "fides_env(dev)"`
+* New command `nox -s "fides_env(test)"` creates a complete test environment with seed data (similar to `fides_env(dev)`) but with the production fides image so the built UI can be accessed at `localhost:8080` [#2399](https://github.com/ethyca/fides/pull/2399)
+* Change from code climate to codecov for coverage reporting [#2402](https://github.com/ethyca/fides/pull/2402)
+>>>>>>> main
### Fixed
diff --git a/clients/admin-ui/cypress/e2e/taxonomy.cy.ts b/clients/admin-ui/cypress/e2e/taxonomy.cy.ts
index e19b718ce9..3f304bd5c7 100644
--- a/clients/admin-ui/cypress/e2e/taxonomy.cy.ts
+++ b/clients/admin-ui/cypress/e2e/taxonomy.cy.ts
@@ -14,7 +14,6 @@ describe("Taxonomy management page", () => {
cy.getByTestId("tab-Data Categories");
cy.getByTestId("tab-Data Uses");
cy.getByTestId("tab-Data Subjects");
- cy.getByTestId("tab-Identifiability");
});
describe("Can view data", () => {
@@ -26,8 +25,6 @@ describe("Taxonomy management page", () => {
cy.wait("@getDataUses");
cy.getByTestId("tab-Data Subjects").click();
cy.wait("@getDataSubjects");
- cy.getByTestId("tab-Identifiability").click();
- cy.wait("@getDataQualifiers");
cy.getByTestId("tab-Data Categories").click();
cy.wait("@getDataCategories");
});
@@ -123,16 +120,6 @@ describe("Taxonomy management page", () => {
isParent: false,
request: "@putDataSubject",
},
- {
- tab: "Identifiability",
- name: "Aggregated Data",
- key: "aggregated",
- description:
- "Statistical data that does not contain individually identifying information but includes information about groups of individuals that renders individual identification impossible.",
- parentKey: "",
- isParent: true,
- request: "@putDataQualifier",
- },
];
expectedTabValues.forEach((tabValue) => {
cy.getByTestId(`tab-${tabValue.tab}`).click();
@@ -367,11 +354,6 @@ describe("Taxonomy management page", () => {
name: "Data subject",
request: "@postDataSubject",
},
- {
- tab: "Identifiability",
- name: "Data qualifier",
- request: "@postDataQualifier",
- },
];
expectedTabValues.forEach((tabValue) => {
cy.getByTestId(`tab-${tabValue.tab}`).click();
@@ -510,7 +492,6 @@ describe("Taxonomy management page", () => {
const tabValues = [
{ tab: "Data Categories", request: "@deleteDataCategory" },
{ tab: "Data Uses", request: "@deleteDataUse" },
- { tab: "Identifiability", request: "@deleteDataQualifier" },
];
tabValues.forEach((tabValue) => {
cy.getByTestId(`tab-${tabValue.tab}`).click();
diff --git a/clients/admin-ui/src/features/dataset/helpers.ts b/clients/admin-ui/src/features/dataset/helpers.ts
index 87ed21f558..a7b0096091 100644
--- a/clients/admin-ui/src/features/dataset/helpers.ts
+++ b/clients/admin-ui/src/features/dataset/helpers.ts
@@ -98,7 +98,7 @@ export const getUpdatedDatasetFromClassifyDataset = (
draftCollection.name
);
- if (classifyCollection?.name !== activeCollection) {
+ if (activeCollection && classifyCollection?.name !== activeCollection) {
return;
}
diff --git a/clients/admin-ui/src/features/taxonomy/TaxonomyTabs.tsx b/clients/admin-ui/src/features/taxonomy/TaxonomyTabs.tsx
index 180bee07b0..88add844e4 100644
--- a/clients/admin-ui/src/features/taxonomy/TaxonomyTabs.tsx
+++ b/clients/admin-ui/src/features/taxonomy/TaxonomyTabs.tsx
@@ -3,12 +3,7 @@ import { Box, Button } from "@fidesui/react";
import { useAppDispatch } from "~/app/hooks";
import DataTabs, { TabData } from "../common/DataTabs";
-import {
- useDataCategory,
- useDataQualifier,
- useDataSubject,
- useDataUse,
-} from "./hooks";
+import { useDataCategory, useDataSubject, useDataUse } from "./hooks";
import { setIsAddFormOpen } from "./taxonomy.slice";
import TaxonomyTabContent from "./TaxonomyTabContent";
@@ -25,10 +20,6 @@ const TABS: TabData[] = [
label: "Data Subjects",
content: ,
},
- {
- label: "Identifiability",
- content: ,
- },
];
const TaxonomyTabs = () => {
const dispatch = useAppDispatch();
diff --git a/clients/admin-ui/src/features/taxonomy/hooks.tsx b/clients/admin-ui/src/features/taxonomy/hooks.tsx
index 6f809e617f..945c560fb5 100644
--- a/clients/admin-ui/src/features/taxonomy/hooks.tsx
+++ b/clients/admin-ui/src/features/taxonomy/hooks.tsx
@@ -5,7 +5,6 @@ import { useCustomFields } from "~/features/common/custom-fields/hooks";
import { RTKResult } from "~/features/common/types";
import {
DataCategory,
- DataQualifier,
DataSubject,
DataSubjectRightsEnum,
DataUse,
@@ -23,12 +22,6 @@ import {
CustomTextInput,
} from "../common/form/inputs";
import { enumToOptions } from "../common/helpers";
-import {
- useCreateDataQualifierMutation,
- useDeleteDataQualifierMutation,
- useGetAllDataQualifiersQuery,
- useUpdateDataQualifierMutation,
-} from "../data-qualifier/data-qualifier.slice";
import {
useCreateDataSubjectMutation,
useDeleteDataSubjectMutation,
@@ -468,43 +461,3 @@ export const useDataSubject = (): TaxonomyHookData => {
transformEntityToInitialValues,
};
};
-
-export const useDataQualifier = (): TaxonomyHookData => {
- const { data, isLoading } = useGetAllDataQualifiersQuery();
- const [entityToEdit, setEntityToEdit] = useState(null);
-
- const labels = {
- fides_key: "Data qualifier",
- name: "Data qualifier name",
- description: "Data qualifier description",
- parent_key: "Parent data qualifier",
- };
-
- const [createDataQualifierMutationTrigger] = useCreateDataQualifierMutation();
- const [updateDataQualifierMutationTrigger] = useUpdateDataQualifierMutation();
- const [deleteDataQualifierMutationTrigger] = useDeleteDataQualifierMutation();
-
- const handleCreate = (initialValues: FormValues, newValues: FormValues) =>
- createDataQualifierMutationTrigger(
- transformBaseFormValuesToEntity(initialValues, newValues)
- );
-
- const handleEdit = (initialValues: FormValues, newValues: FormValues) =>
- updateDataQualifierMutationTrigger(
- transformBaseFormValuesToEntity(initialValues, newValues)
- );
-
- const handleDelete = deleteDataQualifierMutationTrigger;
-
- return {
- data,
- isLoading,
- labels,
- entityToEdit,
- setEntityToEdit,
- handleCreate,
- handleEdit,
- handleDelete,
- transformEntityToInitialValues: transformTaxonomyBaseToInitialValues,
- };
-};
diff --git a/clients/privacy-center/__tests__/config/examples.test.ts b/clients/privacy-center/__tests__/config/examples.test.ts
new file mode 100644
index 0000000000..2196f2aa0e
--- /dev/null
+++ b/clients/privacy-center/__tests__/config/examples.test.ts
@@ -0,0 +1,27 @@
+import { Config } from "~/types/config";
+import minimalJson from "~/config/examples/minimal.json";
+import basicJson from "~/config/examples/basic.json";
+import fullJson from "~/config/examples/full.json";
+
+describe("The Config type", () => {
+ /**
+ * If this test is failing, it's because the Config type has been updated in a way that is not
+ * backwards-compatible. To fix this, modify the type so that it is more permissive with what it
+ * allows, for example by making a property optional. Then you must handle the old/new
+ * compatibility at runtime with sensible defaults.
+ *
+ * DO NOT make this test pass by updating any example JSON files. Those files are real snapshots
+ * of configurations the PC should support.
+ *
+ * Discussion: https://github.com/ethyca/fides/discussions/2392
+ */
+ it("is backwards-compatible with old JSON files", () => {
+ const minimalTyped: Config = minimalJson;
+ const basicTyped: Config = basicJson;
+ const fullTyped: Config = fullJson;
+
+ expect(minimalTyped).toBeDefined();
+ expect(basicTyped).toBeDefined();
+ expect(fullTyped).toBeDefined();
+ });
+});
diff --git a/clients/privacy-center/__tests__/config/validate-config.test.ts b/clients/privacy-center/__tests__/config/validate-config.test.ts
new file mode 100644
index 0000000000..702d1eb0cc
--- /dev/null
+++ b/clients/privacy-center/__tests__/config/validate-config.test.ts
@@ -0,0 +1,41 @@
+import { produce } from "immer";
+
+import { configIsValid } from "~/scripts/validate-config";
+import minimalJson from "~/config/examples/minimal.json";
+import fullJson from "~/config/examples/full.json";
+
+describe("configIsValid", () => {
+ const testCases = [
+ {
+ name: "no consent options",
+ config: minimalJson,
+ expected: {
+ isValid: true,
+ },
+ },
+ {
+ name: "valid consent options",
+ config: fullJson,
+ expected: {
+ isValid: true,
+ },
+ },
+ {
+ name: "multiple executable consent options",
+ config: produce(fullJson, (draftConfig) => {
+ draftConfig.consent.consentOptions[0].executable = true;
+ draftConfig.consent.consentOptions[1].executable = true;
+ }),
+ expected: {
+ isValid: false,
+ message: "Cannot have more than one consent option be executable",
+ },
+ },
+ ];
+
+ testCases.forEach((tc) => {
+ test(tc.name, () => {
+ expect(configIsValid(tc.config)).toMatchObject(tc.expected);
+ });
+ });
+});
diff --git a/clients/privacy-center/config/__mocks__/config.json b/clients/privacy-center/config/examples/basic.json
similarity index 100%
rename from clients/privacy-center/config/__mocks__/config.json
rename to clients/privacy-center/config/examples/basic.json
diff --git a/clients/privacy-center/config/examples/full.json b/clients/privacy-center/config/examples/full.json
new file mode 100644
index 0000000000..36a7ad46a7
--- /dev/null
+++ b/clients/privacy-center/config/examples/full.json
@@ -0,0 +1,80 @@
+{
+ "title": "Take control of your data",
+ "description": "When you use our services, you’re trusting us with your information. We understand this is a big responsibility and work hard to protect your information and put you in control.",
+ "server_url_development": "http://localhost:8080/api/v1",
+ "server_url_production": "http://localhost:8080/api/v1",
+ "logo_path": "/logo.svg",
+ "actions": [
+ {
+ "policy_key": "default_access_policy",
+ "icon_path": "/download.svg",
+ "title": "Access your data",
+ "description": "We will provide you a report of all your personal data.",
+ "identity_inputs": {
+ "name": "optional",
+ "email": "required",
+ "phone": "optional"
+ }
+ },
+ {
+ "policy_key": "default_erasure_policy",
+ "icon_path": "/delete.svg",
+ "title": "Erase your data",
+ "description": "We will erase all of your personal data. This action cannot be undone.",
+ "identity_inputs": {
+ "name": "optional",
+ "email": "required",
+ "phone": "optional"
+ }
+ }
+ ],
+ "includeConsent": true,
+ "consent": {
+ "icon_path": "/consent.svg",
+ "title": "Manage your consent",
+ "description": "Manage your consent preferences, including the option to select 'Do Not Sell My Personal Information'.",
+ "identity_inputs": {
+ "email": "required",
+ "phone": "optional"
+ },
+ "policy_key": "default_consent_policy",
+ "consentOptions": [
+ {
+ "fidesDataUseKey": "advertising",
+ "name": "Data Sales or Sharing",
+ "description": "We may use some of your personal information for behavioral advertising purposes, which may be interpreted as 'Data Sales' or 'Data Sharing' under regulations such as CCPA, CPRA, VCDPA, etc.",
+ "url": "https://example.com/privacy#data-sales",
+ "default": {
+ "value": true,
+ "globalPrivacyControl": false
+ },
+ "highlight": false,
+ "cookieKeys": ["data_sales"],
+ "executable": false
+ },
+ {
+ "fidesDataUseKey": "advertising.first_party",
+ "name": "Email Marketing",
+ "description": "We may use some of your personal information to contact you about our products & services.",
+ "url": "https://example.com/privacy#email-marketing",
+ "default": {
+ "value": true,
+ "globalPrivacyControl": false
+ },
+ "highlight": false,
+ "cookieKeys": ["tracking"],
+ "executable": false
+ },
+ {
+ "fidesDataUseKey": "improve",
+ "name": "Product Analytics",
+ "description": "We may use some of your personal information to collect analytics about how you use our products & services.",
+ "url": "https://example.com/privacy#analytics",
+ "default": true,
+ "highlight": false,
+ "cookieKeys": ["tracking"],
+ "executable": false
+ }
+ ]
+ }
+}
diff --git a/clients/privacy-center/config/examples/minimal.json b/clients/privacy-center/config/examples/minimal.json
new file mode 100644
index 0000000000..9479860eda
--- /dev/null
+++ b/clients/privacy-center/config/examples/minimal.json
@@ -0,0 +1,13 @@
+{
+ "title": "Privacy center",
+ "description": "Privacy center test configuration",
+ "logo_path": "/logo.svg",
+ "actions": [
+ {
+ "policy_key": "download",
+ "icon_path": "/download.svg",
+ "title": "Download your data",
+ "description": "We will email you a report of the data related to your account."
+ }
+ ]
+}
diff --git a/clients/privacy-center/constants/index.ts b/clients/privacy-center/constants/index.ts
index a8a280830d..74c7010c20 100644
--- a/clients/privacy-center/constants/index.ts
+++ b/clients/privacy-center/constants/index.ts
@@ -1,6 +1,6 @@
/* eslint-disable import/prefer-default-export */
-import { Config } from "~/types/config";
+import { Config, IdentityInputs } from "~/types/config";
import configJson from "~/config/config.json";
export const config: Config = configJson;
@@ -13,4 +13,4 @@ export const hostUrl =
? config.server_url_development || (config as any).fidesops_host_development
: config.server_url_production || (config as any).fidesops_host_production;
-export const defaultIdentityInput = { email: "optional" };
+export const defaultIdentityInput: IdentityInputs = { email: "optional" };
diff --git a/clients/privacy-center/cypress/e2e/consent.cy.ts b/clients/privacy-center/cypress/e2e/consent.cy.ts
index dd89b998c6..c876b2c8e0 100644
--- a/clients/privacy-center/cypress/e2e/consent.cy.ts
+++ b/clients/privacy-center/cypress/e2e/consent.cy.ts
@@ -1,6 +1,7 @@
import { hostUrl } from "~/constants";
import { CONSENT_COOKIE_NAME } from "fides-consent";
import { GpcStatus } from "~/features/consent/types";
+import { ConsentPreferencesWithVerificationCode } from "~/types/api";
describe("Consent settings", () => {
describe("when the user isn't verified", () => {
@@ -140,20 +141,37 @@ describe("Consent settings", () => {
cy.getByTestId("save-btn").click();
cy.wait("@patchConsentPreferences").then((interception) => {
- const consent = interception.request.body.consent.find(
- (c: any) => c.data_use === "advertising"
+ const body = interception.request
+ .body as ConsentPreferencesWithVerificationCode;
+
+ const advertisingConsent = body.consent.find(
+ (c) => c.data_use === "advertising"
+ );
+ expect(advertisingConsent?.opt_in).to.eq(true);
+
+ const gpcConsent = body.consent.find(
+ (c) => c.data_use === "collect.gpc"
);
- expect(consent?.opt_in).to.eq(true);
+ expect(gpcConsent?.opt_in).to.eq(true);
+ expect(gpcConsent?.has_gpc_flag).to.eq(false);
+ expect(gpcConsent?.conflicts_with_gpc).to.eq(false);
// there should be no browser identity
- expect(interception.request.body.browser_identity).to.eql(undefined);
+ expect(body.browser_identity).to.eql(undefined);
});
- // The cookie should also have been updated.
- cy.getCookie(CONSENT_COOKIE_NAME).should((cookie) => {
- const cookieKeyConsent = JSON.parse(decodeURIComponent(cookie!.value));
- expect(cookieKeyConsent.data_sales).to.eq(true);
- });
+ // The cookie should also have been updated. This may take a moment in CI,
+ // so we `waitUntil` the value becomes what we expect.
+ // https://github.com/cypress-io/cypress/issues/4802#issuecomment-941891554
+ cy.waitUntil(() =>
+ cy.getCookie(CONSENT_COOKIE_NAME).then((cookie) => {
+ const cookieKeyConsent = JSON.parse(
+ decodeURIComponent(cookie!.value)
+ );
+ // `waitUntil` retries until we return a truthy value.
+ return cookieKeyConsent.data_sales === true;
+ })
+ );
});
it("can grab cookies and send to a consent request", () => {
@@ -169,9 +187,10 @@ describe("Consent settings", () => {
cy.getByTestId("save-btn").click();
cy.wait("@patchConsentPreferences").then((interception) => {
- const { body } = interception.request;
- expect(body.browser_identity.ga_client_id).to.eq(clientId);
- expect(body.browser_identity.ljt_readerID).to.eq(sovrnCookieValue);
+ const body = interception.request
+ .body as ConsentPreferencesWithVerificationCode;
+ expect(body.browser_identity?.ga_client_id).to.eq(clientId);
+ expect(body.browser_identity?.ljt_readerID).to.eq(sovrnCookieValue);
});
});
@@ -222,7 +241,7 @@ describe("Consent settings", () => {
});
describe("when globalPrivacyControl is enabled", () => {
- it("lets the user consent to override GPC", () => {
+ it("applies the GPC defaults", () => {
cy.visit("/consent?globalPrivacyControl=true");
cy.getByTestId("consent");
@@ -234,6 +253,48 @@ describe("Consent settings", () => {
cy.getByTestId("gpc-badge").should("contain", GpcStatus.APPLIED);
});
+
+ cy.getByTestId("save-btn").click();
+
+ cy.wait("@patchConsentPreferences").then((interception) => {
+ const body = interception.request
+ .body as ConsentPreferencesWithVerificationCode;
+
+ const gpcConsent = body.consent.find(
+ (c) => c.data_use === "collect.gpc"
+ );
+ expect(gpcConsent?.opt_in).to.eq(false);
+ expect(gpcConsent?.has_gpc_flag).to.eq(true);
+ expect(gpcConsent?.conflicts_with_gpc).to.eq(false);
+ });
+ });
+
+ it("lets the user consent to override GPC", () => {
+ cy.visit("/consent?globalPrivacyControl=true");
+ cy.getByTestId("consent");
+
+ cy.getByTestId("gpc-banner");
+
+ cy.getByTestId(`consent-item-card-collect.gpc`).within(() => {
+ cy.contains("GPC test");
+ cy.getRadio().should("not.be.checked").check({ force: true });
+
+ cy.getByTestId("gpc-badge").should("contain", GpcStatus.OVERRIDDEN);
+ });
+
+ cy.getByTestId("save-btn").click();
+
+ cy.wait("@patchConsentPreferences").then((interception) => {
+ const body = interception.request
+ .body as ConsentPreferencesWithVerificationCode;
+
+ const gpcConsent = body.consent.find(
+ (c) => c.data_use === "collect.gpc"
+ );
+ expect(gpcConsent?.opt_in).to.eq(true);
+ expect(gpcConsent?.has_gpc_flag).to.eq(true);
+ expect(gpcConsent?.conflicts_with_gpc).to.eq(true);
+ });
});
});
});
diff --git a/clients/privacy-center/cypress/support/commands.ts b/clients/privacy-center/cypress/support/commands.ts
index d35d1c526c..4f95a84abf 100644
--- a/clients/privacy-center/cypress/support/commands.ts
+++ b/clients/privacy-center/cypress/support/commands.ts
@@ -1,4 +1,6 @@
///
+// eslint-disable-next-line import/no-extraneous-dependencies
+import "cypress-wait-until";
import type { AppDispatch } from "~/app/store";
diff --git a/clients/privacy-center/cypress/tsconfig.json b/clients/privacy-center/cypress/tsconfig.json
index 6b6615ec67..539a639661 100644
--- a/clients/privacy-center/cypress/tsconfig.json
+++ b/clients/privacy-center/cypress/tsconfig.json
@@ -4,7 +4,7 @@
"jsx": "react",
"target": "es5",
"lib": ["es5", "dom"],
- "types": ["cypress", "node"]
+ "types": ["cypress", "cypress-wait-until", "node"]
},
"include": ["**/*.ts", "**/*.tsx", "../cypress.config.ts"],
"exclude": []
diff --git a/clients/privacy-center/next.config.js b/clients/privacy-center/next.config.js
index 1a04eae615..aa5028ddd1 100644
--- a/clients/privacy-center/next.config.js
+++ b/clients/privacy-center/next.config.js
@@ -1,43 +1,13 @@
const path = require("path");
-const privacyCenterConfig = require("./config/config.json");
+const { validateConfig } = require("./scripts/validate-config.js");
+
+validateConfig();
/** @type {import('next').NextConfig} */
const withBundleAnalyzer = require("@next/bundle-analyzer")({
enabled: process.env.ANALYZE === "true",
});
-/**
- * We can catch business logic problems with the supplied config.json here.
- *
- * Typescript errors will be caught by the initial instantiation of the config object.
- */
-const configIsValid = () => {
- // Cannot currently have more than one consent be executable
- if (privacyCenterConfig.consent) {
- const executables = privacyCenterConfig.consent.consentOptions.filter(
- (option) => option.executable
- );
- if (executables.length > 1) {
- return {
- isValid: false,
- message: "Cannot have more than one consent option be executable",
- };
- }
- }
- return { isValid: true };
-};
-
-// Check that our config.json is valid
-const { isValid, message } = configIsValid();
-if (!isValid) {
- // Throw a red warning
- console.error(
- "\x1b[31m%s",
- `Error with privacy center configuration: ${message}`
- );
- throw "Privacy center config invalid. Please fix, then rerun the application.";
-}
-
const nextConfig = {
reactStrictMode: true,
poweredByHeader: false,
diff --git a/clients/privacy-center/package-lock.json b/clients/privacy-center/package-lock.json
index a92797d585..5beece4f51 100644
--- a/clients/privacy-center/package-lock.json
+++ b/clients/privacy-center/package-lock.json
@@ -40,6 +40,7 @@
"babel-jest": "^27.5.1",
"cross-env": "^7.0.3",
"cypress": "^10.10.0",
+ "cypress-wait-until": "^1.7.2",
"eslint": "^8.23.0",
"eslint-config-airbnb": "^19.0.4",
"eslint-config-airbnb-typescript": "^16.1.0",
@@ -5730,6 +5731,12 @@
"node": ">=12.0.0"
}
},
+ "node_modules/cypress-wait-until": {
+ "version": "1.7.2",
+ "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-1.7.2.tgz",
+ "integrity": "sha512-uZ+M8/MqRcpf+FII/UZrU7g1qYZ4aVlHcgyVopnladyoBrpoaMJ4PKZDrdOJ05H5RHbr7s9Tid635X3E+ZLU/Q==",
+ "dev": true
+ },
"node_modules/cypress/node_modules/@types/node": {
"version": "14.18.32",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.32.tgz",
@@ -19930,6 +19937,12 @@
}
}
},
+ "cypress-wait-until": {
+ "version": "1.7.2",
+ "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-1.7.2.tgz",
+ "integrity": "sha512-uZ+M8/MqRcpf+FII/UZrU7g1qYZ4aVlHcgyVopnladyoBrpoaMJ4PKZDrdOJ05H5RHbr7s9Tid635X3E+ZLU/Q==",
+ "dev": true
+ },
"damerau-levenshtein": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz",
diff --git a/clients/privacy-center/package.json b/clients/privacy-center/package.json
index 6dc1ac46a8..7978036d76 100644
--- a/clients/privacy-center/package.json
+++ b/clients/privacy-center/package.json
@@ -12,7 +12,7 @@
"format": "prettier --write types/ theme/ pages/ config/ components/ __tests__/",
"format:ci": "prettier --check types/ theme/ pages/ config/ components/ __tests__/",
"test": "jest --watch",
- "test:ci": "jest",
+ "test:ci": "tsc --noEmit && jest",
"cy:open": "cypress open",
"cy:run": "cypress run",
"cy:start": "export NODE_ENV=test && npm run build && npm run start",
@@ -57,6 +57,7 @@
"babel-jest": "^27.5.1",
"cross-env": "^7.0.3",
"cypress": "^10.10.0",
+ "cypress-wait-until": "^1.7.2",
"eslint": "^8.23.0",
"eslint-config-airbnb": "^19.0.4",
"eslint-config-airbnb-typescript": "^16.1.0",
diff --git a/clients/privacy-center/pages/consent.tsx b/clients/privacy-center/pages/consent.tsx
index 7dfd01f503..60005353d0 100644
--- a/clients/privacy-center/pages/consent.tsx
+++ b/clients/privacy-center/pages/consent.tsx
@@ -40,6 +40,7 @@ import { getGpcStatus, makeCookieKeyConsent } from "~/features/consent/helpers";
import { useGetIdVerificationConfigQuery } from "~/features/id-verification";
import { ConsentPreferences } from "~/types/api";
import { GpcBanner } from "~/features/consent/GpcMessages";
+import { GpcStatus } from "~/features/consent/types";
const Consent: NextPage = () => {
const [consentRequestId] = useLocalStorage("consentRequestId", "");
@@ -219,11 +220,18 @@ const Consent: NextPage = () => {
const consent = consentOptions.map((option) => {
const defaultValue = resolveConsentValue(option.default, consentContext);
const value = fidesKeyToConsent[option.fidesDataUseKey] ?? defaultValue;
+ const gpcStatus = getGpcStatus({
+ value,
+ consentOption: option,
+ consentContext,
+ });
return {
data_use: option.fidesDataUseKey,
data_use_description: option.description,
opt_in: value,
+ has_gpc_flag: gpcStatus !== GpcStatus.NONE,
+ conflicts_with_gpc: gpcStatus === GpcStatus.OVERRIDDEN,
};
});
diff --git a/clients/privacy-center/scripts/README.md b/clients/privacy-center/scripts/README.md
new file mode 100644
index 0000000000..fe692143e7
--- /dev/null
+++ b/clients/privacy-center/scripts/README.md
@@ -0,0 +1,9 @@
+## Scripts
+
+This directory contains JS files that are not part of the built app, but are
+used during build or some other development step.
+
+Note that these scripts aren't written TS because that would require a
+compilation step for code that runs within Node, which isn't very convenient.
+However, we can use JSDoc comments to provide basic TS support:
+https://www.typescriptlang.org/docs/handbook/jsdoc-supported-types.html#types-1
\ No newline at end of file
diff --git a/clients/privacy-center/scripts/validate-config.js b/clients/privacy-center/scripts/validate-config.js
new file mode 100644
index 0000000000..0da0a65305
--- /dev/null
+++ b/clients/privacy-center/scripts/validate-config.js
@@ -0,0 +1,44 @@
+/**
+ * We can catch business logic problems with the supplied config.json here.
+ *
+ * Typescript errors will be caught by the initial instantiation of the config object.
+ */
+const configIsValid = (
+ /** @type {import('~/types/config').Config} */
+ config
+) => {
+ // Cannot currently have more than one consent be executable
+ if (config.consent) {
+ const executables = config.consent.consentOptions.filter(
+ (option) => option.executable
+ );
+ if (executables.length > 1) {
+ return {
+ isValid: false,
+ message: "Cannot have more than one consent option be executable",
+ };
+ }
+ }
+ return { isValid: true };
+};
+
+/**
+ * Loads `config.json`, validates it, and throws an exception if it is invalid.
+ */
+const validateConfig = () => {
+ const privacyCenterConfig = require("../config/config.json");
+ const { isValid, message } = configIsValid(privacyCenterConfig);
+ if (!isValid) {
+ // Throw a red warning
+ console.error(
+ "\x1b[31m%s",
+ `Error with privacy center configuration: ${message}`
+ );
+ throw "Privacy center config invalid. Please fix, then rerun the application.";
+ }
+};
+
+module.exports = {
+ configIsValid,
+ validateConfig,
+};
diff --git a/clients/privacy-center/types/api/models/Consent.ts b/clients/privacy-center/types/api/models/Consent.ts
index 21cf76c1b2..47578114be 100644
--- a/clients/privacy-center/types/api/models/Consent.ts
+++ b/clients/privacy-center/types/api/models/Consent.ts
@@ -9,4 +9,6 @@ export type Consent = {
data_use: string;
data_use_description?: string;
opt_in: boolean;
+ has_gpc_flag?: boolean;
+ conflicts_with_gpc?: boolean;
};
diff --git a/clients/privacy-center/types/api/models/Identity.ts b/clients/privacy-center/types/api/models/Identity.ts
index eb6ce4a1c0..8bb2ae6dd1 100644
--- a/clients/privacy-center/types/api/models/Identity.ts
+++ b/clients/privacy-center/types/api/models/Identity.ts
@@ -9,4 +9,5 @@ export type Identity = {
phone_number?: string;
email?: string;
ga_client_id?: string;
+ ljt_readerID?: string;
};
diff --git a/clients/privacy-center/types/config.ts b/clients/privacy-center/types/config.ts
index 5ffe3e5e77..0767b613f1 100644
--- a/clients/privacy-center/types/config.ts
+++ b/clients/privacy-center/types/config.ts
@@ -1,5 +1,11 @@
import { ConsentValue } from "fides-consent";
+export type IdentityInputs = {
+ name?: string;
+ email?: string;
+ phone?: string;
+};
+
export type Config = {
title: string;
description: string;
@@ -7,12 +13,12 @@ export type Config = {
server_url_production?: string;
logo_path: string;
actions: PrivacyRequestOption[];
- includeConsent: boolean;
+ includeConsent?: boolean;
consent?: {
icon_path: string;
title: string;
description: string;
- identity_inputs?: Record;
+ identity_inputs?: IdentityInputs;
policy_key?: string;
consentOptions: ConfigConsentOption[];
};
@@ -23,7 +29,7 @@ export type PrivacyRequestOption = {
icon_path: string;
title: string;
description: string;
- identity_inputs?: Record;
+ identity_inputs?: IdentityInputs;
};
export type ConfigConsentOption = {
diff --git a/data/saas/config/kustomer_config.yml b/data/saas/config/kustomer_config.yml
new file mode 100644
index 0000000000..feb6b67584
--- /dev/null
+++ b/data/saas/config/kustomer_config.yml
@@ -0,0 +1,52 @@
+saas_config:
+ fides_key:
+ name: Kustomer SaaS Config
+ type: kustomer
+ description: A sample schema representing the Kustomer connector for Fides
+ version: 0.1.0
+
+ connector_params:
+ - name: domain
+ default_value: api.kustomerapp.com
+ - name: api_key
+
+ client_config:
+ protocol: https
+ host:
+ authentication:
+ strategy: bearer
+ configuration:
+ token:
+
+ test_request:
+ method: GET
+ path: /v1/audit-logs
+ query_params:
+ - name: count
+ value: 1
+
+ endpoints:
+ - name: customer
+ requests:
+ read:
+ - method: GET
+ path: /v1/customers/email=
+ param_values:
+ - name: email
+ identity: email
+ data_path: data
+ - method: GET
+ path: /v1/customers/phone=
+ param_values:
+ - name: phone_number
+ identity: phone_number
+ data_path: data
+ delete:
+ method: DELETE
+ path: /v1/customers/
+ param_values:
+ - name: customer_id
+ references:
+ - dataset:
+ field: customer.id
+ direction: from
diff --git a/data/saas/config/saas_erasure_order_config.yml b/data/saas/config/saas_erasure_order_config.yml
new file mode 100644
index 0000000000..41517854c6
--- /dev/null
+++ b/data/saas/config/saas_erasure_order_config.yml
@@ -0,0 +1,88 @@
+saas_config:
+ fides_key:
+ name: SaaS Erasure Order Config
+ type: custom
+ description: A sample schema for testing
+ version: 0.0.1
+
+ connector_params:
+ - name: domain
+
+ client_config:
+ protocol: https
+ host:
+
+ test_request:
+ method: GET
+ path: /test/
+
+ endpoints:
+ - name: orders
+ erase_after:
+ - .orders_to_refunds
+ - .refunds_to_orders
+ requests:
+ read:
+ request_override: read_no_op
+ param_values:
+ - name: email
+ identity: email
+ delete:
+ request_override: delete_no_op
+ - name: refunds
+ erase_after:
+ - .orders_to_refunds
+ - .refunds_to_orders
+ requests:
+ read:
+ request_override: read_no_op
+ param_values:
+ - name: email
+ identity: email
+ delete:
+ request_override: delete_no_op
+ - name: labels
+ erase_after:
+ - .orders
+ - .refunds
+ requests:
+ read:
+ request_override: read_no_op
+ param_values:
+ - name: email
+ identity: email
+ delete:
+ request_override: delete_no_op
+ - name: orders_to_refunds
+ requests:
+ read:
+ request_override: read_no_op
+ param_values:
+ - name: order_id
+ references:
+ - dataset:
+ field: orders.orders_id
+ direction: from
+ delete:
+ request_override: delete_no_op
+ - name: refunds_to_orders
+ requests:
+ read:
+ request_override: read_no_op
+ param_values:
+ - name: refund_id
+ references:
+ - dataset:
+ field: refunds.refunds_id
+ direction: from
+ delete:
+ request_override: delete_no_op
+ - name: products
+ requests:
+ read:
+ request_override: read_no_op
+ param_values:
+ - name: email
+ identity: email
+ delete:
+ request_override: delete_no_op
diff --git a/data/saas/dataset/kustomer_dataset.yml b/data/saas/dataset/kustomer_dataset.yml
new file mode 100644
index 0000000000..949c9be3c9
--- /dev/null
+++ b/data/saas/dataset/kustomer_dataset.yml
@@ -0,0 +1,280 @@
+dataset:
+ - fides_key:
+ name: kustomer
+ description: A sample dataset representing the Kustomer connector for Fides
+ collections:
+ - name: customer
+ fields:
+ - name: type
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: id
+ data_categories: [user]
+ fidesops_meta:
+ data_type: string
+ primary_key: True
+ - name: attributes
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: name
+ data_categories: [user.name]
+ fidesops_meta:
+ data_type: string
+ - name: displayName
+ data_categories: [user]
+ fidesops_meta:
+ data_type: string
+ - name: displayColor
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: displayIcon
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: externalIds
+ - name: sharedExternalIds
+ - name: emails
+ fidesops_meta:
+ data_type: "object[]"
+ fields:
+ - name: type
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: verified
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: boolean
+ - name: externalVerified
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: boolean
+ - name: email
+ data_categories: [user.contact.email]
+ fidesops_meta:
+ data_type: string
+ - name: sharedEmails
+ - name: phones
+ fidesops_meta:
+ data_type: "object[]"
+ fields:
+ - name: type
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: verified
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: boolean
+ - name: phone
+ data_categories: [user.contact.phone_number]
+ fidesops_meta:
+ data_type: string
+ - name: sharedPhones
+ - name: whatsapps
+ - name: facebookIds
+ - name: instagramIds
+ - name: socials
+ - name: sharedSocials
+ - name: urls
+ - name: locations
+ - name: activeUsers
+ - name: watchers
+ - name: recentLocation
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: updatedAt
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: createdAt
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: updatedAt
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: modifiedAt
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: lastActivityAt
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: deleted
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: boolean
+ - name: lastMessageIn
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: sentiment
+ - name: lastMessageOut
+ - name: lastMessageUnrespondedTo
+ - name: lastConversation
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: sentiment
+ - name: channels
+ - name: tags
+ - name: conversationCounts
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: done
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
+ - name: open
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
+ - name: snoozed
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
+ - name: all
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
+ - name: preview
+ - name: tags
+ - name: progressiveStatus
+ - name: verified
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: boolean
+ - name: rev
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
+ - name: recentItems
+ - name: satisfactionLevel
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: firstSatisfaction
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: sentByTeams
+ - name: lastSatisfaction
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: sentByTeams
+ - name: roleGroupVersions
+ - name: accessOverride
+ - name: firstName
+ data_categories: [user.name]
+ fidesops_meta:
+ data_type: string
+ - name: lastName
+ data_categories: [user.name]
+ fidesops_meta:
+ data_type: string
+ - name: unmaskingWindow
+ - name: relationships
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: messages
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: links
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: self
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: createdBy
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: links
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: self
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: data
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: type
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: id
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: modifiedBy
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: links
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: self
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: data
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: type
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: id
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: org
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: data
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: type
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: id
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: links
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: self
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
+ - name: links
+ fidesops_meta:
+ data_type: object
+ fields:
+ - name: self
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: string
diff --git a/data/saas/dataset/saas_erasure_order_dataset.yml b/data/saas/dataset/saas_erasure_order_dataset.yml
new file mode 100644
index 0000000000..513088b6e2
--- /dev/null
+++ b/data/saas/dataset/saas_erasure_order_dataset.yml
@@ -0,0 +1,46 @@
+dataset:
+ - fides_key:
+ name: SaaS Erasure Order Dataset
+ description: A sample dataset for testing
+ collections:
+ - name: orders
+ fields:
+ - name: id
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
+ primary_key: True
+ - name: refunds
+ fields:
+ - name: id
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
+ primary_key: True
+ - name: labels
+ fields:
+ - name: id
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
+ primary_key: True
+ - name: orders_to_refunds
+ fields:
+ - name: id
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
+ primary_key: True
+ - name: refunds_to_orders
+ fields:
+ - name: id
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
+ primary_key: True
+ - name: products
+ fields:
+ - name: id
+ data_categories: [system.operations]
+ fidesops_meta:
+ data_type: integer
diff --git a/data/saas/saas_connector_registry.toml b/data/saas/saas_connector_registry.toml
index f91eb05444..b9936f41f0 100644
--- a/data/saas/saas_connector_registry.toml
+++ b/data/saas/saas_connector_registry.toml
@@ -177,3 +177,9 @@ config = "data/saas/config/universal_analytics_config.yml"
dataset = "data/saas/dataset/universal_analytics_dataset.yml"
icon = "data/saas/icon/google_analytics.svg"
human_readable = "Universal Analytics"
+
+[kustomer]
+config = "data/saas/config/kustomer_config.yml"
+dataset = "data/saas/dataset/kustomer_dataset.yml"
+icon = "data/saas/icon/default.svg"
+human_readable = "Kustomer"
diff --git a/docker-compose.child-env.yml b/docker-compose.child-env.yml
index 926b4209bf..7488fbd508 100644
--- a/docker-compose.child-env.yml
+++ b/docker-compose.child-env.yml
@@ -66,9 +66,7 @@ services:
redis-child:
image: "redis:6.2.5-alpine"
- command: redis-server --requirepass testpassword
- environment:
- - REDIS_PASSWORD=testpassword
+ command: redis-server --requirepass redispassword
expose:
- 6379
ports:
diff --git a/docker-compose.yml b/docker-compose.yml
index a891fcf613..7d3e052307 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -127,13 +127,23 @@ services:
redis:
image: "redis:6.2.5-alpine"
- command: redis-server --requirepass testpassword
- environment:
- - REDIS_PASSWORD=testpassword
+ # AUTH option #1: no authentication at all
+ # command: redis-server
+ # AUTH option #2: require password
+ command: redis-server --requirepass redispassword
+ # AUTH option #3: Redis ACL defined in redis.conf
+ # command: redis-server /usr/local/etc/redis/redis.conf
expose:
- 6379
ports:
- "0.0.0.0:6379:6379"
+ volumes:
+ # Mount a redis.conf file for configuration
+ # NOTE: Only used by "AUTH option #3" above!
+ - type: bind
+ source: ./docker/redis
+ target: /usr/local/etc/redis
+ read_only: False
volumes:
postgres: null
diff --git a/docker/docker-compose.minimal-config.yml b/docker/docker-compose.minimal-config.yml
index c86c4e064f..15e49ea9e6 100644
--- a/docker/docker-compose.minimal-config.yml
+++ b/docker/docker-compose.minimal-config.yml
@@ -22,6 +22,7 @@ services:
FIDES__DATABASE__PASSWORD: "fides"
FIDES__DATABASE__PORT: "5432"
FIDES__DATABASE__DB: "fides"
+ FIDES__REDIS__PASSWORD: "redispassword"
FIDES__USER__ANALYTICS_OPT_OUT: "True"
FIDES__SECURITY__APP_ENCRYPTION_KEY: "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3"
FIDES__SECURITY__OAUTH_ROOT_CLIENT_ID: "fidesadmin"
@@ -52,9 +53,7 @@ services:
redis:
image: "redis:6.2.5-alpine"
- command: redis-server --requirepass testpassword
- environment:
- - REDIS_PASSWORD=testpassword
+ command: redis-server --requirepass redispassword
expose:
- 6379
ports:
diff --git a/docker/redis/redis.conf b/docker/redis/redis.conf
new file mode 100644
index 0000000000..18dec3e0c6
--- /dev/null
+++ b/docker/redis/redis.conf
@@ -0,0 +1,15 @@
+# Redis configuration file for local Fides development
+#
+# Note that this file is not loaded by default, and it is checked in here for
+# manual testing in the future. To use this redis.conf file, do the following:
+# 1) Check docker-compose.yml is mounting this to /usr/local/etc/redis
+# 2) Edit docker-compose.yml to swap in the `command` for "AUTH option #3",
+# which should look like this:
+# ```
+# command: redis-server /usr/local/etc/redis/redis.conf
+# ```
+# 3) Make any edits to this file and bring up redis with `nox -s dev` or similar
+
+# Enable an ACL that gives access to all keys and all commands, but requires
+# a login with user="redisadmin" and password="redispassword"
+user redisadmin on ~* +@all >redispassword
\ No newline at end of file
diff --git a/docs/fides/docs/development/postman/Fides.postman_collection.json b/docs/fides/docs/development/postman/Fides.postman_collection.json
index 10867778d3..a35dfa4cc5 100644
--- a/docs/fides/docs/development/postman/Fides.postman_collection.json
+++ b/docs/fides/docs/development/postman/Fides.postman_collection.json
@@ -1,7 +1,7 @@
{
"info": {
"_postman_id": "03d28ae1-a240-42f8-839e-92a4d43132ed",
- "name": "Fidesops",
+ "name": "Fides",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
},
"item": [
diff --git a/noxfiles/ci_nox.py b/noxfiles/ci_nox.py
index 90657faa2c..5fbfc4964e 100644
--- a/noxfiles/ci_nox.py
+++ b/noxfiles/ci_nox.py
@@ -103,7 +103,11 @@ def xenon(session: nox.Session) -> None:
##################
@nox.session()
def check_install(session: nox.Session) -> None:
- """Check that fides installs works correctly."""
+ """
+ Check that fides installs and works correctly.
+
+ This is also a good sanity check for correct syntax.
+ """
session.install(".")
REQUIRED_ENV_VARS = {
@@ -212,6 +216,9 @@ def validate_test_matrix(session: nox.Session) -> None:
def collect_tests(session: nox.Session) -> None:
"""
Collect all pytests as a validity check.
+
+ Good to run as a sanity check that there aren't any obvious syntax
+ errors within the test code.
"""
session.install(".")
install_requirements(session)
diff --git a/noxfiles/dev_nox.py b/noxfiles/dev_nox.py
index 8ed905ba06..044a9833fc 100644
--- a/noxfiles/dev_nox.py
+++ b/noxfiles/dev_nox.py
@@ -21,7 +21,26 @@
@nox_session()
def dev(session: Session) -> None:
- """Spin up the application. Uses positional arguments for additional features."""
+ """
+ Spin up the Fides webserver in development mode alongside it's Postgres
+ database and Redis cache. Use positional arguments to run other services
+ like privacy center, shell, admin UI, etc. (see usage for examples)
+
+ Usage:
+ 'nox -s dev' - runs the Fides weserver, database, and cache
+ 'nox -s dev -- shell' - also open a shell on the Fides webserver
+ 'nox -s dev -- ui' - also build and run the Admin UI
+ 'nox -s dev -- pc' - also build and run the Privacy Center
+ 'nox -s dev -- remote_debug' - run with remote debugging enabled (see docker-compose.remote-debug.yml)
+ 'nox -s dev -- worker' - also run a Fides worker
+ 'nox -s dev -- child' - also run a Fides child node
+ 'nox -s dev -- ' - also run a test datastore (e.g. 'mssql', 'mongodb')
+
+ Note that you can combine any of the above arguments together, for example:
+ 'nox -s dev -- shell ui pc'
+
+ See noxfiles/dev_nox.py for more info
+ """
build(session, "dev")
session.notify("teardown")
diff --git a/src/fides/__init__.py b/src/fides/__init__.py
index 341d702df2..58a25603de 100644
--- a/src/fides/__init__.py
+++ b/src/fides/__init__.py
@@ -1,10 +1,5 @@
"""The root module for the Fides package."""
-from fides.core.config import get_config
-
from ._version import get_versions
-# Load the config here as a work around to the timing issue with environment variables
-get_config()
-
__version__ = get_versions()["version"]
del get_versions
diff --git a/src/fides/api/ctl/database/database.py b/src/fides/api/ctl/database/database.py
index a5c86cc1c7..c18fe570eb 100644
--- a/src/fides/api/ctl/database/database.py
+++ b/src/fides/api/ctl/database/database.py
@@ -13,7 +13,6 @@
from sqlalchemy_utils.types.encrypted.encrypted_type import InvalidCiphertextError
from fides.api.ctl.utils.errors import get_full_exception_name
-from fides.core.config import get_config
from fides.core.utils import get_db_engine
from fides.lib.db.base import Base # type: ignore[attr-defined]
@@ -22,8 +21,6 @@
DatabaseHealth = Literal["healthy", "unhealthy", "needs migration"]
-CONFIG = get_config()
-
def get_alembic_config(database_url: str) -> Config:
"""
diff --git a/src/fides/api/ctl/database/seed.py b/src/fides/api/ctl/database/seed.py
index 8689f0e539..c1471f83ec 100644
--- a/src/fides/api/ctl/database/seed.py
+++ b/src/fides/api/ctl/database/seed.py
@@ -17,7 +17,7 @@
PRIVACY_REQUEST_TRANSFER,
)
from fides.api.ops.models.policy import ActionType, DrpAction, Policy, Rule, RuleTarget
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.base_class import FidesBase
from fides.lib.exceptions import KeyOrNameAlreadyExists
from fides.lib.models.client import ClientDetail
@@ -27,7 +27,6 @@
from .crud import create_resource, list_resource
-CONFIG = get_config()
FIDESOPS_AUTOGENERATED_CLIENT_KEY = "default_oauth_client"
DEFAULT_ACCESS_POLICY = "default_access_policy"
DEFAULT_ACCESS_POLICY_RULE = "default_access_policy_rule"
diff --git a/src/fides/api/ctl/database/session.py b/src/fides/api/ctl/database/session.py
index 313104232d..8690b27f00 100644
--- a/src/fides/api/ctl/database/session.py
+++ b/src/fides/api/ctl/database/session.py
@@ -4,17 +4,15 @@
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import Session, sessionmaker
-from fides.core.config import get_config
-
-config = get_config()
+from fides.core.config import CONFIG
engine = create_async_engine(
- config.database.async_database_uri,
+ CONFIG.database.async_database_uri,
echo=False,
)
async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
-sync_engine = create_engine(config.database.sync_database_uri, echo=False)
+sync_engine = create_engine(CONFIG.database.sync_database_uri, echo=False)
sync_session = sessionmaker(
sync_engine,
class_=Session,
diff --git a/src/fides/api/ctl/migrations/env.py b/src/fides/api/ctl/migrations/env.py
index 41144afb20..7af11829a7 100644
--- a/src/fides/api/ctl/migrations/env.py
+++ b/src/fides/api/ctl/migrations/env.py
@@ -4,12 +4,12 @@
from sqlalchemy import engine_from_config, pool
from fides.api.ctl.utils.logger import setup as setup_fidesapi_logger
-from fides.core.config import get_config
+from fides.core.config import CONFIG
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
alembic_config = context.config
-fides_config = get_config()
+fides_config = CONFIG
# Interpret the config file for Python logging.
# This line sets up loggers basically.
diff --git a/src/fides/api/ctl/migrations/versions/7abe778b7082_update_fideslang_data_categories.py b/src/fides/api/ctl/migrations/versions/7abe778b7082_update_fideslang_data_categories.py
index 29f378dfa2..d468463fd5 100644
--- a/src/fides/api/ctl/migrations/versions/7abe778b7082_update_fideslang_data_categories.py
+++ b/src/fides/api/ctl/migrations/versions/7abe778b7082_update_fideslang_data_categories.py
@@ -12,11 +12,9 @@
from sqlalchemy.exc import ProgrammingError
from fides.api.ops.db.base import DatasetConfig
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.session import get_db_session
-CONFIG = get_config()
-
logger = logging.getLogger(__name__)
# revision identifiers, used by Alembic.
diff --git a/src/fides/api/ctl/migrations/versions/d65bbc647083_adds_gpc_info_to_consent_table.py b/src/fides/api/ctl/migrations/versions/d65bbc647083_adds_gpc_info_to_consent_table.py
new file mode 100644
index 0000000000..e4404a2be5
--- /dev/null
+++ b/src/fides/api/ctl/migrations/versions/d65bbc647083_adds_gpc_info_to_consent_table.py
@@ -0,0 +1,43 @@
+"""adds GPC info to consent table
+
+Revision ID: d65bbc647083
+Revises: c9ee230fa6da
+Create Date: 2023-02-15 13:57:36.159161
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d65bbc647083"
+down_revision = "c9ee230fa6da"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column("consent", sa.Column("has_gpc_flag", sa.Boolean(), nullable=True))
+ op.add_column(
+ "consent", sa.Column("conflicts_with_gpc", sa.Boolean(), nullable=True)
+ )
+ op.execute("UPDATE consent SET has_gpc_flag = false")
+ op.alter_column(
+ "consent", "has_gpc_flag", server_default="f", default=False, nullable=False
+ )
+ op.execute("UPDATE consent SET conflicts_with_gpc = false")
+ op.alter_column(
+ "consent",
+ "conflicts_with_gpc",
+ server_default="f",
+ default=False,
+ nullable=False,
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column("consent", "conflicts_with_gpc")
+ op.drop_column("consent", "has_gpc_flag")
+ # ### end Alembic commands ###
diff --git a/src/fides/api/ctl/routes/admin.py b/src/fides/api/ctl/routes/admin.py
index d13d586df3..9500d5e59f 100644
--- a/src/fides/api/ctl/routes/admin.py
+++ b/src/fides/api/ctl/routes/admin.py
@@ -9,9 +9,8 @@
from fides.api.ctl.utils.api_router import APIRouter
from fides.api.ops.api.v1 import scope_registry
from fides.api.ops.util.oauth_util import verify_oauth_client_cli
-from fides.core.config import FidesConfig, get_config
+from fides.core.config import CONFIG
-CONFIG: FidesConfig = get_config()
router = APIRouter(prefix=API_PREFIX, tags=["Admin"])
diff --git a/src/fides/api/ctl/routes/health.py b/src/fides/api/ctl/routes/health.py
index d0f63fbf5a..b754ca4a78 100644
--- a/src/fides/api/ctl/routes/health.py
+++ b/src/fides/api/ctl/routes/health.py
@@ -14,7 +14,7 @@
from fides.api.ops.tasks import celery_app, get_worker_ids
from fides.api.ops.util.cache import get_cache
from fides.api.ops.util.logger import Pii
-from fides.core.config import FidesConfig, get_config
+from fides.core.config import CONFIG
CacheHealth = Literal["healthy", "unhealthy", "no cache configured"]
@@ -30,8 +30,6 @@ class CoreHealthCheck(BaseModel):
workers: List[Optional[str]]
-CONFIG: FidesConfig = get_config()
-
router = APIRouter(tags=["Health"])
diff --git a/src/fides/api/ctl/sql_models.py b/src/fides/api/ctl/sql_models.py
index 828e237c4c..5a8f36f887 100644
--- a/src/fides/api/ctl/sql_models.py
+++ b/src/fides/api/ctl/sql_models.py
@@ -25,7 +25,7 @@
from sqlalchemy.sql import func
from sqlalchemy.sql.sqltypes import DateTime
-from fides.core.config import FidesConfig, get_config
+from fides.core.config import CONFIG
from fides.lib.db.base import ( # type: ignore[attr-defined]
Base,
ClientDetail,
@@ -34,8 +34,6 @@
)
from fides.lib.db.base_class import FidesBase as FideslibBase
-CONFIG: FidesConfig = get_config()
-
class FidesBase(FideslibBase):
"""
diff --git a/src/fides/api/main.py b/src/fides/api/main.py
index e07aa2c9b1..0510c525ee 100644
--- a/src/fides/api/main.py
+++ b/src/fides/api/main.py
@@ -3,7 +3,7 @@
"""
from datetime import datetime, timezone
from logging import DEBUG, WARNING
-from typing import Callable, List, Optional
+from typing import Callable, List, Optional, Pattern, Union
from fastapi import FastAPI, HTTPException, Request, Response, status
from fastapi.responses import FileResponse
@@ -70,11 +70,10 @@
from fides.api.ops.util.cache import get_cache
from fides.api.ops.util.logger import _log_exception
from fides.api.ops.util.oauth_util import get_root_client, verify_oauth_client_cli
-from fides.core.config import FidesConfig, get_config
+from fides.core.config import CONFIG
from fides.core.config.helpers import check_required_webserver_config_values
from fides.lib.oauth.api.routes.user_endpoints import router as user_router
-CONFIG: FidesConfig = get_config()
VERSION = fides.__version__
ROUTERS = crud.routers + [ # type: ignore[attr-defined]
@@ -89,13 +88,14 @@
def create_fides_app(
- cors_origins: List[str] = CONFIG.security.cors_origins,
+ cors_origins: Union[str, List[str]] = CONFIG.security.cors_origins,
+ cors_origin_regex: Optional[Pattern] = CONFIG.security.cors_origin_regex,
routers: List = ROUTERS,
app_version: str = VERSION,
api_prefix: str = API_PREFIX,
request_rate_limit: str = CONFIG.security.request_rate_limit,
rate_limit_prefix: str = CONFIG.security.rate_limit_prefix,
- security_env: str = CONFIG.security.env.value,
+ security_env: str = CONFIG.security.env,
) -> FastAPI:
"""Return a properly configured application."""
@@ -112,10 +112,11 @@ def create_fides_app(
fastapi_app.add_exception_handler(FunctionalityNotConfigured, handler)
fastapi_app.add_middleware(SlowAPIMiddleware)
- if cors_origins:
+ if cors_origins or cors_origin_regex:
fastapi_app.add_middleware(
CORSMiddleware,
allow_origins=[str(origin) for origin in cors_origins],
+ allow_origin_regex=cors_origin_regex,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
diff --git a/src/fides/api/ops/analytics.py b/src/fides/api/ops/analytics.py
index 43a958f1ac..a07f79e2a9 100644
--- a/src/fides/api/ops/analytics.py
+++ b/src/fides/api/ops/analytics.py
@@ -9,9 +9,7 @@
from fides import __version__ as fides_version
from fides.api.ops.models.registration import UserRegistration
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
def in_docker_container() -> bool:
diff --git a/src/fides/api/ops/api/deps.py b/src/fides/api/ops/api/deps.py
index 7e844af625..4c459cdfd4 100644
--- a/src/fides/api/ops/api/deps.py
+++ b/src/fides/api/ops/api/deps.py
@@ -5,13 +5,12 @@
from fides.api.ops.common_exceptions import FunctionalityNotConfigured
from fides.api.ops.util.cache import get_cache as get_redis_connection
-from fides.core.config import FidesConfig
+from fides.core.config import CONFIG, FidesConfig
from fides.core.config import get_config as get_app_config
from fides.core.config.config_proxy import ConfigProxy
from fides.lib.db.session import get_db_engine, get_db_session
_engine = None
-CONFIG = get_app_config()
def get_config() -> FidesConfig:
diff --git a/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py
index 35873f28c7..255e7b58be 100644
--- a/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py
+++ b/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py
@@ -57,12 +57,12 @@
from fides.api.ops.util.api_router import APIRouter
from fides.api.ops.util.logger import Pii
from fides.api.ops.util.oauth_util import verify_oauth_client
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.core.config.config_proxy import ConfigProxy
router = APIRouter(tags=["Consent"], prefix=V1_URL_PREFIX)
-CONFIG = get_config()
+
CONFIG_JSON_PATH = "clients/privacy-center/config/config.json"
@@ -507,6 +507,8 @@ def _prepare_consent_preferences(
data_use=x.data_use,
data_use_description=x.data_use_description,
opt_in=x.opt_in,
+ has_gpc_flag=x.has_gpc_flag,
+ conflicts_with_gpc=x.conflicts_with_gpc,
)
for x in consent
],
diff --git a/src/fides/api/ops/api/v1/endpoints/drp_endpoints.py b/src/fides/api/ops/api/v1/endpoints/drp_endpoints.py
index 5e9f3b3c3e..b10ed55cdf 100644
--- a/src/fides/api/ops/api/v1/endpoints/drp_endpoints.py
+++ b/src/fides/api/ops/api/v1/endpoints/drp_endpoints.py
@@ -46,11 +46,11 @@
from fides.api.ops.util.cache import FidesopsRedis
from fides.api.ops.util.logger import Pii
from fides.api.ops.util.oauth_util import verify_oauth_client
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.core.config.config_proxy import ConfigProxy
router = APIRouter(tags=["DRP"], prefix=urls.V1_URL_PREFIX)
-CONFIG = get_config()
+
EMBEDDED_EXECUTION_LOG_LIMIT = 50
diff --git a/src/fides/api/ops/api/v1/endpoints/encryption_endpoints.py b/src/fides/api/ops/api/v1/endpoints/encryption_endpoints.py
index c33218b653..88558f7004 100644
--- a/src/fides/api/ops/api/v1/endpoints/encryption_endpoints.py
+++ b/src/fides/api/ops/api/v1/endpoints/encryption_endpoints.py
@@ -24,16 +24,13 @@
encrypt_verify_secret_length as aes_gcm_encrypt,
)
from fides.api.ops.util.oauth_util import verify_oauth_client
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography import cryptographic_util
from fides.lib.cryptography.cryptographic_util import b64_str_to_bytes, bytes_to_b64_str
router = APIRouter(tags=["Encryption"], prefix=V1_URL_PREFIX)
-CONFIG = get_config()
-
-
@router.get(
ENCRYPTION_KEY,
dependencies=[Security(verify_oauth_client, scopes=[ENCRYPTION_EXEC])],
diff --git a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py
index bd9389fe22..8e122be528 100644
--- a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py
+++ b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py
@@ -24,8 +24,12 @@
MESSAGING_READ,
)
from fides.api.ops.api.v1.urn_registry import (
+ MESSAGING_ACTIVE_DEFAULT,
MESSAGING_BY_KEY,
MESSAGING_CONFIG,
+ MESSAGING_DEFAULT,
+ MESSAGING_DEFAULT_BY_TYPE,
+ MESSAGING_DEFAULT_SECRETS,
MESSAGING_SECRETS,
MESSAGING_TEST,
V1_URL_PREFIX,
@@ -34,11 +38,18 @@
MessageDispatchException,
MessagingConfigNotFoundException,
)
-from fides.api.ops.models.messaging import MessagingConfig, get_schema_for_secrets
+from fides.api.ops.models.messaging import (
+ MessagingConfig,
+ default_messaging_config_key,
+ default_messaging_config_name,
+ get_schema_for_secrets,
+)
from fides.api.ops.schemas.messaging.messaging import (
MessagingActionType,
MessagingConfigRequest,
+ MessagingConfigRequestBase,
MessagingConfigResponse,
+ MessagingServiceType,
TestMessagingStatusMessage,
)
from fides.api.ops.schemas.messaging.messaging_secrets_docs_only import (
@@ -55,12 +66,10 @@
from fides.api.ops.util.api_router import APIRouter
from fides.api.ops.util.logger import Pii
from fides.api.ops.util.oauth_util import verify_oauth_client
-from fides.core.config import get_config
+from fides.core.config.config_proxy import ConfigProxy
router = APIRouter(tags=["messaging"], prefix=V1_URL_PREFIX)
-CONFIG = get_config()
-
@router.post(
MESSAGING_CONFIG,
@@ -135,6 +144,84 @@ def patch_config_by_key(
)
+# this needs to come before other `/default/{messaging_type}` routes so that `/status`
+# isn't picked up as a path param
+@router.get(
+ MESSAGING_ACTIVE_DEFAULT,
+ dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_READ])],
+ response_model=MessagingConfigResponse,
+)
+def get_active_default_config(*, db: Session = Depends(deps.get_db)) -> MessagingConfig:
+ """
+ Retrieves the active default messaging config.
+ """
+ logger.info("Finding active default messaging config")
+ messaging_config = MessagingConfig.get_active_default(db)
+ if not messaging_config:
+ raise HTTPException(
+ status_code=HTTP_404_NOT_FOUND,
+ detail="No active default messaging config found.",
+ )
+ return messaging_config
+
+
+@router.put(
+ MESSAGING_DEFAULT,
+ dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_CREATE_OR_UPDATE])],
+ response_model=MessagingConfigResponse,
+)
+def put_default_config(
+ *,
+ db: Session = Depends(deps.get_db),
+ messaging_config: MessagingConfigRequestBase,
+) -> Optional[MessagingConfigResponse]:
+ """
+ Updates default messaging config for given service type.
+ """
+ logger.info(
+ "Starting upsert for default messaging config of type '{}'",
+ messaging_config.service_type,
+ )
+ incoming_data = messaging_config.dict()
+ existing_default = MessagingConfig.get_by_type(db, messaging_config.service_type)
+ if existing_default:
+ # take the key of the existing default and add that to the incoming data, to ensure we overwrite the same record
+ incoming_data["key"] = existing_default.key
+ incoming_data["name"] = existing_default.name
+ else:
+ # set a key and name for our config if we're creating a new default
+ incoming_data["name"] = default_messaging_config_name(
+ messaging_config.service_type.value
+ )
+ incoming_data["key"] = default_messaging_config_key(
+ messaging_config.service_type.value
+ )
+ return create_or_update_messaging_config(
+ db, MessagingConfigRequest(**incoming_data)
+ )
+
+
+@router.put(
+ MESSAGING_DEFAULT_SECRETS,
+ status_code=HTTP_200_OK,
+ dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_CREATE_OR_UPDATE])],
+ response_model=TestMessagingStatusMessage,
+)
+def put_default_config_secrets(
+ service_type: MessagingServiceType,
+ *,
+ db: Session = Depends(deps.get_db),
+ unvalidated_messaging_secrets: possible_messaging_secrets,
+) -> TestMessagingStatusMessage:
+ messaging_config = MessagingConfig.get_by_type(db, service_type=service_type)
+ if not messaging_config:
+ raise HTTPException(
+ status_code=HTTP_404_NOT_FOUND,
+ detail=f"No default messaging config found of type '{service_type}'",
+ )
+ return update_config_secrets(db, messaging_config, unvalidated_messaging_secrets)
+
+
@router.put(
MESSAGING_SECRETS,
status_code=HTTP_200_OK,
@@ -157,10 +244,18 @@ def put_config_secrets(
status_code=HTTP_404_NOT_FOUND,
detail=f"No messaging configuration with key {config_key}.",
)
+ return update_config_secrets(db, messaging_config, unvalidated_messaging_secrets)
+
+
+def update_config_secrets(
+ db: Session,
+ messaging_config: MessagingConfig,
+ unvalidated_messaging_secrets: possible_messaging_secrets,
+) -> TestMessagingStatusMessage:
try:
secrets_schema = get_schema_for_secrets(
- service_type=messaging_config.service_type,
+ service_type=messaging_config.service_type, # type: ignore
secrets=unvalidated_messaging_secrets,
)
except KeyError as exc:
@@ -175,16 +270,17 @@ def put_config_secrets(
)
logger.info(
- "Updating messaging config secrets for config with key '{}'", config_key
+ "Updating messaging config secrets for config with key '{}'",
+ messaging_config.key,
)
try:
- messaging_config.set_secrets(db=db, messaging_secrets=secrets_schema.dict())
+ messaging_config.set_secrets(db=db, messaging_secrets=secrets_schema.dict()) # type: ignore
except ValueError as exc:
raise HTTPException(
status_code=HTTP_400_BAD_REQUEST,
detail=exc.args[0],
)
- msg = f"Secrets updated for MessagingConfig with key: {config_key}."
+ msg = f"Secrets updated for MessagingConfig with key: {messaging_config.key}."
# todo- implement test status for messaging service
return TestMessagingStatusMessage(msg=msg, test_status=None)
@@ -231,6 +327,28 @@ def get_config_by_key(
)
+@router.get(
+ MESSAGING_DEFAULT_BY_TYPE,
+ dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_READ])],
+ response_model=MessagingConfigResponse,
+)
+def get_default_config_by_type(
+ service_type: MessagingServiceType, *, db: Session = Depends(deps.get_db)
+) -> MessagingConfig:
+ """
+ Retrieves default config for messaging service by type.
+ """
+ logger.info("Finding default messaging config of type '{}'", service_type)
+
+ messaging_config = MessagingConfig.get_by_type(db, service_type)
+ if not messaging_config:
+ raise HTTPException(
+ status_code=HTTP_404_NOT_FOUND,
+ detail=f"No default messaging config found of type '{service_type}'",
+ )
+ return messaging_config
+
+
@router.delete(
MESSAGING_BY_KEY,
status_code=HTTP_204_NO_CONTENT,
@@ -269,7 +387,9 @@ def delete_config_by_key(
},
)
def send_test_message(
- message_info: Identity, db: Session = Depends(deps.get_db)
+ message_info: Identity,
+ db: Session = Depends(deps.get_db),
+ config_proxy: ConfigProxy = Depends(deps.get_config_proxy),
) -> Dict[str, str]:
"""Sends a test message."""
try:
@@ -277,7 +397,7 @@ def send_test_message(
db,
action_type=MessagingActionType.TEST_MESSAGE,
to_identity=message_info,
- service_type=CONFIG.notifications.notification_service_type,
+ service_type=config_proxy.notifications.notification_service_type,
)
except MessageDispatchException as e:
raise HTTPException(
diff --git a/src/fides/api/ops/api/v1/endpoints/oauth_endpoints.py b/src/fides/api/ops/api/v1/endpoints/oauth_endpoints.py
index 066dd32d16..bdfd9bc058 100644
--- a/src/fides/api/ops/api/v1/endpoints/oauth_endpoints.py
+++ b/src/fides/api/ops/api/v1/endpoints/oauth_endpoints.py
@@ -47,7 +47,7 @@
)
from fides.api.ops.util.api_router import APIRouter
from fides.api.ops.util.oauth_util import verify_oauth_client
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.models.client import ClientDetail
from fides.lib.oauth.schemas.oauth import (
AccessToken,
@@ -57,9 +57,6 @@
router = APIRouter(tags=["OAuth"], prefix=V1_URL_PREFIX)
-CONFIG = get_config()
-
-
@router.post(
TOKEN,
response_model=AccessToken,
diff --git a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py
index 7622cfc82a..6156353f4c 100644
--- a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py
+++ b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py
@@ -146,13 +146,13 @@
from fides.api.ops.util.enums import ColumnSort
from fides.api.ops.util.logger import Pii
from fides.api.ops.util.oauth_util import verify_callback_oauth, verify_oauth_client
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.core.config.config_proxy import ConfigProxy
from fides.lib.models.audit_log import AuditLog, AuditLogAction
from fides.lib.models.client import ClientDetail
router = APIRouter(tags=["Privacy Requests"], prefix=V1_URL_PREFIX)
-CONFIG = get_config()
+
EMBEDDED_EXECUTION_LOG_LIMIT = 50
diff --git a/src/fides/api/ops/api/v1/endpoints/storage_endpoints.py b/src/fides/api/ops/api/v1/endpoints/storage_endpoints.py
index 86964387af..3163c8f2f8 100644
--- a/src/fides/api/ops/api/v1/endpoints/storage_endpoints.py
+++ b/src/fides/api/ops/api/v1/endpoints/storage_endpoints.py
@@ -308,6 +308,29 @@ def delete_config_by_key(
storage_config.delete(db)
+# this needs to come before other `/default/{storage_type}` routes so that `/status`
+# isn't picked up as a path param
+@router.get(
+ STORAGE_ACTIVE_DEFAULT,
+ dependencies=[Security(verify_oauth_client, scopes=[STORAGE_READ])],
+ response_model=StorageDestinationResponse,
+)
+def get_active_default_config(
+ *, db: Session = Depends(deps.get_db)
+) -> Optional[StorageConfig]:
+ """
+ Retrieves the active default storage config.
+ """
+ logger.info("Finding active default storage config")
+ storage_config = get_active_default_storage_config(db)
+ if not storage_config:
+ raise HTTPException(
+ status_code=HTTP_404_NOT_FOUND,
+ detail="No active default storage config found.",
+ )
+ return storage_config
+
+
@router.put(
STORAGE_DEFAULT,
status_code=HTTP_200_OK,
@@ -480,24 +503,3 @@ def get_default_config_by_type(
detail=f"No default config for storage type {storage_type.value}.",
)
return storage_config
-
-
-@router.get(
- STORAGE_ACTIVE_DEFAULT,
- dependencies=[Security(verify_oauth_client, scopes=[STORAGE_READ])],
- response_model=StorageDestinationResponse,
-)
-def get_active_default_config(
- *, db: Session = Depends(deps.get_db)
-) -> Optional[StorageConfig]:
- """
- Retrieves the active default storage config.
- """
- logger.info("Finding active default storage config")
- storage_config = get_active_default_storage_config(db)
- if not storage_config:
- raise HTTPException(
- status_code=HTTP_404_NOT_FOUND,
- detail="No active default storage config found.",
- )
- return storage_config
diff --git a/src/fides/api/ops/api/v1/endpoints/user_endpoints.py b/src/fides/api/ops/api/v1/endpoints/user_endpoints.py
index d5c91752c6..600e493ebe 100644
--- a/src/fides/api/ops/api/v1/endpoints/user_endpoints.py
+++ b/src/fides/api/ops/api/v1/endpoints/user_endpoints.py
@@ -27,7 +27,7 @@
oauth2_scheme,
verify_oauth_client,
)
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.cryptographic_util import b64_str_to_str
from fides.lib.cryptography.schemas.jwt import JWE_PAYLOAD_CLIENT_ID
from fides.lib.exceptions import AuthenticationError
@@ -41,7 +41,6 @@
UserUpdate,
)
-CONFIG = get_config()
router = APIRouter(tags=["Users"], prefix=V1_URL_PREFIX)
diff --git a/src/fides/api/ops/api/v1/endpoints/user_permission_endpoints.py b/src/fides/api/ops/api/v1/endpoints/user_permission_endpoints.py
index 88a725a78a..8650ed81a2 100644
--- a/src/fides/api/ops/api/v1/endpoints/user_permission_endpoints.py
+++ b/src/fides/api/ops/api/v1/endpoints/user_permission_endpoints.py
@@ -26,11 +26,10 @@
oauth2_scheme,
verify_oauth_client,
)
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.models.fides_user import FidesUser
from fides.lib.models.fides_user_permissions import FidesUserPermissions
-CONFIG = get_config()
router = APIRouter(tags=["User Permissions"], prefix=V1_URL_PREFIX)
diff --git a/src/fides/api/ops/api/v1/urn_registry.py b/src/fides/api/ops/api/v1/urn_registry.py
index 3fe83c662f..42db40bf1d 100644
--- a/src/fides/api/ops/api/v1/urn_registry.py
+++ b/src/fides/api/ops/api/v1/urn_registry.py
@@ -37,14 +37,18 @@
STORAGE_BY_KEY = "/storage/config/{config_key}"
STORAGE_UPLOAD = "/storage/{request_id}"
STORAGE_DEFAULT = "/storage/default"
+STORAGE_ACTIVE_DEFAULT = "/storage/default/active"
STORAGE_DEFAULT_SECRETS = "/storage/default/{storage_type}/secret"
STORAGE_DEFAULT_BY_TYPE = "/storage/default/{storage_type}"
-STORAGE_ACTIVE_DEFAULT = "/storage/active/default"
# Email URLs
MESSAGING_CONFIG = "/messaging/config"
MESSAGING_SECRETS = "/messaging/config/{config_key}/secret"
MESSAGING_BY_KEY = "/messaging/config/{config_key}"
+MESSAGING_DEFAULT = "/messaging/default"
+MESSAGING_ACTIVE_DEFAULT = "/messaging/default/active"
+MESSAGING_DEFAULT_SECRETS = "/messaging/default/{service_type}/secret"
+MESSAGING_DEFAULT_BY_TYPE = "/messaging/default/{service_type}"
MESSAGING_TEST = "/messaging/config/test"
# Policy URLs
diff --git a/src/fides/api/ops/graph/analytics_events.py b/src/fides/api/ops/graph/analytics_events.py
index 8e6924b57d..de719fc74c 100644
--- a/src/fides/api/ops/graph/analytics_events.py
+++ b/src/fides/api/ops/graph/analytics_events.py
@@ -15,13 +15,11 @@
from fides.api.ops.models.privacy_request import PrivacyRequest
from fides.api.ops.task.task_resources import TaskResources
from fides.api.ops.util.collection_util import Row
-from fides.core.config import get_config
+from fides.core.config import CONFIG
if TYPE_CHECKING:
from fides.api.ops.task.graph_task import GraphTask
-CONFIG = get_config()
-
async def fideslog_graph_failure(event: Optional[AnalyticsEvent]) -> None:
"""Send an Analytics Event if privacy request execution has failed"""
diff --git a/src/fides/api/ops/graph/config.py b/src/fides/api/ops/graph/config.py
index 2fe6aed445..ffd4ef77eb 100644
--- a/src/fides/api/ops/graph/config.py
+++ b/src/fides/api/ops/graph/config.py
@@ -416,6 +416,7 @@ class Collection(BaseModel):
fields: List[Field]
# an optional list of collections that this collection must run after
after: Set[CollectionAddress] = set()
+ erase_after: Set[CollectionAddress] = set()
# An optional set of dependent fields that need to be queried together
grouped_inputs: Set[str] = set()
diff --git a/src/fides/api/ops/models/application_config.py b/src/fides/api/ops/models/application_config.py
index 79f8223671..c7c744a959 100644
--- a/src/fides/api/ops/models/application_config.py
+++ b/src/fides/api/ops/models/application_config.py
@@ -15,11 +15,9 @@
)
from fides.api.ops.db.base_class import JSONTypeOverride
-from fides.core.config import FidesConfig, get_config
+from fides.core.config import CONFIG, FidesConfig
from fides.lib.db.base_class import Base
-CONFIG = get_config()
-
class ApplicationConfig(Base):
"""
diff --git a/src/fides/api/ops/models/connectionconfig.py b/src/fides/api/ops/models/connectionconfig.py
index 1dc1048142..ef9a6a78e9 100644
--- a/src/fides/api/ops/models/connectionconfig.py
+++ b/src/fides/api/ops/models/connectionconfig.py
@@ -16,13 +16,11 @@
from fides.api.ctl.sql_models import System # type: ignore[attr-defined]
from fides.api.ops.db.base_class import JSONTypeOverride
from fides.api.ops.schemas.saas.saas_config import SaaSConfig
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.base import Base # type: ignore[attr-defined]
from fides.lib.db.base_class import get_key_from_data
from fides.lib.exceptions import KeyOrNameAlreadyExists
-CONFIG = get_config()
-
class ConnectionTestStatus(enum.Enum):
"""Enum for supplying statuses of validating credentials for a Connection Config to the user"""
diff --git a/src/fides/api/ops/models/messaging.py b/src/fides/api/ops/models/messaging.py
index bdc6f1699e..55eaae6336 100644
--- a/src/fides/api/ops/models/messaging.py
+++ b/src/fides/api/ops/models/messaging.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
from typing import Optional
from loguru import logger
@@ -27,11 +29,10 @@
possible_messaging_secrets,
)
from fides.api.ops.util.logger import Pii
-from fides.core.config import get_config
+from fides.core.config import CONFIG
+from fides.core.config.config_proxy import ConfigProxy
from fides.lib.db.base import Base # type: ignore[attr-defined]
-CONFIG = get_config()
-
def get_messaging_method(
service_type: Optional[str],
@@ -115,6 +116,17 @@ def get_configuration(cls, db: Session, service_type: str) -> Base:
)
return instance
+ @classmethod
+ def get_by_type(
+ cls,
+ db: Session,
+ service_type: MessagingServiceType,
+ ) -> Optional[MessagingConfig]:
+ """
+ Retrieve the messaging config of the given type
+ """
+ return db.query(cls).filter_by(service_type=service_type).first()
+
def set_secrets(
self,
*,
@@ -144,3 +156,44 @@ def set_secrets(
self.secrets = messaging_secrets
self.save(db=db)
+
+ @classmethod
+ def get_active_default(cls, db: Session) -> Optional[MessagingConfig]:
+ """
+ Utility method to return the active default messaging configuration.
+
+ This is determined by looking at the `notifications.notification_service_type`
+ config property. We determine that config property's value by using
+ the ConfigProxy, which resolves the value based on API-set or traditional
+ config-set mechanisms.
+ """
+ active_default_messaging_type = ConfigProxy(
+ db
+ ).notifications.notification_service_type
+ if not active_default_messaging_type:
+ return None
+ try:
+ service_type = MessagingServiceType[active_default_messaging_type]
+ return cls.get_by_type(db, service_type)
+ except KeyError:
+ raise ValueError(
+ f"Unknown notification_service_type {active_default_messaging_type} configured"
+ )
+
+
+def default_messaging_config_name(service_type: str) -> str:
+ """
+ Utility function for consistency in generating default message config names.
+
+ Returns a name to be used in a default messaging config for the given type.
+ """
+ return f"Default Messaging Config [{service_type}]"
+
+
+def default_messaging_config_key(service_type: str) -> str:
+ """
+ Utility function for consistency in generating default message config keys.
+
+ Returns a key to be used in a default messaging config for the given type.
+ """
+ return f"default_messaging_config_{service_type.lower()}"
diff --git a/src/fides/api/ops/models/policy.py b/src/fides/api/ops/models/policy.py
index 12b252fdc4..9c6478979f 100644
--- a/src/fides/api/ops/models/policy.py
+++ b/src/fides/api/ops/models/policy.py
@@ -27,12 +27,10 @@
get_active_default_storage_config,
)
from fides.api.ops.util.data_category import _validate_data_category
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.base_class import Base, FidesBase
from fides.lib.models.client import ClientDetail
-CONFIG = get_config()
-
class CurrentStep(EnumType):
pre_webhooks = "pre_webhooks"
diff --git a/src/fides/api/ops/models/privacy_request.py b/src/fides/api/ops/models/privacy_request.py
index c9ef4a1588..7f5f2bc636 100644
--- a/src/fides/api/ops/models/privacy_request.py
+++ b/src/fides/api/ops/models/privacy_request.py
@@ -62,7 +62,7 @@
from fides.api.ops.util.collection_util import Row
from fides.api.ops.util.constants import API_DATE_FORMAT
from fides.api.ops.util.identity_verification import IdentityVerificationMixin
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.cryptographic_util import hash_with_salt
from fides.lib.db.base import Base # type: ignore[attr-defined]
from fides.lib.models.audit_log import AuditLog
@@ -70,9 +70,6 @@
from fides.lib.models.fides_user import FidesUser
from fides.lib.oauth.jwt import generate_jwe
-CONFIG = get_config()
-
-
# Locations from which privacy request execution can be resumed, in order.
EXECUTION_CHECKPOINTS = [
CurrentStep.pre_webhooks,
@@ -856,6 +853,10 @@ class Consent(Base):
data_use = Column(String, nullable=False)
data_use_description = Column(String)
opt_in = Column(Boolean, nullable=False)
+ has_gpc_flag = Column(Boolean, server_default="f", default=False, nullable=False)
+ conflicts_with_gpc = Column(
+ Boolean, server_default="f", default=False, nullable=False
+ )
provided_identity = relationship(ProvidedIdentity, back_populates="consent")
diff --git a/src/fides/api/ops/models/registration.py b/src/fides/api/ops/models/registration.py
index e7e3111e8d..710ace01ad 100644
--- a/src/fides/api/ops/models/registration.py
+++ b/src/fides/api/ops/models/registration.py
@@ -6,11 +6,9 @@
from sqlalchemy_utils import StringEncryptedType
from sqlalchemy_utils.types.encrypted.encrypted_type import AesEngine
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.base_class import Base, FidesBase
-CONFIG = get_config()
-
class UserRegistration(Base):
"""
diff --git a/src/fides/api/ops/models/storage.py b/src/fides/api/ops/models/storage.py
index 3d0630144e..962779b9d5 100644
--- a/src/fides/api/ops/models/storage.py
+++ b/src/fides/api/ops/models/storage.py
@@ -20,12 +20,10 @@
)
from fides.api.ops.util.logger import Pii
from fides.api.ops.util.storage_util import get_schema_for_secrets
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.core.config.config_proxy import ConfigProxy
from fides.lib.db.base_class import Base
-CONFIG = get_config()
-
class StorageConfig(Base):
"""The DB ORM model for StorageConfig"""
diff --git a/src/fides/api/ops/schemas/encryption_request.py b/src/fides/api/ops/schemas/encryption_request.py
index a2fcc6562b..41741b5e4a 100644
--- a/src/fides/api/ops/schemas/encryption_request.py
+++ b/src/fides/api/ops/schemas/encryption_request.py
@@ -3,9 +3,7 @@
from fides.api.ops.util.encryption.aes_gcm_encryption_scheme import (
verify_encryption_key,
)
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
class AesEncryptionRequest(BaseModel):
diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py
index dee5d740f2..e5ef26f067 100644
--- a/src/fides/api/ops/schemas/messaging/messaging.py
+++ b/src/fides/api/ops/schemas/messaging/messaging.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
from enum import Enum
from re import compile as regex
from typing import Any, Dict, List, Optional, Tuple, Type, Union
@@ -26,6 +28,16 @@ class MessagingServiceType(Enum):
TWILIO_TEXT = "TWILIO_TEXT"
TWILIO_EMAIL = "TWILIO_EMAIL"
+ @classmethod
+ def _missing_(
+ cls: Type[MessagingServiceType], value: Any
+ ) -> Optional[MessagingServiceType]:
+ value = value.upper()
+ for member in cls:
+ if member.value == value:
+ return member
+ return None
+
EMAIL_MESSAGING_SERVICES: Tuple[str, ...] = (
MessagingServiceType.MAILGUN.value,
@@ -254,11 +266,9 @@ class Config:
extra = Extra.forbid
-class MessagingConfigRequest(BaseModel):
- """Messaging Config Request Schema"""
+class MessagingConfigBase(BaseModel):
+ """Base model shared by messaging config related models"""
- name: str
- key: Optional[FidesKey]
service_type: MessagingServiceType
details: Optional[
Union[MessagingServiceDetailsMailgun, MessagingServiceDetailsTwilioEmail]
@@ -267,14 +277,24 @@ class MessagingConfigRequest(BaseModel):
class Config:
use_enum_values = False
orm_mode = True
+ extra = Extra.forbid
+
+
+class MessagingConfigRequestBase(MessagingConfigBase):
+ """Base model shared by messaging config requests to provide validation on request inputs"""
@root_validator(pre=True)
def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]:
service_type_pre = values.get("service_type")
if service_type_pre:
# uppercase to match enums in database
- values["service_type"] = service_type_pre.upper()
- service_type: MessagingServiceType = values["service_type"]
+ if isinstance(service_type_pre, str):
+ service_type_pre = service_type_pre.upper()
+ service_type: str = service_type_pre
+
+ # assign the transformed service_type value back into the values dict
+ values["service_type"] = service_type
+
if service_type == MessagingServiceType.MAILGUN.value:
cls._validate_details_schema(
values=values, schema=MessagingServiceDetailsMailgun
@@ -298,13 +318,18 @@ def _validate_details_schema(
schema.validate(values.get("details"))
-class MessagingConfigResponse(BaseModel):
+class MessagingConfigRequest(MessagingConfigRequestBase):
+ """Messaging Config Request Schema"""
+
+ name: str
+ key: Optional[FidesKey]
+
+
+class MessagingConfigResponse(MessagingConfigBase):
"""Messaging Config Response Schema"""
name: str
key: FidesKey
- service_type: MessagingServiceType
- details: Optional[Dict[MessagingServiceDetails, Any]]
class Config:
orm_mode = True
diff --git a/src/fides/api/ops/schemas/privacy_request.py b/src/fides/api/ops/schemas/privacy_request.py
index 4a48a1e7e4..6b108e4285 100644
--- a/src/fides/api/ops/schemas/privacy_request.py
+++ b/src/fides/api/ops/schemas/privacy_request.py
@@ -18,12 +18,10 @@
from fides.api.ops.util.encryption.aes_gcm_encryption_scheme import (
verify_encryption_key,
)
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.models.audit_log import AuditLogAction
from fides.lib.oauth.schemas.user import PrivacyRequestReviewer
-CONFIG = get_config()
-
class PrivacyRequestDRPStatus(EnumType):
"""A list of privacy request statuses specified by the Data Rights Protocol."""
@@ -60,6 +58,8 @@ class Consent(BaseSchema):
data_use: str
data_use_description: Optional[str] = None
opt_in: bool
+ has_gpc_flag: bool = False
+ conflicts_with_gpc: bool = False
class PrivacyRequestCreate(BaseSchema):
diff --git a/src/fides/api/ops/schemas/saas/saas_config.py b/src/fides/api/ops/schemas/saas/saas_config.py
index 9d02cde015..05b36fd893 100644
--- a/src/fides/api/ops/schemas/saas/saas_config.py
+++ b/src/fides/api/ops/schemas/saas/saas_config.py
@@ -241,6 +241,7 @@ class Endpoint(BaseModel):
name: str
requests: SaaSRequestMap
after: List[FidesCollectionKey] = []
+ erase_after: List[FidesCollectionKey] = []
@validator("requests")
def validate_grouped_inputs(
@@ -410,6 +411,10 @@ def get_graph(self, secrets: Dict[str, Any]) -> GraphDataset:
after={
CollectionAddress(*s.split(".")) for s in endpoint.after
},
+ erase_after={
+ CollectionAddress(*s.split("."))
+ for s in endpoint.erase_after
+ },
)
)
diff --git a/src/fides/api/ops/service/_verification.py b/src/fides/api/ops/service/_verification.py
index 3e4b011c2f..cd2ff77eba 100644
--- a/src/fides/api/ops/service/_verification.py
+++ b/src/fides/api/ops/service/_verification.py
@@ -12,11 +12,9 @@
from fides.api.ops.service.privacy_request.request_runner_service import (
generate_id_verification_code,
)
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.core.config.config_proxy import ConfigProxy
-CONFIG = get_config()
-
def send_verification_code_to_user(
db: Session, request: ConsentRequest | PrivacyRequest, to_identity: Identity | None
diff --git a/src/fides/api/ops/service/authentication/authentication_strategy_oauth2_authorization_code.py b/src/fides/api/ops/service/authentication/authentication_strategy_oauth2_authorization_code.py
index 6beb282d9f..2cbdb61b61 100644
--- a/src/fides/api/ops/service/authentication/authentication_strategy_oauth2_authorization_code.py
+++ b/src/fides/api/ops/service/authentication/authentication_strategy_oauth2_authorization_code.py
@@ -15,9 +15,7 @@
OAuth2AuthenticationStrategyBase,
)
from fides.api.ops.util.saas_util import assign_placeholders, map_param_values
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
class OAuth2AuthorizationCodeAuthenticationStrategy(OAuth2AuthenticationStrategyBase):
diff --git a/src/fides/api/ops/service/connectors/base_connector.py b/src/fides/api/ops/service/connectors/base_connector.py
index d7d0b90b1a..048bdf0af1 100644
--- a/src/fides/api/ops/service/connectors/base_connector.py
+++ b/src/fides/api/ops/service/connectors/base_connector.py
@@ -8,9 +8,8 @@
from fides.api.ops.models.privacy_request import Consent, PrivacyRequest
from fides.api.ops.service.connectors.query_config import QueryConfig
from fides.api.ops.util.collection_util import Row
-from fides.core.config import get_config
+from fides.core.config import CONFIG
-CONFIG = get_config()
DB_CONNECTOR_TYPE = TypeVar("DB_CONNECTOR_TYPE")
diff --git a/src/fides/api/ops/service/connectors/fides/fides_client.py b/src/fides/api/ops/service/connectors/fides/fides_client.py
index 5c24ba4bd6..f45dbca240 100644
--- a/src/fides/api/ops/service/connectors/fides/fides_client.py
+++ b/src/fides/api/ops/service/connectors/fides/fides_client.py
@@ -20,11 +20,8 @@
)
from fides.api.ops.util.collection_util import Row
from fides.api.ops.util.wrappers import sync
-from fides.core.config import get_config
from fides.lib.oauth.schemas.user import UserLogin
-CONFIG = get_config()
-
COMPLETION_STATUSES = [
PrivacyRequestStatus.complete,
PrivacyRequestStatus.canceled,
diff --git a/src/fides/api/ops/service/connectors/saas/authenticated_client.py b/src/fides/api/ops/service/connectors/saas/authenticated_client.py
index 2837048298..ac87848ce7 100644
--- a/src/fides/api/ops/service/connectors/saas/authenticated_client.py
+++ b/src/fides/api/ops/service/connectors/saas/authenticated_client.py
@@ -20,7 +20,7 @@
RateLimiterPeriod,
RateLimiterRequest,
)
-from fides.core.config import get_config
+from fides.core.config import CONFIG
if TYPE_CHECKING:
from fides.api.ops.models.connectionconfig import ConnectionConfig
@@ -29,9 +29,6 @@
from fides.api.ops.schemas.saas.shared_schemas import SaaSRequestParams
-CONFIG = get_config()
-
-
class AuthenticatedClient:
"""
A helper class to build authenticated HTTP requests based on
diff --git a/src/fides/api/ops/service/connectors/saas_query_config.py b/src/fides/api/ops/service/connectors/saas_query_config.py
index 38809b53ec..1924427012 100644
--- a/src/fides/api/ops/service/connectors/saas_query_config.py
+++ b/src/fides/api/ops/service/connectors/saas_query_config.py
@@ -26,9 +26,7 @@
get_identity,
unflatten_dict,
)
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
T = TypeVar("T")
diff --git a/src/fides/api/ops/service/masking/strategy/masking_strategy_hash.py b/src/fides/api/ops/service/masking/strategy/masking_strategy_hash.py
index aef14b1a1e..db2f9c5219 100644
--- a/src/fides/api/ops/service/masking/strategy/masking_strategy_hash.py
+++ b/src/fides/api/ops/service/masking/strategy/masking_strategy_hash.py
@@ -18,9 +18,7 @@
)
from fides.api.ops.service.masking.strategy.masking_strategy import MaskingStrategy
from fides.api.ops.util.encryption.secrets_util import SecretsUtil
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
class HashMaskingStrategy(MaskingStrategy):
diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py
index aa003a4696..0e7372eff9 100644
--- a/src/fides/api/ops/service/messaging/message_dispatch_service.py
+++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py
@@ -41,12 +41,8 @@
from fides.api.ops.schemas.redis_cache import Identity
from fides.api.ops.tasks import MESSAGING_QUEUE_NAME, DatabaseTask, celery_app
from fides.api.ops.util.logger import Pii
-from fides.core.config import get_config
from fides.core.config.config_proxy import ConfigProxy
-CONFIG = get_config()
-
-
EMAIL_JOIN_STRING = ", "
diff --git a/src/fides/api/ops/service/privacy_request/consent_email_batch_service.py b/src/fides/api/ops/service/privacy_request/consent_email_batch_service.py
index 98af5dc676..b8bed1d925 100644
--- a/src/fides/api/ops/service/privacy_request/consent_email_batch_service.py
+++ b/src/fides/api/ops/service/privacy_request/consent_email_batch_service.py
@@ -244,7 +244,7 @@ def requeue_privacy_requests_after_consent_email_send(
def initiate_scheduled_batch_consent_email_send() -> None:
"""Initiates scheduler to add weekly batch consent email send"""
- if CONFIG.is_test_mode:
+ if CONFIG.test_mode:
return
logger.info("Initiating scheduler for batch consent email send")
diff --git a/src/fides/api/ops/service/privacy_request/request_runner_service.py b/src/fides/api/ops/service/privacy_request/request_runner_service.py
index cde6490519..b325841581 100644
--- a/src/fides/api/ops/service/privacy_request/request_runner_service.py
+++ b/src/fides/api/ops/service/privacy_request/request_runner_service.py
@@ -82,14 +82,12 @@
from fides.api.ops.util.collection_util import Row
from fides.api.ops.util.logger import Pii, _log_exception, _log_warning
from fides.api.ops.util.wrappers import sync
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.core.config.config_proxy import ConfigProxy
from fides.lib.db.session import get_db_session
from fides.lib.models.audit_log import AuditLog, AuditLogAction
from fides.lib.schemas.base_class import BaseSchema
-CONFIG = get_config()
-
class ManualWebhookResults(BaseSchema):
"""Represents manual webhook data retrieved from the cache and whether privacy request execution should continue"""
diff --git a/src/fides/api/ops/service/privacy_request/request_service.py b/src/fides/api/ops/service/privacy_request/request_service.py
index 33fbda00cb..7022c33e41 100644
--- a/src/fides/api/ops/service/privacy_request/request_service.py
+++ b/src/fides/api/ops/service/privacy_request/request_service.py
@@ -16,9 +16,6 @@
from fides.api.ops.schemas.privacy_request import PrivacyRequestResponse
from fides.api.ops.schemas.redis_cache import Identity
from fides.api.ops.service.masking.strategy.masking_strategy import MaskingStrategy
-from fides.core.config import get_config
-
-CONFIG = get_config()
def build_required_privacy_request_kwargs(
diff --git a/src/fides/api/ops/service/saas_request/override_implementations/friendbuy_nextgen_request_overrides.py b/src/fides/api/ops/service/saas_request/override_implementations/friendbuy_nextgen_request_overrides.py
index 18383b2595..477b9bd3e3 100644
--- a/src/fides/api/ops/service/saas_request/override_implementations/friendbuy_nextgen_request_overrides.py
+++ b/src/fides/api/ops/service/saas_request/override_implementations/friendbuy_nextgen_request_overrides.py
@@ -11,9 +11,7 @@
SaaSRequestType,
register,
)
-from fides.core.config import get_config
-CONFIG = get_config()
logger = logging.getLogger(__name__)
diff --git a/src/fides/api/ops/task/graph_task.py b/src/fides/api/ops/task/graph_task.py
index 93ac508e01..32bd6ae945 100644
--- a/src/fides/api/ops/task/graph_task.py
+++ b/src/fides/api/ops/task/graph_task.py
@@ -7,6 +7,7 @@
import dask
from dask import delayed # type: ignore[attr-defined]
+from dask.core import getcycle
from dask.threaded import get
from loguru import logger
from sqlalchemy.orm import Session
@@ -16,6 +17,7 @@
NotSupportedForCollection,
PrivacyRequestErasureEmailSendRequired,
PrivacyRequestPaused,
+ TraversalError,
)
from fides.api.ops.graph.analytics_events import (
fideslog_graph_rerun,
@@ -48,12 +50,12 @@
from fides.api.ops.util.collection_util import NodeInput, Row, append, partition
from fides.api.ops.util.logger import Pii
from fides.api.ops.util.saas_util import FIDESOPS_GROUPED_INPUTS
-from fides.core.config import get_config
+from fides.core.config import CONFIG
dask.config.set(scheduler="threads")
COLLECTION_FIELD_PATH_MAP = Dict[CollectionAddress, List[Tuple[FieldPath, FieldPath]]]
-CONFIG = get_config()
+
EMPTY_REQUEST = PrivacyRequest()
@@ -237,7 +239,7 @@ def _combine_seed_data(
seed_index = self.input_keys.index(ROOT_COLLECTION_ADDRESS)
seed_data = data[seed_index]
- for (foreign_field_path, local_field_path) in dependent_field_mappings[
+ for foreign_field_path, local_field_path in dependent_field_mappings[
ROOT_COLLECTION_ADDRESS
]:
dependent_values = consolidate_query_matches(
@@ -511,7 +513,12 @@ def access_request(self, *inputs: List[Row]) -> List[Row]:
return filtered_output
@retry(action_type=ActionType.erasure, default_return=0)
- def erasure_request(self, retrieved_data: List[Row], *inputs: List[Row]) -> int:
+ def erasure_request(
+ self,
+ retrieved_data: List[Row],
+ inputs: List[List[Row]],
+ *erasure_prereqs: int,
+ ) -> int:
"""Run erasure request"""
# if there is no primary key specified in the graph node configuration
# note this in the execution log and perform no erasures on this node
@@ -526,6 +533,8 @@ def erasure_request(self, retrieved_data: List[Row], *inputs: List[Row]) -> int:
ActionType.erasure,
ExecutionLogStatus.complete,
)
+ # Cache that the erasure was performed in case we need to restart
+ self.resources.cache_erasure(self.key.value, 0)
return 0
if not self.can_write_data():
@@ -540,6 +549,7 @@ def erasure_request(self, retrieved_data: List[Row], *inputs: List[Row]) -> int:
ActionType.erasure,
ExecutionLogStatus.error,
)
+ self.resources.cache_erasure(self.key.value, 0)
return 0
formatted_input_data: NodeInput = self.pre_process_input_data(
@@ -707,9 +717,7 @@ def get_cached_data_for_erasures(
def update_erasure_mapping_from_cache(
- dsk: Dict[CollectionAddress, Tuple[Any, ...]],
- resources: TaskResources,
- start_fn: Callable,
+ dsk: Dict[CollectionAddress, Union[Tuple[Any, ...], int]], resources: TaskResources
) -> None:
"""On pause or restart from failure, update the dsk graph to skip running erasures on collections
we've already visited. Instead, just return the previous count of rows affected.
@@ -719,9 +727,9 @@ def update_erasure_mapping_from_cache(
cached_erasures: Dict[str, int] = resources.get_all_cached_erasures()
for collection_name in cached_erasures:
- dsk[CollectionAddress.from_string(collection_name)] = (
- start_fn(cached_erasures[collection_name]),
- )
+ dsk[CollectionAddress.from_string(collection_name)] = cached_erasures[
+ collection_name
+ ]
async def run_erasure( # pylint: disable = too-many-arguments
@@ -746,16 +754,19 @@ def collect_tasks_fn(
if not tn.is_root_node():
data[tn.address] = GraphTask(tn, resources)
+ # We store the end nodes from the traversal for analytics purposes
+ # but we generate a separate erasure_end_nodes list for the actual erasure traversal
env: Dict[CollectionAddress, Any] = {}
- end_nodes = traversal.traverse(env, collect_tasks_fn)
+ access_end_nodes = traversal.traverse(env, collect_tasks_fn)
+ erasure_end_nodes = list(graph.nodes.keys())
- def termination_fn(*dependent_values: int) -> Tuple[int, ...]:
- """The dependent_values here is an int output from each task feeding in, where
- each task reports the output of 'task.rtf(access_request_data)', which is the number of
- records updated.
-
- The termination function just returns this tuple of ints."""
- return dependent_values
+ def termination_fn(*dependent_values: int) -> Dict[str, int]:
+ """
+ The erasure order can be affected in a way that not every node is directly linked
+ to the termination node. This means that we can't just aggregate the inputs directly,
+ we must read the erasure results from the cache.
+ """
+ return resources.get_all_cached_erasures()
access_request_data[ROOT_COLLECTION_ADDRESS.value] = [identity]
@@ -765,34 +776,57 @@ def termination_fn(*dependent_values: int) -> Tuple[int, ...]:
access_request_data.get(
str(k), []
), # Pass in the results of the access request for this collection
- *[
+ [
access_request_data.get(
str(upstream_key), []
) # Additionally pass in the original input data we used for the access request. It's helpful in
# cases like the EmailConnector where the access request doesn't actually retrieve data.
for upstream_key in t.input_keys
],
+ *_evaluate_erasure_dependencies(t, erasure_end_nodes),
)
for k, t in env.items()
}
- # terminator function waits for all keys
- dsk[TERMINATOR_ADDRESS] = (termination_fn, *env.keys())
- update_erasure_mapping_from_cache(dsk, resources, start_function)
+
+ # root node returns 0 to be consistent with the output of the other erasure tasks
+ dsk[ROOT_COLLECTION_ADDRESS] = 0
+ # terminator function reads and returns the cached erasure results for the entire erasure traversal
+ dsk[TERMINATOR_ADDRESS] = (termination_fn, *erasure_end_nodes)
+ update_erasure_mapping_from_cache(dsk, resources)
await fideslog_graph_rerun(
prepare_rerun_graph_analytics_event(
- privacy_request, env, end_nodes, resources, ActionType.erasure
+ privacy_request, env, access_end_nodes, resources, ActionType.erasure
)
)
+
+ # using an existing function from dask.core to detect cycles in the generated graph
+ collection_cycle = getcycle(dsk, None)
+ if collection_cycle:
+ raise TraversalError(
+ f"The values for the `erase_after` fields caused a cycle in the following collections {collection_cycle}"
+ )
+
v = delayed(get(dsk, TERMINATOR_ADDRESS, num_workers=1))
+ return v.compute()
- update_cts: Tuple[int, ...] = v.compute()
- # we combine the output of the termination function with the input keys to provide
- # a map of {collection_name: records_updated}:
- erasure_update_map: Dict[str, int] = dict(
- zip([str(x) for x in env], update_cts)
- )
- return erasure_update_map
+def _evaluate_erasure_dependencies(
+ t: GraphTask, end_nodes: List[CollectionAddress]
+) -> Set[CollectionAddress]:
+ """
+ Return a set of collection addresses corresponding to collections that need
+ to be erased before the given task. Remove the dependent collection addresses
+ from `end_nodes` so they can be executed in the correct order. If a task does
+ not have any dependencies it is linked directly to the root node
+ """
+ erase_after = t.traversal_node.node.collection.erase_after
+ for collection in erase_after:
+ if collection in end_nodes:
+ # end_node list is modified in place
+ end_nodes.remove(collection)
+ # this task will execute after the collections in `erase_after` or
+ # execute at the beginning by linking it to the root node
+ return erase_after if len(erase_after) else {ROOT_COLLECTION_ADDRESS}
async def run_consent_request( # pylint: disable = too-many-arguments
diff --git a/src/fides/api/ops/tasks/__init__.py b/src/fides/api/ops/tasks/__init__.py
index db86edca95..2996ac359b 100644
--- a/src/fides/api/ops/tasks/__init__.py
+++ b/src/fides/api/ops/tasks/__init__.py
@@ -4,10 +4,9 @@
from loguru import logger
from sqlalchemy.orm import Session
-from fides.core.config import FidesConfig, get_config
+from fides.core.config import CONFIG, FidesConfig
from fides.lib.db.session import get_db_engine, get_db_session
-CONFIG = get_config()
MESSAGING_QUEUE_NAME = "fidesops.messaging"
@@ -41,7 +40,7 @@ def get_new_session(self) -> ContextManager[Session]:
return self._sessionmaker()
-def _create_celery(config: FidesConfig = get_config()) -> Celery:
+def _create_celery(config: FidesConfig = CONFIG) -> Celery:
"""
Returns a configured version of the Celery application
"""
@@ -76,7 +75,7 @@ def _create_celery(config: FidesConfig = get_config()) -> Celery:
return app
-celery_app = _create_celery()
+celery_app = _create_celery(CONFIG)
def get_worker_ids() -> List[Optional[str]]:
diff --git a/src/fides/api/ops/tasks/storage.py b/src/fides/api/ops/tasks/storage.py
index ea458a8191..520902e6ac 100644
--- a/src/fides/api/ops/tasks/storage.py
+++ b/src/fides/api/ops/tasks/storage.py
@@ -23,10 +23,9 @@
encrypt_to_bytes_verify_secrets_length,
)
from fides.api.ops.util.storage_authenticator import get_s3_session
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.cryptographic_util import bytes_to_b64_str
-CONFIG = get_config()
LOCAL_FIDES_UPLOAD_DIRECTORY = "fides_uploads"
diff --git a/src/fides/api/ops/util/cache.py b/src/fides/api/ops/util/cache.py
index 3d2b8f4e46..63be1bdafb 100644
--- a/src/fides/api/ops/util/cache.py
+++ b/src/fides/api/ops/util/cache.py
@@ -12,9 +12,7 @@
from fides.api.ops import common_exceptions
from fides.api.ops.schemas.masking.masking_secrets import SecretType
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
# This constant represents every type a redis key may contain, and can be
# extended if needed
@@ -172,6 +170,7 @@ def get_cache() -> FidesopsRedis:
host=CONFIG.redis.host,
port=CONFIG.redis.port,
db=CONFIG.redis.db_index,
+ username=CONFIG.redis.user,
password=CONFIG.redis.password,
ssl=CONFIG.redis.ssl,
ssl_cert_reqs=CONFIG.redis.ssl_cert_reqs,
diff --git a/src/fides/api/ops/util/encryption/aes_gcm_encryption_scheme.py b/src/fides/api/ops/util/encryption/aes_gcm_encryption_scheme.py
index 125ee93313..d122fcf8c0 100644
--- a/src/fides/api/ops/util/encryption/aes_gcm_encryption_scheme.py
+++ b/src/fides/api/ops/util/encryption/aes_gcm_encryption_scheme.py
@@ -3,11 +3,9 @@
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.cryptographic_util import bytes_to_b64_str
-CONFIG = get_config()
-
def encrypt_to_bytes_verify_secrets_length(
plain_value: Optional[str], key: bytes, nonce: bytes
diff --git a/src/fides/api/ops/util/encryption/hmac_encryption_scheme.py b/src/fides/api/ops/util/encryption/hmac_encryption_scheme.py
index a3e430bdeb..0eda2862a0 100644
--- a/src/fides/api/ops/util/encryption/hmac_encryption_scheme.py
+++ b/src/fides/api/ops/util/encryption/hmac_encryption_scheme.py
@@ -3,9 +3,7 @@
from typing import Callable
from fides.api.ops.schemas.masking.masking_configuration import HmacMaskingConfiguration
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
def hmac_encrypt_return_bytes(
diff --git a/src/fides/api/ops/util/identity_verification.py b/src/fides/api/ops/util/identity_verification.py
index 1a81a94487..d54825506a 100644
--- a/src/fides/api/ops/util/identity_verification.py
+++ b/src/fides/api/ops/util/identity_verification.py
@@ -4,9 +4,7 @@
from fides.api.ops.common_exceptions import IdentityVerificationException
from fides.api.ops.util.cache import FidesopsRedis, get_cache
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
class IdentityVerificationMixin:
diff --git a/src/fides/api/ops/util/logger.py b/src/fides/api/ops/util/logger.py
index ee591c2d3c..0481570d43 100644
--- a/src/fides/api/ops/util/logger.py
+++ b/src/fides/api/ops/util/logger.py
@@ -2,12 +2,10 @@
from loguru import logger
-from fides.core.config import get_config
+from fides.core.config import CONFIG
MASKED = "MASKED"
-CONFIG = get_config()
-
class Pii(str):
"""Mask pii data"""
diff --git a/src/fides/api/ops/util/oauth_util.py b/src/fides/api/ops/util/oauth_util.py
index 893a797d05..2ab38aae97 100644
--- a/src/fides/api/ops/util/oauth_util.py
+++ b/src/fides/api/ops/util/oauth_util.py
@@ -20,7 +20,7 @@
from fides.api.ops.api.v1.urn_registry import TOKEN, V1_URL_PREFIX
from fides.api.ops.models.policy import PolicyPreWebhook
from fides.api.ops.schemas.external_https import WebhookJWE
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.schemas.jwt import (
JWE_ISSUED_AT,
JWE_PAYLOAD_CLIENT_ID,
@@ -32,7 +32,6 @@
from fides.lib.oauth.oauth_util import extract_payload, is_token_expired
from fides.lib.oauth.schemas.oauth import OAuth2ClientCredentialsBearer
-CONFIG = get_config()
JWT_ENCRYPTION_ALGORITHM = ALGORITHMS.A256GCM
diff --git a/src/fides/api/ops/util/saas_util.py b/src/fides/api/ops/util/saas_util.py
index a9bf4fad80..b414e45a44 100644
--- a/src/fides/api/ops/util/saas_util.py
+++ b/src/fides/api/ops/util/saas_util.py
@@ -105,7 +105,7 @@ def get_collection_grouped_inputs(
def get_collection_after(
collections: List[Collection], name: str
) -> Set[CollectionAddress]:
- """If specified, return the collections that need to run before the current collection for saas configs"""
+ """If specified, return the collections that need to be read before the current collection for saas configs"""
collection: Collection | None = next(
(collect for collect in collections if collect.name == name), None
)
@@ -114,6 +114,18 @@ def get_collection_after(
return collection.after
+def get_collection_erase_after(
+ collections: List[Collection], name: str
+) -> Set[CollectionAddress]:
+ """If specified, return the collections that need to be erased before the current collection for saas configs"""
+ collection: Collection | None = next(
+ (collect for collect in collections if collect.name == name), None
+ )
+ if not collection:
+ return set()
+ return collection.erase_after
+
+
def merge_datasets(dataset: GraphDataset, config_dataset: GraphDataset) -> GraphDataset:
"""
Merges all Collections and Fields from the "config_dataset" into the "dataset".
@@ -135,6 +147,9 @@ def merge_datasets(dataset: GraphDataset, config_dataset: GraphDataset) -> Graph
config_dataset.collections, collection_name
),
after=get_collection_after(config_dataset.collections, collection_name),
+ erase_after=get_collection_erase_after(
+ config_dataset.collections, collection_name
+ ),
)
)
diff --git a/src/fides/core/config/__init__.py b/src/fides/core/config/__init__.py
index 1ee61f2928..835bc8c87f 100644
--- a/src/fides/core/config/__init__.py
+++ b/src/fides/core/config/__init__.py
@@ -9,6 +9,7 @@
import toml
from loguru import logger as log
+from pydantic import Field
from pydantic.class_validators import _FUNCS
from pydantic.env_settings import SettingsSourceCallable
@@ -41,11 +42,26 @@ class FidesConfig(FidesSettings):
"""
# Root Settings
- test_mode: bool = get_test_mode()
- is_test_mode: bool = test_mode
- hot_reloading: bool = getenv("FIDES__HOT_RELOAD", "").lower() == "true"
- dev_mode: bool = getenv("FIDES__DEV_MODE", "").lower() == "true"
- oauth_instance: Optional[str] = getenv("FIDES__OAUTH_INSTANCE")
+ test_mode: bool = Field(
+ default=get_test_mode(),
+ description="Whether or not the application is being run in test mode.",
+ exclude=True,
+ )
+ hot_reloading: bool = Field(
+ default=getenv("FIDES__HOT_RELOAD", "").lower() == "true",
+ description="Whether or not to enable hot reloading for the webserver.",
+ exclude=True,
+ )
+ dev_mode: bool = Field(
+ default=getenv("FIDES__DEV_MODE", "").lower() == "true",
+ description="Similar to 'test_mode', enables certain features when true.",
+ exclude=True,
+ )
+ oauth_instance: Optional[str] = Field(
+ default=None,
+ description="A value that is prepended to the generated 'state' param in outbound OAuth2 authorization requests. Used during OAuth2 testing to associate callback responses back to this specific Fides instance.",
+ exclude=True,
+ )
# Setting Subsections
# These should match the `settings_map` in `build_config`
@@ -180,14 +196,19 @@ def get_config(config_path_override: str = "", verbose: bool = False) -> FidesCo
try:
settings = toml.load(config_path)
config = build_config(config_dict=settings)
- print(f"Loaded config from: {config_path}")
+ if verbose:
+ print(f"Loaded config from: {config_path}")
return config
except FileNotFoundError:
pass
except IOError:
echo_red(f"Error reading config file: {config_path}")
- print("Using default configuration values.")
+ if verbose: # pragma: no cover
+ print("Using default configuration values.")
config = build_config(config_dict={})
return config
+
+
+CONFIG = get_config()
diff --git a/src/fides/core/config/admin_ui_settings.py b/src/fides/core/config/admin_ui_settings.py
index 6e5f576999..21a5f0d7c0 100644
--- a/src/fides/core/config/admin_ui_settings.py
+++ b/src/fides/core/config/admin_ui_settings.py
@@ -1,10 +1,14 @@
+from pydantic import Field
+
from .fides_settings import FidesSettings
class AdminUISettings(FidesSettings):
"""Configuration settings for Analytics variables."""
- enabled: bool = True
+ enabled: bool = Field(
+ default=True, description="Toggle whether the Admin UI is served."
+ )
class Config:
env_prefix = "FIDES__ADMIN_UI__"
diff --git a/src/fides/core/config/cli_settings.py b/src/fides/core/config/cli_settings.py
index 4e09638c9d..a832b43250 100644
--- a/src/fides/core/config/cli_settings.py
+++ b/src/fides/core/config/cli_settings.py
@@ -2,7 +2,7 @@
from typing import Dict, Optional
from fideslog.sdk.python.utils import FIDESCTL_CLI, generate_client_id
-from pydantic import AnyHttpUrl, validator
+from pydantic import AnyHttpUrl, Field, validator
from .fides_settings import FidesSettings
@@ -13,15 +13,28 @@
class CLISettings(FidesSettings):
"""Class used to store values from the 'cli' section of the config."""
- local_mode: bool = False
- analytics_id: str = generate_client_id(FIDESCTL_CLI)
-
- # These defaults are required to make connecting to
- # docker instances possible by default
- server_protocol: str = "http"
- server_host: str = "localhost"
- server_port: str = "8080"
- server_url: Optional[AnyHttpUrl]
+ analytics_id: str = Field(
+ default=generate_client_id(FIDESCTL_CLI),
+ description="A fully anonymized unique identifier that is automatically generated by the application and stored in the toml file.",
+ )
+ local_mode: bool = Field(
+ default=False,
+ description="When set to True, disables functionality that requires making calls to a Fides webserver.",
+ )
+ server_protocol: str = Field(
+ default="http", description="The protocol used by the Fides webserver."
+ )
+ server_host: str = Field(
+ default="localhost", description="The hostname of the Fides webserver."
+ )
+ server_port: str = Field(
+ default="8080", description="The port of the Fides webserver"
+ )
+ server_url: Optional[AnyHttpUrl] = Field(
+ default=None,
+ description="The full server url generated from the other server configuration values.",
+ exclude=True,
+ )
@validator("server_url", always=True)
@classmethod
diff --git a/src/fides/core/config/database_settings.py b/src/fides/core/config/database_settings.py
index 6c5e2bd414..d25e4969b6 100644
--- a/src/fides/core/config/database_settings.py
+++ b/src/fides/core/config/database_settings.py
@@ -4,7 +4,7 @@
from typing import Dict, Optional
-from pydantic import PostgresDsn, validator
+from pydantic import Field, PostgresDsn, validator
from fides.core.config.utils import get_test_mode
@@ -16,26 +16,68 @@
class DatabaseSettings(FidesSettings):
"""Configuration settings for Postgres."""
- user: str = "defaultuser"
- password: str = "defaultpassword"
- server: str = "default-db"
- port: str = "5432"
- db: str = "default_db"
- test_db: str = "default_test_db"
-
- api_engine_pool_size: int = 50
- api_engine_max_overflow: int = 50
- task_engine_pool_size: int = 50
- task_engine_max_overflow: int = 50
-
- sqlalchemy_database_uri: Optional[str] = None
- sqlalchemy_test_database_uri: Optional[str] = None
-
- # These values are set by validators, and are never empty strings within the
- # application. The default values here are required in order to prevent the
- # types being set to "Optional[str]", as they are not functionally optional.
- async_database_uri: str = ""
- sync_database_uri: str = ""
+ api_engine_pool_size: int = Field(
+ default=50,
+ description="Number of concurrent database connections Fides will use for API requests. Note that the pool begins with no connections, but as they are requested the connections are maintained and reused up to this limit.",
+ )
+ api_engine_max_overflow: int = Field(
+ default=50,
+ description="Number of additional 'overflow' concurrent database connections Fides will use for API requests if the pool reaches the limit. These overflow connections are discarded afterwards and not maintained.",
+ )
+ db: str = Field(
+ default="default_db", description="The name of the application database."
+ )
+ password: str = Field(
+ default="defaultpassword",
+ description="The password with which to login to the application database.",
+ )
+ port: str = Field(
+ default="5432",
+ description="The port at which the application database will be accessible.",
+ )
+ server: str = Field(
+ default="default-db",
+ description="The hostname of the application database server.",
+ )
+ task_engine_pool_size: int = Field(
+ default=50,
+ description="Number of concurrent database connections Fides will use for executing privacy request tasks, either locally or on each worker. Note that the pool begins with no connections, but as they are requested the connections are maintained and reused up to this limit.",
+ )
+ task_engine_max_overflow: int = Field(
+ default=50,
+ description="Number of additional 'overflow' concurrent database connections Fides will use for executing privacy request tasks, either locally or on each worker, if the pool reaches the limit. These overflow connections are discarded afterwards and not maintained.",
+ )
+ test_db: str = Field(
+ default="default_test_db",
+ description="Used instead of the 'db' value when the FIDES_TEST_MODE environment variable is set to True. Avoids overwriting production data.",
+ exclude=True,
+ )
+ user: str = Field(
+ default="defaultuser",
+ description="The database user with which to login to the application database.",
+ )
+
+ # These must be at the end because they require other values to construct
+ sqlalchemy_database_uri: str = Field(
+ default="",
+ description="Programmatically created connection string for the application database.",
+ exclude=True,
+ )
+ sqlalchemy_test_database_uri: str = Field(
+ default="",
+ description="Programmatically created connection string for the test database.",
+ exclude=True,
+ )
+ async_database_uri: str = Field(
+ default="",
+ description="Programmatically created asynchronous connection string for the configured database (either application or test).",
+ exclude=True,
+ )
+ sync_database_uri: str = Field(
+ default="",
+ description="Programmatically created synchronous connection string for the configured database (either application or test).",
+ exclude=True,
+ )
@validator("sync_database_uri", pre=True)
@classmethod
@@ -43,9 +85,8 @@ def assemble_sync_database_uri(
cls, value: Optional[str], values: Dict[str, str]
) -> str:
"""Join DB connection credentials into a connection string"""
- if isinstance(value, str) and value != "":
- # This validates that the string is a valid PostgresDns.
- return str(PostgresDsn(value))
+ if isinstance(value, str) and value:
+ return value
db_name = values["test_db"] if get_test_mode() else values["db"]
return str(
@@ -64,10 +105,9 @@ def assemble_sync_database_uri(
def assemble_async_database_uri(
cls, value: Optional[str], values: Dict[str, str]
) -> str:
- """Join DB connection credentials into a connection string"""
- if isinstance(value, str) and value != "":
- # This validates that the string is a valid PostgresDns.
- return str(PostgresDsn(value))
+ """Join DB connection credentials into an async connection string."""
+ if isinstance(value, str) and value:
+ return value
db_name = values["test_db"] if get_test_mode() else values["db"]
return str(
@@ -84,8 +124,8 @@ def assemble_async_database_uri(
@validator("sqlalchemy_database_uri", pre=True)
@classmethod
def assemble_db_connection(cls, v: Optional[str], values: Dict[str, str]) -> str:
- """Join DB connection credentials into a connection string"""
- if isinstance(v, str):
+ """Join DB connection credentials into a synchronous connection string."""
+ if isinstance(v, str) and v:
return v
return str(
PostgresDsn.build(
@@ -104,7 +144,7 @@ def assemble_test_db_connection(
cls, v: Optional[str], values: Dict[str, str]
) -> str:
"""Join DB connection credentials into a connection string"""
- if isinstance(v, str):
+ if isinstance(v, str) and v:
return v
return str(
PostgresDsn.build(
diff --git a/src/fides/core/config/execution_settings.py b/src/fides/core/config/execution_settings.py
index caae351727..a6a1fe1b5c 100644
--- a/src/fides/core/config/execution_settings.py
+++ b/src/fides/core/config/execution_settings.py
@@ -1,3 +1,5 @@
+from pydantic import Field
+
from .fides_settings import FidesSettings
ENV_PREFIX = "FIDES__EXECUTION__"
@@ -6,13 +8,32 @@
class ExecutionSettings(FidesSettings):
"""Configuration settings for execution."""
- privacy_request_delay_timeout: int = 3600
- task_retry_count: int = 0
- task_retry_delay: int = 1 # In seconds
- task_retry_backoff: int = 1
- subject_identity_verification_required: bool = False
- require_manual_request_approval: bool = False
- masking_strict: bool = True
+ masking_strict: bool = Field(
+ default=True,
+ description="If set to True, only use UPDATE requests to mask data. If False, Fides will use any defined DELETE or GDPR DELETE endpoints to remove PII, which may extend beyond the specific data categories that configured in your execution policy.",
+ )
+ privacy_request_delay_timeout: int = Field(
+ default=3600,
+ description="The amount of time to wait for actions which delay privacy requests (e.g., pre- and post-processing webhooks).",
+ )
+ require_manual_request_approval: bool = Field(
+ default=False,
+ description="Whether privacy requests require explicit approval to execute.",
+ )
+ subject_identity_verification_required: bool = Field(
+ default=False,
+ description="Whether privacy requests require user identity verification.",
+ )
+ task_retry_backoff: int = Field(
+ default=1,
+ description="The backoff factor for retries, to space out repeated retries.",
+ )
+ task_retry_count: int = Field(
+ default=0, description="The number of times a failed request will be retried."
+ )
+ task_retry_delay: int = Field(
+ default=1, description="The delays between retries in seconds."
+ )
class Config:
env_prefix = ENV_PREFIX
diff --git a/src/fides/core/config/helpers.py b/src/fides/core/config/helpers.py
index 6d18e389e6..a23f8d9a40 100644
--- a/src/fides/core/config/helpers.py
+++ b/src/fides/core/config/helpers.py
@@ -111,28 +111,11 @@ def create_config_file(
Returns the config_path if successful.
"""
- # TODO: These important constants should live elsewhere
fides_dir_name = ".fides"
fides_dir_path = f"{fides_directory_location}/{fides_dir_name}"
config_file_name = "fides.toml"
config_path = f"{fides_dir_path}/{config_file_name}"
- included_values = {
- "database": {
- "server",
- "user",
- "password",
- "port",
- "db",
- },
- "logging": {
- "level",
- "destination",
- "serialization",
- },
- "cli": {"server_protocol", "server_host", "server_port"},
- }
-
# create the .fides dir if it doesn't exist
if not os.path.exists(fides_dir_path):
os.mkdir(fides_dir_path)
@@ -143,7 +126,7 @@ def create_config_file(
# create a fides.toml config file if it doesn't exist
if not os.path.isfile(config_path):
with open(config_path, "w", encoding="utf-8") as config_file:
- config_dict = config.dict(include=included_values) # type: ignore[arg-type]
+ config_dict = config.dict() # type: ignore[arg-type]
toml.dump(config_dict, config_file)
echo(f"Created a fides config file: {config_path}")
else:
diff --git a/src/fides/core/config/logging_settings.py b/src/fides/core/config/logging_settings.py
index a8300880ab..122fc0410f 100644
--- a/src/fides/core/config/logging_settings.py
+++ b/src/fides/core/config/logging_settings.py
@@ -4,7 +4,7 @@
import os
from logging import CRITICAL, DEBUG, ERROR, INFO, WARNING, getLevelName
-from pydantic import validator
+from pydantic import Field, validator
from .fides_settings import FidesSettings
from .utils import get_dev_mode
@@ -16,10 +16,22 @@ class LoggingSettings(FidesSettings):
"""Class used to store values from the 'logging' section of the config."""
# Logging
- destination: str = ""
- level: str = "INFO"
- serialization: str = ""
- log_pii: bool = False
+ destination: str = Field(
+ default="",
+ description="The output location for log files. Accepts any valid file path. If left unset, log entries are printed to stdout and log files are not produced.",
+ )
+ level: str = Field(
+ default="INFO",
+ description="The minimum log entry level to produce. Also accepts TRACE, DEBUG, WARNING, ERROR, or CRITICAL (case insensitive).",
+ )
+ serialization: str = Field(
+ default="",
+ description="The format with which to produce log entries. If left unset, produces log entries formatted using the internal custom formatter. Also accepts 'JSON' (case insensitive).",
+ )
+ log_pii: bool = Field(
+ default=False,
+ description="If True, PII values will display unmasked in log output. This variable should always be set to 'False' in production systems.",
+ )
@validator("destination", pre=True)
@classmethod
diff --git a/src/fides/core/config/notification_settings.py b/src/fides/core/config/notification_settings.py
index 334b764b1e..e7c6e00443 100644
--- a/src/fides/core/config/notification_settings.py
+++ b/src/fides/core/config/notification_settings.py
@@ -1,6 +1,6 @@
from typing import Optional
-from pydantic import validator
+from pydantic import Field, validator
from .fides_settings import FidesSettings
@@ -10,10 +10,22 @@
class NotificationSettings(FidesSettings):
"""Configuration settings for data subject and/or data processor notifications"""
- send_request_completion_notification: bool = False
- send_request_receipt_notification: bool = False
- send_request_review_notification: bool = False
- notification_service_type: Optional[str] = None
+ notification_service_type: Optional[str] = Field(
+ default=None,
+ description="Sets the notification service type used to send notifications. Accepts mailgun, twilio_sms, or twilio_email.",
+ )
+ send_request_completion_notification: bool = Field(
+ default=False,
+ description="When set to True, enables subject notifications upon privacy request completion.",
+ )
+ send_request_receipt_notification: bool = Field(
+ default=False,
+ description="When set to True, enables subject notifications upon privacy request receipt.",
+ )
+ send_request_review_notification: bool = Field(
+ default=False,
+ description="When set to True, enables subject notifications upon privacy request review.",
+ )
@validator("notification_service_type", pre=True)
@classmethod
diff --git a/src/fides/core/config/redis_settings.py b/src/fides/core/config/redis_settings.py
index be89a97613..9c715286c9 100644
--- a/src/fides/core/config/redis_settings.py
+++ b/src/fides/core/config/redis_settings.py
@@ -1,7 +1,7 @@
from typing import Dict, Optional
from urllib.parse import quote_plus
-from pydantic import validator
+from pydantic import Field, validator
from .fides_settings import FidesSettings
@@ -11,19 +11,57 @@
class RedisSettings(FidesSettings):
"""Configuration settings for Redis."""
- host: str = "redis"
- port: int = 6379
- user: Optional[str] = ""
- password: str = "testpassword"
- charset: str = "utf8"
- decode_responses: bool = True
- default_ttl_seconds: int = 604800
- identity_verification_code_ttl_seconds: int = 600
- db_index: Optional[int]
- enabled: bool = False
- ssl: bool = False
- ssl_cert_reqs: Optional[str] = "required"
- connection_url: Optional[str] = None
+ charset: str = Field(
+ default="utf8",
+ description="Character set to use for Redis, defaults to 'utf8'. Not recommended to change.",
+ )
+ db_index: Optional[int] = Field(
+ default=None,
+ description="The application will use this index in the Redis cache to cache data.",
+ )
+ decode_responses: bool = Field(default=True, description="TODO")
+ default_ttl_seconds: int = Field(
+ default=604800,
+ description="The number of seconds for which data will live in Redis before automatically expiring.",
+ )
+ enabled: bool = Field(
+ default=False,
+ description="Whether the application's Redis cache should be enabled. Only set to false for certain narrow uses of the application.",
+ )
+ host: str = Field(
+ default="redis",
+ description="The network address for the application Redis cache.",
+ )
+ identity_verification_code_ttl_seconds: int = Field(
+ default=600,
+ description="Sets TTL for cached identity verification code as part of subject requests.",
+ )
+ password: str = Field(
+ default="testpassword",
+ description="The password with which to login to the Redis cache.",
+ )
+ port: int = Field(
+ default=6379,
+ description="The port at which the application cache will be accessible.",
+ )
+ ssl: bool = Field(
+ default=False,
+ description="Whether the application's connections to the cache should be encrypted using TLS.",
+ )
+ ssl_cert_reqs: Optional[str] = Field(
+ default="required",
+ description="If using TLS encryption, set this to 'required' if you wish to enforce the Redis cache to provide a certificate. Note that not all cache providers support this e.g. AWS Elasticache.",
+ )
+ user: str = Field(
+ default="", description="The user with which to login to the Redis cache."
+ )
+
+ # This relies on other values to get built so must be last
+ connection_url: Optional[str] = Field(
+ default=None,
+ description="A full connection URL to the Redis cache. If not specified, this URL is automatically assembled from the host, port, password and db_index specified above.",
+ exclude=True,
+ )
@validator("connection_url", pre=True)
@classmethod
@@ -37,17 +75,30 @@ def assemble_connection_url(
# If the whole URL is provided via the config, preference that
return v
- db_index = values.get("db_index") if values.get("db_index") is not None else ""
connection_protocol = "redis"
params = ""
use_tls = values.get("ssl", False)
+
+ # These vars are intentionally fetched with `or ""` as the default to account
+ # for the edge case where `None` is explicitly set in `values` by Pydantic because
+ # it is not overridden by the config file or an env var
+ user = values.get("user") or ""
+ password = values.get("password") or ""
+ db_index = values.get("db_index") or ""
if use_tls:
# If using TLS update the connection URL format
connection_protocol = "rediss"
cert_reqs = values.get("ssl_cert_reqs", "none")
params = f"?ssl_cert_reqs={quote_plus(cert_reqs)}"
- return f"{connection_protocol}://{quote_plus(values.get('user', ''))}:{quote_plus(values.get('password', ''))}@{values.get('host', '')}:{values.get('port', '')}/{db_index}{params}"
+ # Configure a basic auth prefix if either user or password is provided, e.g.
+ # redis://:@
+ auth_prefix = ""
+ if password or user:
+ auth_prefix = f"{quote_plus(user)}:{quote_plus(password)}@"
+
+ connection_url = f"{connection_protocol}://{auth_prefix}{values.get('host', '')}:{values.get('port', '')}/{db_index}{params}"
+ return connection_url
class Config:
env_prefix = ENV_PREFIX
diff --git a/src/fides/core/config/security_settings.py b/src/fides/core/config/security_settings.py
index 5dc2f09f0a..f288ff3cb8 100644
--- a/src/fides/core/config/security_settings.py
+++ b/src/fides/core/config/security_settings.py
@@ -1,11 +1,10 @@
"""This module handles finding and parsing fides configuration files."""
# pylint: disable=C0115,C0116, E0213
-from enum import Enum
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Pattern, Tuple, Union
import validators
-from pydantic import validator
+from pydantic import Field, validator
from slowapi.wrappers import parse_many # type: ignore
from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY
@@ -16,42 +15,94 @@
ENV_PREFIX = "FIDES__SECURITY__"
-class SecurityEnv(Enum):
- """
- Defines the potential environments that Fides can
- be configured to run in.
- """
-
- DEV = "dev"
- PROD = "prod"
-
-
class SecuritySettings(FidesSettings):
"""Configuration settings for Security variables."""
- root_user_scopes: Optional[List[str]] = SCOPE_REGISTRY
- subject_request_download_link_ttl_seconds: int = 432000 # 5 days
- request_rate_limit: str = "1000/minute"
- rate_limit_prefix: str = "fides-"
- aes_encryption_key_length: int = 16
- aes_gcm_nonce_length: int = 12
- app_encryption_key: str = ""
- drp_jwt_secret: Optional[str] = None
- root_username: Optional[str] = None
- root_password: Optional[str] = None
- parent_server_username: Optional[str] = None
- parent_server_password: Optional[str] = None
- identity_verification_attempt_limit: int = 3 # 3 attempts
- encoding: str = "UTF-8"
- env: SecurityEnv = SecurityEnv.DEV
-
- cors_origins: List[str] = []
- oauth_root_client_id: str = ""
- oauth_root_client_secret: str = ""
- oauth_root_client_secret_hash: Optional[Tuple]
- oauth_access_token_expire_minutes: int = 60 * 24 * 8
- oauth_client_id_length_bytes = 16
- oauth_client_secret_length_bytes = 16
+ aes_encryption_key_length: int = Field(
+ default=16,
+ description="Length of desired encryption key when using Fides to generate a random secure string used for AES encryption.",
+ )
+ aes_gcm_nonce_length: int = Field(
+ default=12,
+ description="Length of desired random byte str for the AES GCM encryption used throughout Fides.",
+ )
+ app_encryption_key: str = Field(
+ default="", description="The key used to sign Fides API access tokens."
+ )
+ cors_origins: Union[str, List[str]] = Field(
+ default=[],
+ description="A list of pre-approved addresses of clients allowed to communicate with the Fides application server.",
+ )
+ cors_origin_regex: Optional[Pattern] = Field(default=None, description="TODO")
+ drp_jwt_secret: Optional[str] = Field(
+ default=None,
+ description="JWT secret by which passed-in identity is decrypted according to the HS256 algorithm.",
+ )
+ encoding: str = Field(
+ default="UTF-8", description="Text encoding to use for the application."
+ )
+ env: str = Field(
+ default="dev",
+ description="The default, `dev`, does not apply authentication to endpoints typically used by the CLI. The other option, `prod`, requires authentication for _all_ endpoints that may contain sensitive information.",
+ )
+ identity_verification_attempt_limit: int = Field(default=3, description="")
+ oauth_root_client_id: str = Field(
+ default="",
+ description="The value used to identify the Fides application root API client.",
+ )
+ oauth_root_client_secret: str = Field(
+ default="",
+ description="The secret value used to authenticate the Fides application root API client.",
+ )
+ oauth_root_client_secret_hash: Optional[Tuple] = Field(
+ default=None,
+ description="Automatically generated by Fides, and represents a hashed value of the oauth_root_client_secret.",
+ )
+ oauth_access_token_expire_minutes: int = Field(
+ default=11520,
+ description="The time in minutes for which Fides API tokens will be valid. Default value is equal to 8 days.",
+ )
+ oauth_client_id_length_bytes: int = Field(
+ default=16,
+ description="Sets desired length in bytes of generated client id used for oauth.",
+ )
+ oauth_client_secret_length_bytes: int = Field(
+ default=16,
+ description="Sets desired length in bytes of generated client secret used for oauth.",
+ )
+ parent_server_password: Optional[str] = Field(
+ default=None,
+ description="When using a parent/child Fides deployment, this password will be used by the child server to access the parent server.",
+ )
+ parent_server_username: Optional[str] = Field(
+ default=None,
+ description="When using a parent/child Fides deployment, this username will be used by the child server to access the parent server.",
+ )
+ rate_limit_prefix: str = Field(
+ default="fides-",
+ description="The prefix given to keys in the Redis cache used by the rate limiter.",
+ )
+ request_rate_limit: str = Field(
+ default="1000/minute",
+ description="The number of requests from a single IP address allowed to hit an endpoint within a rolling 60 second period.",
+ )
+ root_user_scopes: List[str] = Field(
+ default=SCOPE_REGISTRY,
+ description="The list of scopes that are given to the root user.",
+ )
+ root_password: Optional[str] = Field(
+ default=None,
+ description="If set, this can be used in conjunction with root_username to log in without first creating a user in the database.",
+ )
+
+ root_username: Optional[str] = Field(
+ default=None,
+ description="If set, this can be used in conjunction with root_password to log in without first creating a user in the database.",
+ )
+ subject_request_download_link_ttl_seconds: int = Field(
+ default=432000,
+ description="The number of seconds that a pre-signed download URL when using S3 storage will be valid. The default is equal to 5 days.",
+ )
@validator("app_encryption_key")
@classmethod
@@ -135,5 +186,17 @@ def validate_request_rate_limit(
raise ValueError(message)
return v
+ @validator("env")
+ @classmethod
+ def validate_env(
+ cls,
+ v: str,
+ ) -> str:
+ """Validate the formatting of `request_rate_limit`"""
+ if v not in ["dev", "prod"]:
+ message = "Security environment must be either 'dev' or 'prod'."
+ raise ValueError(message)
+ return v
+
class Config:
env_prefix = ENV_PREFIX
diff --git a/src/fides/core/config/user_settings.py b/src/fides/core/config/user_settings.py
index 80735bbf88..67eb5e8fdd 100644
--- a/src/fides/core/config/user_settings.py
+++ b/src/fides/core/config/user_settings.py
@@ -1,9 +1,10 @@
"""This module handles finding and parsing fides configuration files."""
# pylint: disable=C0115,C0116, E0213
-
from typing import Dict, Optional
+from pydantic import Field
+
from fides.core.utils import create_auth_header, get_auth_header
from .fides_settings import FidesSettings
@@ -22,9 +23,18 @@ def try_get_auth_header() -> Dict[str, str]:
class UserSettings(FidesSettings):
"""Class used to store values from the 'user' section of the config."""
- auth_header: Dict[str, str] = try_get_auth_header()
- analytics_opt_out: Optional[bool]
- encryption_key: str = "test_encryption_key"
+ auth_header: Dict[str, str] = Field(
+ default=try_get_auth_header(),
+ description="Authentication header built automatically from the credentials file.",
+ exclude=True,
+ )
+ analytics_opt_out: Optional[bool] = Field(
+ description="When set to true, prevents sending anonymous analytics data to Ethyca."
+ )
+ encryption_key: str = Field(
+ default="test_encryption_key",
+ description="An arbitrary string used to encrypt the user data stored in the database. Encryption is implemented using PGP.",
+ )
class Config:
env_prefix = ENV_PREFIX
diff --git a/src/fides/core/config/utils.py b/src/fides/core/config/utils.py
index 9d5f8b5a03..1a7d59e5a9 100644
--- a/src/fides/core/config/utils.py
+++ b/src/fides/core/config/utils.py
@@ -23,7 +23,6 @@ def get_dev_mode() -> bool:
"user",
"port",
"db",
- "test_db",
"api_engine_pool_size",
"api_engine_max_overflow",
"task_engine_pool_size",
diff --git a/src/fides/core/user.py b/src/fides/core/user.py
index bea1b11446..6446f6112e 100644
--- a/src/fides/core/user.py
+++ b/src/fides/core/user.py
@@ -4,8 +4,8 @@
import requests
+from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY as SCOPES
from fides.cli.utils import handle_cli_response
-from fides.core.config import get_config
from fides.core.utils import (
Credentials,
echo_green,
@@ -16,9 +16,7 @@
write_credentials_file,
)
from fides.lib.cryptography.cryptographic_util import str_to_b64_str
-from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY as SCOPES
-config = get_config()
CREATE_USER_PATH = "/api/v1/user"
LOGIN_PATH = "/api/v1/login"
USER_PERMISSIONS_PATH = "/api/v1/user/{}/permission"
diff --git a/src/fides/data/test_env/fides.test_env.toml b/src/fides/data/test_env/fides.test_env.toml
index f29ced3c8a..855c5a2afb 100644
--- a/src/fides/data/test_env/fides.test_env.toml
+++ b/src/fides/data/test_env/fides.test_env.toml
@@ -8,7 +8,7 @@ db = "fides"
[redis]
host = "redis"
-password = "testpassword"
+password = "redispassword"
port = 6379
db_index = 0
diff --git a/src/fides/lib/db/session.py b/src/fides/lib/db/session.py
index e38ffe8073..8703f06fe7 100644
--- a/src/fides/lib/db/session.py
+++ b/src/fides/lib/db/session.py
@@ -22,12 +22,12 @@ def get_db_engine(
If the TESTING environment var is set the database engine returned will be
connected to the test DB.
"""
- if config is None and database_uri is None:
+ if not config and not database_uri:
raise ValueError("Either a config or database_uri is required")
- if database_uri is None and config is not None:
+ if not database_uri and config:
# Don't override any database_uri explicitly passed in
- if config.is_test_mode:
+ if config.test_mode:
database_uri = config.database.sqlalchemy_test_database_uri
else:
database_uri = config.database.sqlalchemy_database_uri
diff --git a/src/fides/lib/models/fides_user_permissions.py b/src/fides/lib/models/fides_user_permissions.py
index b0f0557e47..14ef8013cc 100644
--- a/src/fides/lib/models/fides_user_permissions.py
+++ b/src/fides/lib/models/fides_user_permissions.py
@@ -3,10 +3,10 @@
from sqlalchemy import ARRAY, Column, ForeignKey, String
from sqlalchemy.orm import backref, relationship
+from fides.api.ops.api.v1.scope_registry import PRIVACY_REQUEST_READ
from fides.lib.db.base_class import Base
from fides.lib.models.fides_user import FidesUser
from fides.lib.oauth.privileges import privileges
-from fides.api.ops.api.v1.scope_registry import PRIVACY_REQUEST_READ
class FidesUserPermissions(Base):
diff --git a/src/fides/lib/oauth/api/routes/user_endpoints.py b/src/fides/lib/oauth/api/routes/user_endpoints.py
index 290679d2b3..f3d9d37737 100644
--- a/src/fides/lib/oauth/api/routes/user_endpoints.py
+++ b/src/fides/lib/oauth/api/routes/user_endpoints.py
@@ -17,6 +17,12 @@
HTTP_404_NOT_FOUND,
)
+from fides.api.ops.api.v1.scope_registry import (
+ PRIVACY_REQUEST_READ,
+ USER_CREATE,
+ USER_DELETE,
+ USER_READ,
+)
from fides.api.ops.util.oauth_util import verify_oauth_client
from fides.core.config import FidesConfig, get_config
from fides.lib.models.client import ClientDetail
@@ -32,12 +38,6 @@
UserLoginResponse,
UserResponse,
)
-from fides.api.ops.api.v1.scope_registry import (
- PRIVACY_REQUEST_READ,
- USER_CREATE,
- USER_DELETE,
- USER_READ,
-)
router = APIRouter()
diff --git a/tests/conftest.py b/tests/conftest.py
index 944e7a3a78..1b2c81c32c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2,14 +2,14 @@
from loguru import logger
from sqlalchemy.engine.base import Engine
-from fides.core.config import get_config
+from fides.core.config import CONFIG
@pytest.fixture(scope="session")
def config():
- config = get_config()
- config.is_test_mode = True
- yield config
+
+ CONFIG.test_mode = True
+ yield CONFIG
@pytest.fixture
diff --git a/tests/ctl/api/test_seed.py b/tests/ctl/api/test_seed.py
index 1a30751545..27f5ee3905 100644
--- a/tests/ctl/api/test_seed.py
+++ b/tests/ctl/api/test_seed.py
@@ -7,11 +7,9 @@
from fides.api.ctl.database import seed
from fides.api.ops.models.policy import ActionType, DrpAction, Policy, Rule, RuleTarget
from fides.core import api as _api
-from fides.core.config import FidesConfig, get_config
+from fides.core.config import CONFIG, FidesConfig
from fides.lib.models.fides_user import FidesUser
-CONFIG = get_config()
-
@pytest.fixture(scope="function", name="data_category")
def fixture_data_category(test_config: FidesConfig) -> Generator:
diff --git a/tests/ctl/cli/test_deploy.py b/tests/ctl/cli/test_deploy.py
index 715155d053..76c1d3472e 100644
--- a/tests/ctl/cli/test_deploy.py
+++ b/tests/ctl/cli/test_deploy.py
@@ -1,6 +1,7 @@
-import pytest
from unittest import mock
+import pytest
+
from fides.core import deploy
diff --git a/tests/ctl/conftest.py b/tests/ctl/conftest.py
index 3c604c3c3a..4acf44fb88 100644
--- a/tests/ctl/conftest.py
+++ b/tests/ctl/conftest.py
@@ -23,8 +23,7 @@
from fides.api.ctl.database.session import sync_engine, sync_session
from fides.api.ctl.sql_models import FidesUser
from fides.core import api
-from fides.core.config import FidesConfig, get_config
-from fides.core.user import login_command
+from fides.core.config import CONFIG, FidesConfig, get_config
from fides.lib.cryptography.schemas.jwt import (
JWE_ISSUED_AT,
JWE_PAYLOAD_CLIENT_ID,
@@ -36,7 +35,6 @@
TEST_CONFIG_PATH = "tests/ctl/test_config.toml"
TEST_INVALID_CONFIG_PATH = "tests/ctl/test_invalid_config.toml"
TEST_DEPRECATED_CONFIG_PATH = "tests/ctl/test_deprecated_config.toml"
-CONFIG = get_config()
orig_requests_get = requests.get
diff --git a/tests/ctl/core/test_config.py b/tests/ctl/core/test_config.py
index 86acde1da2..c7db322a1d 100644
--- a/tests/ctl/core/test_config.py
+++ b/tests/ctl/core/test_config.py
@@ -7,7 +7,7 @@
from fides.core.config import get_config
from fides.core.config.database_settings import DatabaseSettings
-from fides.core.config.security_settings import SecurityEnv
+from fides.core.config.security_settings import SecuritySettings
REQUIRED_ENV_VARS = {
"FIDES__SECURITY__APP_ENCRYPTION_KEY": "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3",
@@ -17,6 +17,13 @@
}
+@pytest.mark.unit
+def test_get_config_verbose() -> None:
+ """Simple test to check the 'verbose' code path."""
+ config = get_config(verbose=True, config_path_override="fakefiledoesntexist.toml")
+ assert config
+
+
@pytest.mark.unit
class TestSecurityEnv:
def test_security_invalid(self):
@@ -24,7 +31,7 @@ def test_security_invalid(self):
Test that an exception is raised when an invalid Enum value is used.
"""
with pytest.raises(ValueError):
- SecurityEnv("invalid")
+ SecuritySettings(env="invalid")
# Unit
@@ -282,3 +289,61 @@ def test_config_log_level_invalid():
with pytest.raises(ValidationError) as err:
get_config()
assert "Invalid LOG_LEVEL" in str(err.value)
+
+
+class TestBuildingDatabaseValues:
+ def test_validating_included_sqlalchemy_database_uri(self) -> None:
+ """
+ Test that injecting a pre-configured database uri is
+ correctly used as opposed to building a new one.
+ """
+ incorrect_value = "incorrecthost"
+ correct_value = "correcthosthost"
+ database_settings = DatabaseSettings(
+ server=incorrect_value,
+ sqlalchemy_database_uri=f"postgresql://postgres:fides@{correct_value}:5432/fides",
+ )
+ assert incorrect_value not in database_settings.sqlalchemy_database_uri
+ assert correct_value in database_settings.sqlalchemy_database_uri
+
+ def test_validating_included_sqlalchemy_test_database_uri(self) -> None:
+ """
+ Test that injecting a pre-configured test database uri is
+ correctly used as opposed to building a new one.
+ """
+ incorrect_value = "incorrecthost"
+ correct_value = "correcthosthost"
+ database_settings = DatabaseSettings(
+ server=incorrect_value,
+ sqlalchemy_test_database_uri=f"postgresql://postgres:fides@{correct_value}:5432/fides",
+ )
+ assert incorrect_value not in database_settings.sqlalchemy_test_database_uri
+ assert correct_value in database_settings.sqlalchemy_test_database_uri
+
+ def test_validating_included_sync_database_uri(self) -> None:
+ """
+ Test that injecting a pre-configured database uri is
+ correctly used as opposed to building a new one.
+ """
+ incorrect_value = "incorrecthost"
+ correct_value = "correcthosthost"
+ database_settings = DatabaseSettings(
+ server=incorrect_value,
+ sync_database_uri=f"postgresql://postgres:fides@{correct_value}:5432/fides",
+ )
+ assert incorrect_value not in database_settings.sync_database_uri
+ assert correct_value in database_settings.sync_database_uri
+
+ def test_validating_included_async_database_uri(self) -> None:
+ """
+ Test that injecting a pre-configured database uri is
+ correctly used as opposed to building a new one.
+ """
+ incorrect_value = "incorrecthost"
+ correct_value = "correcthosthost"
+ database_settings = DatabaseSettings(
+ server=incorrect_value,
+ async_database_uri=f"postgresql://postgres:fides@{correct_value}:5432/fides",
+ )
+ assert incorrect_value not in database_settings.async_database_uri
+ assert correct_value in database_settings.async_database_uri
diff --git a/tests/ctl/core/test_utils.py b/tests/ctl/core/test_utils.py
index f7de86ea63..09b5637aaf 100644
--- a/tests/ctl/core/test_utils.py
+++ b/tests/ctl/core/test_utils.py
@@ -8,7 +8,7 @@
from fideslang.models import DatasetCollection, DatasetField
from fides.core import utils
-from fides.core.config import FidesConfig, get_config
+from fides.core.config import get_config
@pytest.fixture()
diff --git a/tests/lib/conftest.py b/tests/lib/conftest.py
index e535346c84..f0d31f9358 100644
--- a/tests/lib/conftest.py
+++ b/tests/lib/conftest.py
@@ -11,7 +11,8 @@
from sqlalchemy_utils.functions import create_database, database_exists
from starlette.testclient import TestClient
-from fides.core.config import get_config
+from fides.api.ops.api.v1.scope_registry import PRIVACY_REQUEST_READ
+from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY as SCOPES
from fides.lib.cryptography.schemas.jwt import (
JWE_ISSUED_AT,
JWE_PAYLOAD_CLIENT_ID,
@@ -24,10 +25,6 @@
from fides.lib.models.fides_user_permissions import FidesUserPermissions
from fides.lib.oauth.api.routes.user_endpoints import router
from fides.lib.oauth.jwt import generate_jwe
-from fides.api.ops.api.v1.scope_registry import (
- PRIVACY_REQUEST_READ,
- SCOPE_REGISTRY as SCOPES,
-)
from tests.conftest import create_citext_extension
ROOT_PATH = Path().absolute()
@@ -43,7 +40,7 @@ def db(config):
)
# Create the test DB engine
- assert config.is_test_mode
+ assert config.test_mode
engine = get_db_engine(
database_uri=config.database.sqlalchemy_database_uri,
)
diff --git a/tests/lib/test_client_model.py b/tests/lib/test_client_model.py
index 99d6f272b4..e10be07851 100644
--- a/tests/lib/test_client_model.py
+++ b/tests/lib/test_client_model.py
@@ -4,9 +4,9 @@
import pytest
+from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY as SCOPES
from fides.lib.cryptography.cryptographic_util import hash_with_salt
from fides.lib.models.client import ClientDetail, _get_root_client_detail
-from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY as SCOPES
def test_create_client_and_secret(db, config):
diff --git a/tests/lib/test_fides_user_permissions.py b/tests/lib/test_fides_user_permissions.py
index 66f2f634b0..28a6ba0a4d 100644
--- a/tests/lib/test_fides_user_permissions.py
+++ b/tests/lib/test_fides_user_permissions.py
@@ -2,12 +2,6 @@
from unittest.mock import MagicMock
-# Included so that `AccessManualWebhook` can be located when
-# `ConnectionConfig` is instantiated.
-from fides.api.ops.models.manual_webhook import ( # pylint: disable=unused-import
- AccessManualWebhook,
-)
-from fides.lib.models.fides_user_permissions import FidesUserPermissions
from fides.api.ops.api.v1.scope_registry import (
PRIVACY_REQUEST_READ,
USER_CREATE,
@@ -15,6 +9,13 @@
USER_READ,
)
+# Included so that `AccessManualWebhook` can be located when
+# `ConnectionConfig` is instantiated.
+from fides.api.ops.models.manual_webhook import ( # pylint: disable=unused-import
+ AccessManualWebhook,
+)
+from fides.lib.models.fides_user_permissions import FidesUserPermissions
+
def test_create_user_permissions():
permissions: FidesUserPermissions = FidesUserPermissions.create( # type: ignore
diff --git a/tests/lib/test_oauth_util.py b/tests/lib/test_oauth_util.py
index e0805c9dd4..c163b9916c 100644
--- a/tests/lib/test_oauth_util.py
+++ b/tests/lib/test_oauth_util.py
@@ -6,6 +6,7 @@
import pytest
from fastapi.security import SecurityScopes
+from fides.api.ops.api.v1.scope_registry import USER_DELETE, USER_READ
from fides.api.ops.util.oauth_util import verify_oauth_client
from fides.lib.cryptography.schemas.jwt import (
JWE_ISSUED_AT,
@@ -15,7 +16,6 @@
from fides.lib.exceptions import AuthorizationError
from fides.lib.oauth.jwt import generate_jwe
from fides.lib.oauth.oauth_util import extract_payload, is_token_expired
-from fides.api.ops.api.v1.scope_registry import USER_DELETE, USER_READ
@pytest.fixture
diff --git a/tests/lib/test_session.py b/tests/lib/test_session.py
new file mode 100644
index 0000000000..ab6097dc17
--- /dev/null
+++ b/tests/lib/test_session.py
@@ -0,0 +1,22 @@
+import pytest
+
+from fides.core.config import get_config
+from fides.lib.db import session
+
+
+class TestGetDbEngine:
+ def test_get_session_nothing_provided(self) -> None:
+ """Test getting a db engine without passing in any required vars."""
+ with pytest.raises(ValueError):
+ session.get_db_engine(config=None, database_uri="")
+
+ @pytest.mark.parametrize("test_mode", [True, False])
+ def test_get_session_test_modes(self, test_mode: bool) -> None:
+ """Test getting a db engine without passing in any required vars."""
+ config = get_config()
+ original_value = config.test_mode
+ config.test_mode = test_mode
+
+ db_engine = session.get_db_engine(config=config)
+ config.test_mode = original_value
+ assert db_engine
diff --git a/tests/ops/api/test_deps.py b/tests/ops/api/test_deps.py
index ea7b2818f1..467f5da7b5 100644
--- a/tests/ops/api/test_deps.py
+++ b/tests/ops/api/test_deps.py
@@ -8,9 +8,7 @@
import fides.api.ops.api.deps
from fides.api.ops.api.deps import get_api_session, get_cache
from fides.api.ops.common_exceptions import FunctionalityNotConfigured
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
@pytest.fixture
diff --git a/tests/ops/api/test_ratelimit.py b/tests/ops/api/test_ratelimit.py
index 876b697d28..52c07e77a1 100644
--- a/tests/ops/api/test_ratelimit.py
+++ b/tests/ops/api/test_ratelimit.py
@@ -7,9 +7,8 @@
from fides.api.main import app
from fides.api.ops.api.v1.urn_registry import HEALTH
-from fides.core.config import SecuritySettings, get_config
+from fides.core.config import CONFIG, SecuritySettings
-CONFIG = get_config()
LIMIT = 2
diff --git a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py
index 8e065bddb7..0e18cfd03c 100644
--- a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py
@@ -23,9 +23,7 @@
ProvidedIdentity,
)
from fides.api.ops.schemas.messaging.messaging import MessagingServiceType
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
@pytest.fixture(scope="function")
@@ -353,7 +351,23 @@ def test_consent_verify_consent_preferences(
)
assert response.status_code == 200
mock_verify_identity.assert_called_with(verification_code)
- assert response.json()["consent"] == consent_data
+ expected_consent_data: list[dict[str, Any]] = [
+ {
+ "data_use": "email",
+ "data_use_description": None,
+ "opt_in": True,
+ "has_gpc_flag": False,
+ "conflicts_with_gpc": False,
+ },
+ {
+ "data_use": "location",
+ "data_use_description": "Location data",
+ "opt_in": False,
+ "has_gpc_flag": False,
+ "conflicts_with_gpc": False,
+ },
+ ]
+ assert response.json()["consent"] == expected_consent_data
class TestGetConsentUnverified:
@@ -453,7 +467,23 @@ def test_consent_unverified_consent_preferences(
)
assert response.status_code == 200
assert not mock_verify_identity.called
- assert response.json()["consent"] == consent_data
+ expected_consent_data: list[dict[str, Any]] = [
+ {
+ "data_use": "email",
+ "data_use_description": None,
+ "opt_in": True,
+ "has_gpc_flag": False,
+ "conflicts_with_gpc": False,
+ },
+ {
+ "data_use": "location",
+ "data_use_description": "Location data",
+ "opt_in": False,
+ "has_gpc_flag": False,
+ "conflicts_with_gpc": False,
+ },
+ ]
+ assert response.json()["consent"] == expected_consent_data
class TestSaveConsent:
@@ -574,6 +604,8 @@ def test_verify_then_set_consent_preferences(
# Assert the code verification endpoint also returns existing consent preferences
assert response.json()["consent"][0]["data_use"] == "email"
assert response.json()["consent"][0]["opt_in"] is True
+ assert response.json()["consent"][0]["has_gpc_flag"] is False
+ assert response.json()["consent"][0]["conflicts_with_gpc"] is False
@pytest.mark.usefixtures(
"subject_identity_verification_required",
@@ -697,6 +729,8 @@ def test_set_consent_consent_preferences(
"data_use": "advertising",
"data_use_description": None,
"opt_in": True,
+ "has_gpc_flag": True,
+ "conflicts_with_gpc": False,
},
{
"data_use": "improve",
@@ -728,7 +762,23 @@ def test_set_consent_consent_preferences(
)
assert response.status_code == 200
- assert response.json()["consent"] == consent_data
+ expected_consent_data: list[dict[str, Any]] = [
+ {
+ "data_use": "advertising",
+ "data_use_description": None,
+ "opt_in": True,
+ "has_gpc_flag": True,
+ "conflicts_with_gpc": False,
+ },
+ {
+ "data_use": "improve",
+ "data_use_description": None,
+ "opt_in": False,
+ "has_gpc_flag": False,
+ "conflicts_with_gpc": False,
+ },
+ ]
+ assert response.json()["consent"] == expected_consent_data
mock_verify_identity.assert_called_with(verification_code)
db.refresh(consent_request)
@@ -747,7 +797,13 @@ def test_set_consent_consent_preferences(
"to a Privacy Request provided identity"
)
assert consent_request.privacy_request.consent_preferences == [
- {"opt_in": True, "data_use": "advertising", "data_use_description": None},
+ {
+ "conflicts_with_gpc": False,
+ "opt_in": True,
+ "data_use": "advertising",
+ "has_gpc_flag": True,
+ "data_use_description": None,
+ },
], "Only executable consent preferences stored"
assert mock_run_privacy_request.called
@@ -790,7 +846,23 @@ def test_set_consent_consent_preferences_without_verification(
json=data,
)
assert response.status_code == 200
- assert response.json()["consent"] == consent_data
+ expected_consent_data: list[dict[str, Any]] = [
+ {
+ "data_use": "email",
+ "data_use_description": None,
+ "opt_in": True,
+ "conflicts_with_gpc": False,
+ "has_gpc_flag": False,
+ },
+ {
+ "data_use": "location",
+ "data_use_description": "Location data",
+ "opt_in": False,
+ "conflicts_with_gpc": False,
+ "has_gpc_flag": False,
+ },
+ ]
+ assert response.json()["consent"] == expected_consent_data
assert not mock_verify_identity.called
@@ -867,4 +939,20 @@ def test_get_consent_preferences(
)
assert response.status_code == 200
- assert response.json()["consent"] == consent_data
+ expected_consent_data: list[dict[str, Any]] = [
+ {
+ "data_use": "email",
+ "data_use_description": None,
+ "opt_in": True,
+ "conflicts_with_gpc": False,
+ "has_gpc_flag": False,
+ },
+ {
+ "data_use": "location",
+ "data_use_description": "Location data",
+ "opt_in": False,
+ "conflicts_with_gpc": False,
+ "has_gpc_flag": False,
+ },
+ ]
+ assert response.json()["consent"] == expected_consent_data
diff --git a/tests/ops/api/v1/endpoints/test_drp_endpoints.py b/tests/ops/api/v1/endpoints/test_drp_endpoints.py
index 5dc362b2a3..4a0a0774e4 100644
--- a/tests/ops/api/v1/endpoints/test_drp_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_drp_endpoints.py
@@ -33,9 +33,7 @@
get_drp_request_body_cache_key,
get_identity_cache_key,
)
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
class TestCreateDrpPrivacyRequest:
diff --git a/tests/ops/api/v1/endpoints/test_encryption_endpoints.py b/tests/ops/api/v1/endpoints/test_encryption_endpoints.py
index bade450e7a..7ca4d558d5 100644
--- a/tests/ops/api/v1/endpoints/test_encryption_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_encryption_endpoints.py
@@ -19,11 +19,9 @@
decrypt,
encrypt_verify_secret_length,
)
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.cryptographic_util import b64_str_to_bytes, bytes_to_b64_str
-CONFIG = get_config()
-
class TestGetEncryptionKey:
@pytest.fixture
diff --git a/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py b/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py
index 6a798c9352..67dd9d3070 100644
--- a/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py
@@ -15,6 +15,7 @@ def url(self) -> str:
def subject_identity_verification_required(self, db):
"""Override autouse fixture to enable identity verification for tests"""
config = get_config()
+
original_value = config.execution.subject_identity_verification_required
config.execution.subject_identity_verification_required = True
ApplicationConfig.update_config_set(db, config)
diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py
index b33df57373..0c9e371aa0 100644
--- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py
@@ -1,5 +1,6 @@
import json
-from unittest.mock import patch
+from unittest import mock
+from unittest.mock import Mock, patch
import pytest
from fastapi_pagination import Params
@@ -12,21 +13,28 @@
MESSAGING_READ,
)
from fides.api.ops.api.v1.urn_registry import (
+ MESSAGING_ACTIVE_DEFAULT,
MESSAGING_BY_KEY,
MESSAGING_CONFIG,
+ MESSAGING_DEFAULT,
+ MESSAGING_DEFAULT_BY_TYPE,
+ MESSAGING_DEFAULT_SECRETS,
MESSAGING_SECRETS,
MESSAGING_TEST,
V1_URL_PREFIX,
)
from fides.api.ops.common_exceptions import MessageDispatchException
+from fides.api.ops.models.application_config import ApplicationConfig
from fides.api.ops.models.messaging import MessagingConfig
from fides.api.ops.schemas.messaging.messaging import (
MessagingServiceDetails,
MessagingServiceSecrets,
MessagingServiceType,
)
+from fides.core.config import get_config
PAGE_SIZE = Params().size
+CONFIG = get_config()
class TestPostMessagingConfig:
@@ -59,6 +67,13 @@ def payload_twilio_sms(self):
"service_type": MessagingServiceType.TWILIO_TEXT.value,
}
+ @pytest.fixture(scope="function")
+ def payload_twilio_sms_lowered(self):
+ return {
+ "name": "twilio_sms",
+ "service_type": MessagingServiceType.TWILIO_TEXT.value.lower(),
+ }
+
def test_post_email_config_not_authenticated(
self, api_client: TestClient, payload, url
):
@@ -92,6 +107,23 @@ def test_post_email_config_with_invalid_mailgun_details(
assert response.json()["detail"][0]["msg"] == "field required"
assert response.json()["detail"][1]["msg"] == "extra fields not permitted"
+ def test_post_mailgun_email_config_with_a_twilio_detail(
+ self,
+ db: Session,
+ api_client: TestClient,
+ url,
+ payload,
+ generate_auth_header,
+ ):
+ # add a twilio detail field to a mailgun request, should receive a validation error
+ payload["details"] = {"twilio_email_from": "invalid"}
+
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+ response = api_client.post(url, headers=auth_header, json=payload)
+ assert 422 == response.status_code
+ assert response.json()["detail"][0]["msg"] == "field required"
+ assert response.json()["detail"][1]["msg"] == "extra fields not permitted"
+
def test_post_email_config_with_not_supported_service_type(
self,
db: Session,
@@ -278,6 +310,40 @@ def test_post_twilio_sms_config(
assert expected_response == response_body
email_config.delete(db)
+ def test_post_twilio_sms_config_lowercased(
+ self,
+ db: Session,
+ api_client: TestClient,
+ payload_twilio_sms_lowered,
+ url,
+ generate_auth_header,
+ ):
+ """
+ Ensure lower-cased `service_type` values are handled well
+ """
+
+ payload_twilio_sms_lowered["key"] = "my_twilio_sms_config"
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+
+ response = api_client.post(
+ url, headers=auth_header, json=payload_twilio_sms_lowered
+ )
+ assert 200 == response.status_code
+
+ response_body = json.loads(response.text)
+ email_config = db.query(MessagingConfig).filter_by(key="my_twilio_sms_config")[
+ 0
+ ]
+
+ expected_response = {
+ "key": "my_twilio_sms_config",
+ "name": "twilio_sms",
+ "service_type": MessagingServiceType.TWILIO_TEXT.value,
+ "details": None,
+ }
+ assert expected_response == response_body
+ email_config.delete(db)
+
class TestPatchMessagingConfig:
@pytest.fixture(scope="function")
@@ -762,6 +828,581 @@ def test_delete_config(
assert config is None
+class TestGetDefaultMessagingConfig:
+ @pytest.fixture(scope="function")
+ def url(self, messaging_config: MessagingConfig) -> str:
+ return (V1_URL_PREFIX + MESSAGING_DEFAULT_BY_TYPE).format(
+ service_type=messaging_config.service_type.value
+ )
+
+ def test_get_default_config_not_authenticated(self, url, api_client: TestClient):
+ response = api_client.get(url)
+ assert 401 == response.status_code
+
+ def test_get_default_config_wrong_scope(
+ self, url, api_client: TestClient, generate_auth_header
+ ):
+ auth_header = generate_auth_header([MESSAGING_DELETE])
+ response = api_client.get(url, headers=auth_header)
+ assert 403 == response.status_code
+
+ def test_get_default_config_invalid(
+ self,
+ url,
+ api_client: TestClient,
+ generate_auth_header,
+ ):
+ auth_header = generate_auth_header([MESSAGING_READ])
+ response = api_client.get(
+ (V1_URL_PREFIX + MESSAGING_DEFAULT_BY_TYPE).format(service_type="invalid"),
+ headers=auth_header,
+ )
+ assert 422 == response.status_code
+
+ def test_get_default_config_not_exist(
+ self,
+ api_client: TestClient,
+ generate_auth_header,
+ ):
+ auth_header = generate_auth_header([MESSAGING_READ])
+ response = api_client.get(
+ (V1_URL_PREFIX + MESSAGING_DEFAULT_BY_TYPE).format(
+ service_type=MessagingServiceType.MAILGUN.value
+ ),
+ headers=auth_header,
+ )
+ assert 404 == response.status_code
+
+ def test_get_default_config(
+ self,
+ url,
+ api_client: TestClient,
+ generate_auth_header,
+ messaging_config: MessagingConfig,
+ ):
+ auth_header = generate_auth_header([MESSAGING_READ])
+ response = api_client.get(url, headers=auth_header)
+ assert response.status_code == 200
+
+ response_body = response.json()
+
+ assert response_body == {
+ "key": "my_mailgun_messaging_config",
+ "name": messaging_config.name,
+ "service_type": MessagingServiceType.MAILGUN.value,
+ "details": {
+ MessagingServiceDetails.API_VERSION.value: "v3",
+ MessagingServiceDetails.DOMAIN.value: "some.domain",
+ MessagingServiceDetails.IS_EU_DOMAIN.value: False,
+ },
+ }
+
+ def test_get_default_config_lowered_url(
+ self,
+ url,
+ api_client: TestClient,
+ generate_auth_header,
+ messaging_config: MessagingConfig,
+ ):
+ """
+ Ensure that a lowercased URL can be used, since by default we're
+ using the uppercased enum values in our URL
+ """
+ auth_header = generate_auth_header([MESSAGING_READ])
+ response = api_client.get(url.lower(), headers=auth_header)
+ assert response.status_code == 200
+
+ response_body = response.json()
+
+ assert response_body == {
+ "key": "my_mailgun_messaging_config",
+ "name": messaging_config.name,
+ "service_type": MessagingServiceType.MAILGUN.value,
+ "details": {
+ MessagingServiceDetails.API_VERSION.value: "v3",
+ MessagingServiceDetails.DOMAIN.value: "some.domain",
+ MessagingServiceDetails.IS_EU_DOMAIN.value: False,
+ },
+ }
+
+
+class TestPutDefaultMessagingConfig:
+ @pytest.fixture(scope="function")
+ def url(self) -> str:
+ return V1_URL_PREFIX + MESSAGING_DEFAULT
+
+ @pytest.fixture(scope="function")
+ def payload(self):
+ return {
+ "service_type": MessagingServiceType.MAILGUN.value,
+ "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"},
+ }
+
+ def test_put_default_messaging_config_not_authenticated(
+ self,
+ api_client: TestClient,
+ payload,
+ url,
+ ):
+ response = api_client.put(url, headers={}, json=payload)
+ assert 401 == response.status_code
+
+ def test_put_default_messaging_config_incorrect_scope(
+ self,
+ api_client: TestClient,
+ payload,
+ url,
+ generate_auth_header,
+ ):
+ auth_header = generate_auth_header([MESSAGING_READ])
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert 403 == response.status_code
+
+ def test_put_default_messaging(
+ self,
+ db: Session,
+ api_client: TestClient,
+ payload,
+ url,
+ generate_auth_header,
+ ):
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+ response = api_client.put(url, headers=auth_header, json=payload)
+
+ assert 200 == response.status_code
+ response_body = json.loads(response.text)
+
+ assert response_body["key"] is not None
+ assert response_body["service_type"] == payload["service_type"]
+ assert response_body["details"]["domain"] == payload["details"]["domain"]
+ messaging_configs = (
+ db.query(MessagingConfig)
+ .filter_by(service_type=payload["service_type"])
+ .all()
+ )
+ assert len(messaging_configs) == 1
+ assert messaging_configs[0].key == response_body["key"]
+ assert messaging_configs[0].service_type.value == payload["service_type"]
+ assert messaging_configs[0].details["domain"] == payload["details"]["domain"]
+
+ messaging_configs[0].delete(db)
+
+ def test_put_messaging_default_config_with_key_rejected(
+ self,
+ db: Session,
+ api_client: TestClient,
+ payload,
+ url,
+ generate_auth_header,
+ ):
+ payload["key"] = "my_messaging_config_key"
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert 422 == response.status_code
+
+ def test_put_messaging_default_config_with_name_rejected(
+ self,
+ db: Session,
+ api_client: TestClient,
+ payload,
+ url,
+ generate_auth_header,
+ ):
+ payload["name"] = "my_messaging_config_name"
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert 422 == response.status_code
+
+ @pytest.mark.parametrize(
+ "service_type, details",
+ [
+ (
+ MessagingServiceType.TWILIO_EMAIL.value,
+ {"twilio_email_from": "test_email@test.com"},
+ ),
+ (
+ MessagingServiceType.TWILIO_EMAIL.value.lower(),
+ {"twilio_email_from": "test_email@test.com"},
+ ),
+ (MessagingServiceType.TWILIO_TEXT.value, None),
+ ],
+ )
+ def test_put_default_messaging_config_with_different_service_types(
+ self,
+ db: Session,
+ api_client: TestClient,
+ service_type,
+ details,
+ url,
+ generate_auth_header,
+ ):
+ payload = {"service_type": service_type}
+ if details:
+ payload["details"] = details
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+ response = api_client.put(url, headers=auth_header, json=payload)
+
+ assert 200 == response.status_code
+ response_body = json.loads(response.text)
+ config_key = response_body["key"]
+ messaging_config = MessagingConfig.get_by(db, field="key", value=config_key)
+ assert service_type.upper() == response_body["service_type"]
+ assert service_type.upper() == messaging_config.service_type.value
+ messaging_config.delete(db)
+
+ def test_put_default_messaging_config_twice_only_one_record(
+ self,
+ db: Session,
+ api_client: TestClient,
+ payload,
+ url,
+ generate_auth_header,
+ ):
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+
+ # try an initial put, assert it works well
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert 200 == response.status_code
+ response_body = json.loads(response.text)
+ config_key = response_body["key"]
+ messaging_configs = (
+ db.query(MessagingConfig)
+ .filter_by(service_type=payload["service_type"])
+ .all()
+ )
+ assert len(messaging_configs) == 1
+ messaging_config = messaging_configs[0]
+ assert messaging_config.key == config_key
+ assert messaging_config.details["domain"] == payload["details"]["domain"]
+
+ # try a follow-up put after changing a detail assert it made the update to existing record
+ payload["details"][MessagingServiceDetails.DOMAIN.value] = "a.new.domain"
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert 200 == response.status_code
+ response_body = json.loads(response.text)
+ assert config_key == response_body["key"]
+ messaging_configs = (
+ db.query(MessagingConfig)
+ .filter_by(service_type=payload["service_type"])
+ .all()
+ )
+ assert len(messaging_configs) == 1
+ messaging_config = messaging_configs[0]
+ db.refresh(messaging_config)
+ assert messaging_config.key == config_key
+ assert (
+ messaging_config.details[MessagingServiceDetails.DOMAIN.value]
+ == "a.new.domain"
+ )
+
+ messaging_config.delete(db)
+
+ def test_put_default_config_invalid_details(
+ self,
+ url,
+ api_client: TestClient,
+ generate_auth_header,
+ payload,
+ ):
+ payload["details"] = {"invalid": "invalid"}
+
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert response.status_code == 422
+
+ def test_put_default_config_invalid_details_for_type(
+ self,
+ url,
+ api_client: TestClient,
+ generate_auth_header,
+ payload,
+ ):
+ # add a twilio detail field to a mailgun request, should receive a validation error
+ payload["details"] = {"twilio_email_from": "invalid"}
+
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert response.status_code == 422
+
+
+class TestPutDefaultMessagingConfigSecrets:
+ @pytest.fixture(scope="function")
+ def url(self, messaging_config) -> str:
+ return (V1_URL_PREFIX + MESSAGING_DEFAULT_SECRETS).format(
+ service_type=MessagingServiceType.MAILGUN.value
+ )
+
+ @pytest.fixture(scope="function")
+ def payload(self):
+ return {
+ MessagingServiceSecrets.MAILGUN_API_KEY.value: "1345234524",
+ }
+
+ def test_put_default_messaging_secrets_unauthenticated(
+ self, api_client: TestClient, payload, url
+ ):
+ response = api_client.put(url, headers={}, json=payload)
+ assert 401 == response.status_code
+
+ def test_put_default_messaging_secrets_wrong_scope(
+ self, api_client: TestClient, payload, url, generate_auth_header
+ ):
+ auth_header = generate_auth_header([MESSAGING_READ])
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert 403 == response.status_code
+
+ def test_put_default_messaging_secret_invalid_config(
+ self, api_client: TestClient, payload, generate_auth_header
+ ):
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+ url = (V1_URL_PREFIX + MESSAGING_DEFAULT_SECRETS).format(
+ service_type="invalid_type"
+ )
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert 422 == response.status_code
+
+ def test_put_default_messaging_secret_missing_config(
+ self, api_client: TestClient, payload, generate_auth_header
+ ):
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+ url = (V1_URL_PREFIX + MESSAGING_DEFAULT_SECRETS).format(
+ service_type=MessagingServiceType.MAILGUN.value
+ )
+ # this should get a 404 because we have not added the messaging config
+ # through a fixture
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert 404 == response.status_code
+
+ def test_update_default_with_invalid_secrets_key(
+ self, api_client: TestClient, generate_auth_header, url
+ ):
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+ response = api_client.put(url, headers=auth_header, json={"bad_key": "12345"})
+ assert response.status_code == 400
+ assert "field required" in response.text
+ assert "extra fields not permitted" in response.text
+
+ @mock.patch("fides.api.ops.models.messaging.MessagingConfig.set_secrets")
+ def test_update_default_set_secrets_error(
+ self,
+ set_secrets_mock: Mock,
+ api_client: TestClient,
+ generate_auth_header,
+ url,
+ payload,
+ ):
+ set_secrets_mock.side_effect = ValueError(
+ "This object must have a `type` to validate secrets."
+ )
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert response.status_code == 400
+
+ def test_put_default_config_secrets(
+ self,
+ db: Session,
+ api_client: TestClient,
+ payload,
+ url,
+ generate_auth_header,
+ messaging_config,
+ ):
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+ response = api_client.put(url, headers=auth_header, json=payload)
+ assert 200 == response.status_code
+
+ db.refresh(messaging_config)
+
+ assert json.loads(response.text) == {
+ "msg": f"Secrets updated for MessagingConfig with key: {messaging_config.key}.",
+ "test_status": None,
+ "failure_reason": None,
+ }
+ assert messaging_config.secrets == payload
+
+ def test_put_default_config_secrets_lowered(
+ self,
+ db: Session,
+ api_client: TestClient,
+ payload,
+ url,
+ generate_auth_header,
+ messaging_config,
+ ):
+ """
+ Ensure that a lowercased URL can be used, since by default we're
+ using the uppercased enum values in our URL
+ """
+
+ auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE])
+ response = api_client.put(url.lower(), headers=auth_header, json=payload)
+ assert 200 == response.status_code
+
+ db.refresh(messaging_config)
+
+ assert json.loads(response.text) == {
+ "msg": f"Secrets updated for MessagingConfig with key: {messaging_config.key}.",
+ "test_status": None,
+ "failure_reason": None,
+ }
+ assert messaging_config.secrets == payload
+
+
+class TestGetActiveDefaultMessagingConfig:
+ @pytest.fixture(scope="function")
+ def url(self) -> str:
+ return V1_URL_PREFIX + MESSAGING_ACTIVE_DEFAULT
+
+ def test_get_active_default_config_not_authenticated(
+ self, url, api_client: TestClient
+ ):
+ response = api_client.get(url)
+ assert 401 == response.status_code
+
+ def test_get_active_default_config_wrong_scope(
+ self, url, api_client: TestClient, generate_auth_header
+ ):
+ auth_header = generate_auth_header([MESSAGING_DELETE])
+ response = api_client.get(url, headers=auth_header)
+ assert 403 == response.status_code
+
+ def test_get_active_default_none_set(
+ self,
+ url,
+ api_client: TestClient,
+ generate_auth_header,
+ ):
+ auth_header = generate_auth_header([MESSAGING_READ])
+ response = api_client.get(
+ url,
+ headers=auth_header,
+ )
+ assert 404 == response.status_code
+
+ @pytest.mark.usefixtures("notification_service_type_none")
+ def test_get_active_default_app_setting_not_set(
+ self,
+ url,
+ api_client: TestClient,
+ generate_auth_header,
+ messaging_config,
+ ):
+ """
+ Even with `messaing_config` fixture creating a default messaging config,
+ we should still not get an active default config if the
+ `notifications.notification_service_type` config property has not been set
+
+ """
+ auth_header = generate_auth_header([MESSAGING_READ])
+ response = api_client.get(
+ url,
+ headers=auth_header,
+ )
+ assert 404 == response.status_code
+
+ @pytest.fixture(scope="function")
+ def notification_service_type_mailgun(self, db):
+ """Set mailgun as the `notification_service_type` property"""
+ original_value = CONFIG.notifications.notification_service_type
+ CONFIG.notifications.notification_service_type = "MAILGUN"
+ ApplicationConfig.update_config_set(db, CONFIG)
+ yield
+ CONFIG.notifications.notification_service_type = original_value
+ ApplicationConfig.update_config_set(db, CONFIG)
+
+ @pytest.fixture(scope="function")
+ def notification_service_type_none(self, db):
+ """Set the `notification_service_type` property to `None`"""
+ original_value = CONFIG.notifications.notification_service_type
+ CONFIG.notifications.notification_service_type = None
+ ApplicationConfig.update_config_set(db, CONFIG)
+ yield
+ CONFIG.notifications.notification_service_type = original_value
+ ApplicationConfig.update_config_set(db, CONFIG)
+
+ @pytest.fixture(scope="function")
+ def notification_service_type_invalid(self, db):
+ """Set an invalid string as the `notification_service_type` property"""
+ original_value = CONFIG.notifications.notification_service_type
+ CONFIG.notifications.notification_service_type = "invalid_service_type"
+ ApplicationConfig.update_config_set(db, CONFIG)
+ yield
+ CONFIG.notifications.notification_service_type = original_value
+ ApplicationConfig.update_config_set(db, CONFIG)
+
+ @pytest.mark.usefixtures("notification_service_type_mailgun")
+ def test_get_active_default_app_setting_but_not_configured(
+ self,
+ url,
+ api_client: TestClient,
+ generate_auth_header,
+ ):
+ """
+ Without `messaging_config` fixture creating a default messaging config,
+ but by setting the application setting for "notification_service_type" to mailgun,
+ we should get a 404, since we have no mailgun default configured.
+ """
+ auth_header = generate_auth_header([MESSAGING_READ])
+ response = api_client.get(
+ url,
+ headers=auth_header,
+ )
+ assert 404 == response.status_code
+
+ @pytest.mark.usefixtures("notification_service_type_invalid")
+ def test_get_active_default_app_setting_invalid(
+ self,
+ url,
+ api_client: TestClient,
+ generate_auth_header,
+ ):
+ """
+ This is contrived and should not be able to occur, but here we test what happens
+ if somehow the `notifications.notification_service_type` config property is set
+ to an invalid value.
+ """
+ auth_header = generate_auth_header([MESSAGING_READ])
+ with pytest.raises(ValueError) as e:
+ api_client.get(
+ url,
+ headers=auth_header,
+ )
+ assert "Unknown notification_service_type" in str(e)
+
+ @pytest.mark.usefixtures("notification_service_type_mailgun")
+ def test_get_active_default_config(
+ self,
+ url,
+ api_client: TestClient,
+ generate_auth_header,
+ messaging_config: MessagingConfig,
+ ):
+ """
+ We should get back our mailgun config default now that
+ we set mailgun as the notification_service_type via app setting
+ """
+ auth_header = generate_auth_header([MESSAGING_READ])
+ response = api_client.get(url, headers=auth_header)
+ assert response.status_code == 200
+
+ response_body = response.json()
+
+ assert response_body == {
+ "key": "my_mailgun_messaging_config",
+ "name": messaging_config.name,
+ "service_type": MessagingServiceType.MAILGUN.value,
+ "details": {
+ MessagingServiceDetails.API_VERSION.value: "v3",
+ MessagingServiceDetails.DOMAIN.value: "some.domain",
+ MessagingServiceDetails.IS_EU_DOMAIN.value: False,
+ },
+ }
+
+
class TestTestMesage:
@pytest.fixture
def url(self):
diff --git a/tests/ops/api/v1/endpoints/test_oauth_endpoints.py b/tests/ops/api/v1/endpoints/test_oauth_endpoints.py
index 27fd9bb959..d48e0aa1a4 100644
--- a/tests/ops/api/v1/endpoints/test_oauth_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_oauth_endpoints.py
@@ -27,7 +27,7 @@
from fides.api.ops.common_exceptions import OAuth2TokenException
from fides.api.ops.models.authentication_request import AuthenticationRequest
from fides.core.api import get
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.schemas.jwt import (
JWE_ISSUED_AT,
JWE_PAYLOAD_CLIENT_ID,
@@ -37,8 +37,6 @@
from fides.lib.oauth.jwt import generate_jwe
from fides.lib.oauth.oauth_util import extract_payload
-CONFIG = get_config()
-
class TestCreateClient:
@pytest.fixture(scope="function")
diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py
index b59ae34ce6..28d9911030 100644
--- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py
@@ -85,7 +85,7 @@
get_identity_cache_key,
get_masking_secret_cache_key,
)
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.schemas.jwt import (
JWE_ISSUED_AT,
JWE_PAYLOAD_CLIENT_ID,
@@ -95,7 +95,6 @@
from fides.lib.models.client import ClientDetail
from fides.lib.oauth.jwt import generate_jwe
-CONFIG = get_config()
page_size = Params().size
diff --git a/tests/ops/api/v1/endpoints/test_user_endpoints.py b/tests/ops/api/v1/endpoints/test_user_endpoints.py
index f23ef4cb95..04cf876de6 100644
--- a/tests/ops/api/v1/endpoints/test_user_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_user_endpoints.py
@@ -32,7 +32,7 @@
USERS,
V1_URL_PREFIX,
)
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.cryptographic_util import str_to_b64_str
from fides.lib.cryptography.schemas.jwt import (
JWE_ISSUED_AT,
@@ -46,7 +46,6 @@
from fides.lib.oauth.oauth_util import extract_payload
from tests.ops.conftest import generate_auth_header_for_user
-CONFIG = get_config()
page_size = Params().size
diff --git a/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py b/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py
index 16058db17e..3befd6e338 100644
--- a/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py
+++ b/tests/ops/api/v1/endpoints/test_user_permission_endpoints.py
@@ -19,14 +19,12 @@
USER_PERMISSION_UPDATE,
)
from fides.api.ops.api.v1.urn_registry import USER_PERMISSIONS, V1_URL_PREFIX
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.models.client import ClientDetail
from fides.lib.models.fides_user import FidesUser
from fides.lib.models.fides_user_permissions import FidesUserPermissions
from tests.ops.conftest import generate_auth_header_for_user
-CONFIG = get_config()
-
class TestCreateUserPermissions:
@pytest.fixture(scope="function")
diff --git a/tests/ops/api/v1/test_exception_handlers.py b/tests/ops/api/v1/test_exception_handlers.py
index 0ffee2e7eb..68e61cf00a 100644
--- a/tests/ops/api/v1/test_exception_handlers.py
+++ b/tests/ops/api/v1/test_exception_handlers.py
@@ -5,9 +5,7 @@
from fides.api.ops.api.v1.scope_registry import CLIENT_CREATE
from fides.api.ops.api.v1.urn_registry import HEALTH, PRIVACY_REQUESTS, V1_URL_PREFIX
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
@pytest.fixture
diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py
index e72a468964..0349c7de81 100644
--- a/tests/ops/conftest.py
+++ b/tests/ops/conftest.py
@@ -17,8 +17,7 @@
from fides.api.ops.models.privacy_request import generate_request_callback_jwe
from fides.api.ops.tasks.scheduled.scheduler import scheduler
from fides.api.ops.util.cache import get_cache
-from fides.core.api import db_action
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.core.config.config_proxy import ConfigProxy
from fides.lib.cryptography.schemas.jwt import (
JWE_ISSUED_AT,
@@ -44,12 +43,11 @@
from .fixtures.postgres_fixtures import *
from .fixtures.redshift_fixtures import *
from .fixtures.saas import *
+from .fixtures.saas_erasure_order_fixtures import *
from .fixtures.saas_example_fixtures import *
from .fixtures.snowflake_fixtures import *
from .fixtures.timescale_fixtures import *
-CONFIG = get_config()
-
@pytest.fixture(scope="session", autouse=True)
def setup_db(api_client):
diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py
index 6444ee67f5..9a5a647abb 100644
--- a/tests/ops/fixtures/application_fixtures.py
+++ b/tests/ops/fixtures/application_fixtures.py
@@ -64,7 +64,7 @@
StringRewriteMaskingStrategy,
)
from fides.api.ops.util.data_category import DataCategory
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.core.config.helpers import load_file
from fides.lib.models.audit_log import AuditLog, AuditLogAction
from fides.lib.models.client import ClientDetail
@@ -76,8 +76,6 @@
faker = Faker()
integration_config = load_toml("tests/ops/integration_test_config.toml")
-CONFIG = get_config()
-
# Unified list of connections to integration dbs specified from fides.api-integration.toml
diff --git a/tests/ops/fixtures/mariadb_fixtures.py b/tests/ops/fixtures/mariadb_fixtures.py
index dd9bacfe88..81f9d0a967 100644
--- a/tests/ops/fixtures/mariadb_fixtures.py
+++ b/tests/ops/fixtures/mariadb_fixtures.py
@@ -13,13 +13,11 @@
)
from fides.api.ops.models.datasetconfig import DatasetConfig
from fides.api.ops.service.connectors import MariaDBConnector
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.session import get_db_engine, get_db_session
from .application_fixtures import integration_secrets
-CONFIG = get_config()
-
@pytest.fixture(scope="function")
def connection_config_mariadb(db: Session) -> Generator:
diff --git a/tests/ops/fixtures/mssql_fixtures.py b/tests/ops/fixtures/mssql_fixtures.py
index 18e56bce82..b875b21b39 100644
--- a/tests/ops/fixtures/mssql_fixtures.py
+++ b/tests/ops/fixtures/mssql_fixtures.py
@@ -12,13 +12,11 @@
)
from fides.api.ops.models.datasetconfig import DatasetConfig
from fides.api.ops.service.connectors import MicrosoftSQLServerConnector
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.session import get_db_engine, get_db_session
from .application_fixtures import integration_secrets
-CONFIG = get_config()
-
@pytest.fixture
def mssql_example_test_dataset_config(
diff --git a/tests/ops/fixtures/mysql_fixtures.py b/tests/ops/fixtures/mysql_fixtures.py
index cb7576206e..24cfc63454 100644
--- a/tests/ops/fixtures/mysql_fixtures.py
+++ b/tests/ops/fixtures/mysql_fixtures.py
@@ -12,13 +12,11 @@
)
from fides.api.ops.models.datasetconfig import DatasetConfig
from fides.api.ops.service.connectors import MySQLConnector
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.session import get_db_engine, get_db_session
from .application_fixtures import integration_secrets
-CONFIG = get_config()
-
@pytest.fixture(scope="function")
def dataset_config_mysql(
diff --git a/tests/ops/fixtures/postgres_fixtures.py b/tests/ops/fixtures/postgres_fixtures.py
index cbfb188a70..e7dfce3cf9 100644
--- a/tests/ops/fixtures/postgres_fixtures.py
+++ b/tests/ops/fixtures/postgres_fixtures.py
@@ -19,14 +19,12 @@
PrivacyRequest,
)
from fides.api.ops.service.connectors import PostgreSQLConnector
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.session import get_db_engine, get_db_session
from tests.ops.test_helpers.db_utils import seed_postgres_data
from .application_fixtures import integration_secrets
-CONFIG = get_config()
-
@pytest.fixture
def postgres_example_test_dataset_config(
diff --git a/tests/ops/fixtures/saas/kustomer_fixtures.py b/tests/ops/fixtures/saas/kustomer_fixtures.py
new file mode 100644
index 0000000000..7ba99e65a1
--- /dev/null
+++ b/tests/ops/fixtures/saas/kustomer_fixtures.py
@@ -0,0 +1,138 @@
+from time import sleep
+from typing import Any, Dict, Generator
+
+import pydash
+import pytest
+import requests
+from sqlalchemy.orm import Session
+
+from fides.api.ctl.sql_models import Dataset as CtlDataset
+from fides.api.ops.models.connectionconfig import (
+ AccessLevel,
+ ConnectionConfig,
+ ConnectionType,
+)
+from fides.api.ops.models.datasetconfig import DatasetConfig
+from fides.api.ops.util.saas_util import (
+ load_config_with_replacement,
+ load_dataset_with_replacement,
+)
+from fides.lib.cryptography import cryptographic_util
+from tests.ops.test_helpers.vault_client import get_secrets
+
+secrets = get_secrets("kustomer")
+
+
+@pytest.fixture(scope="session")
+def kustomer_secrets(saas_config):
+ return {
+ "domain": pydash.get(saas_config, "kustomer.domain") or secrets["domain"],
+ "api_key": pydash.get(saas_config, "kustomer.api_key") or secrets["api_key"],
+ }
+
+
+@pytest.fixture(scope="session")
+def kustomer_identity_email(saas_config):
+ return (
+ pydash.get(saas_config, "kustomer.identity_email") or secrets["identity_email"]
+ )
+
+
+@pytest.fixture(scope="session")
+def kustomer_identity_phone_number(saas_config):
+ return (
+ pydash.get(saas_config, "kustomer.identity_phone_number")
+ or secrets["identity_phone_number"]
+ )
+
+
+@pytest.fixture(scope="function")
+def kustomer_erasure_identity_email() -> str:
+ return f"{cryptographic_util.generate_secure_random_string(13)}@email.com"
+
+
+@pytest.fixture
+def kustomer_config() -> Dict[str, Any]:
+ return load_config_with_replacement(
+ "data/saas/config/kustomer_config.yml",
+ "",
+ "kustomer_instance",
+ )
+
+
+@pytest.fixture
+def kustomer_dataset() -> Dict[str, Any]:
+ return load_dataset_with_replacement(
+ "data/saas/dataset/kustomer_dataset.yml",
+ "",
+ "kustomer_instance",
+ )[0]
+
+
+@pytest.fixture(scope="function")
+def kustomer_connection_config(
+ db: Session, kustomer_config, kustomer_secrets
+) -> Generator:
+ fides_key = kustomer_config["fides_key"]
+ connection_config = ConnectionConfig.create(
+ db=db,
+ data={
+ "key": fides_key,
+ "name": fides_key,
+ "connection_type": ConnectionType.saas,
+ "access": AccessLevel.write,
+ "secrets": kustomer_secrets,
+ "saas_config": kustomer_config,
+ },
+ )
+ yield connection_config
+ connection_config.delete(db)
+
+
+@pytest.fixture
+def kustomer_dataset_config(
+ db: Session,
+ kustomer_connection_config: ConnectionConfig,
+ kustomer_dataset: Dict[str, Any],
+) -> Generator:
+ fides_key = kustomer_dataset["fides_key"]
+ kustomer_connection_config.name = fides_key
+ kustomer_connection_config.key = fides_key
+ kustomer_connection_config.save(db=db)
+
+ ctl_dataset = CtlDataset.create_from_dataset_dict(db, kustomer_dataset)
+
+ dataset = DatasetConfig.create(
+ db=db,
+ data={
+ "connection_config_id": kustomer_connection_config.id,
+ "fides_key": fides_key,
+ "ctl_dataset_id": ctl_dataset.id,
+ },
+ )
+ yield dataset
+ dataset.delete(db=db)
+ ctl_dataset.delete(db=db)
+
+
+@pytest.fixture(scope="function")
+def kustomer_create_erasure_data(
+ kustomer_connection_config: ConnectionConfig, kustomer_erasure_identity_email: str
+) -> None:
+
+ kustomer_secrets = kustomer_connection_config.secrets
+ base_url = f"https://{kustomer_secrets['domain']}"
+ headers = {
+ "Authorization": f"Bearer {kustomer_secrets['api_key']}",
+ }
+ # create customer
+ body = {
+ "name": "Ethyca Test Erasure",
+ "emails": [{"email": kustomer_erasure_identity_email}],
+ }
+
+ customer_response = requests.post(
+ url=f"{base_url}/v1/customers", headers=headers, json=body
+ )
+ customer = customer_response.json()
+ yield customer
diff --git a/tests/ops/fixtures/saas_erasure_order_fixtures.py b/tests/ops/fixtures/saas_erasure_order_fixtures.py
new file mode 100644
index 0000000000..56f299b4a0
--- /dev/null
+++ b/tests/ops/fixtures/saas_erasure_order_fixtures.py
@@ -0,0 +1,87 @@
+from typing import Any, Dict, Generator
+
+import pytest
+from sqlalchemy.orm import Session
+
+from fides.api.ctl.sql_models import Dataset as CtlDataset
+from fides.api.ops.models.connectionconfig import (
+ AccessLevel,
+ ConnectionConfig,
+ ConnectionType,
+)
+from fides.api.ops.models.datasetconfig import DatasetConfig
+from fides.api.ops.util.saas_util import (
+ load_config_with_replacement,
+ load_dataset_with_replacement,
+)
+
+
+@pytest.fixture(scope="function")
+def saas_erasure_order_secrets():
+ return {"domain": "domain"}
+
+
+@pytest.fixture
+def saas_erasure_order_config() -> Dict:
+ return load_config_with_replacement(
+ "data/saas/config/saas_erasure_order_config.yml",
+ "",
+ "saas_erasure_order_instance",
+ )
+
+
+@pytest.fixture
+def saas_erasure_order_dataset() -> Dict:
+ return load_dataset_with_replacement(
+ "data/saas/dataset/saas_erasure_order_dataset.yml",
+ "",
+ "saas_erasure_order_instance",
+ )[0]
+
+
+@pytest.fixture(scope="function")
+def saas_erasure_order_connection_config(
+ db: Session,
+ saas_erasure_order_config: Dict[str, Any],
+ saas_erasure_order_secrets: Dict[str, Any],
+) -> Generator:
+ fides_key = saas_erasure_order_config["fides_key"]
+ connection_config = ConnectionConfig.create(
+ db=db,
+ data={
+ "key": fides_key,
+ "name": fides_key,
+ "connection_type": ConnectionType.saas,
+ "access": AccessLevel.write,
+ "secrets": saas_erasure_order_secrets,
+ "saas_config": saas_erasure_order_config,
+ },
+ )
+ yield connection_config
+ connection_config.delete(db)
+
+
+@pytest.fixture
+def saas_erasure_order_dataset_config(
+ db: Session,
+ saas_erasure_order_connection_config: ConnectionConfig,
+ saas_erasure_order_dataset: Dict,
+) -> Generator:
+ fides_key = saas_erasure_order_dataset["fides_key"]
+ saas_erasure_order_connection_config.name = fides_key
+ saas_erasure_order_connection_config.key = fides_key
+ saas_erasure_order_connection_config.save(db=db)
+
+ ctl_dataset = CtlDataset.create_from_dataset_dict(db, saas_erasure_order_dataset)
+
+ dataset = DatasetConfig.create(
+ db=db,
+ data={
+ "connection_config_id": saas_erasure_order_connection_config.id,
+ "fides_key": fides_key,
+ "ctl_dataset_id": ctl_dataset.id,
+ },
+ )
+ yield dataset
+ dataset.delete(db=db)
+ ctl_dataset.delete(db)
diff --git a/tests/ops/fixtures/timescale_fixtures.py b/tests/ops/fixtures/timescale_fixtures.py
index 90b90e1b86..592901b7be 100644
--- a/tests/ops/fixtures/timescale_fixtures.py
+++ b/tests/ops/fixtures/timescale_fixtures.py
@@ -11,14 +11,12 @@
ConnectionType,
)
from fides.api.ops.service.connectors import TimescaleConnector
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.session import get_db_engine, get_db_session
from tests.ops.test_helpers.db_utils import seed_postgres_data
from .application_fixtures import integration_secrets
-CONFIG = get_config()
-
@pytest.fixture(scope="function")
def timescale_connection_config(
diff --git a/tests/ops/graph/test_graph.py b/tests/ops/graph/test_graph.py
index 416c523d5e..7f816ae4fb 100644
--- a/tests/ops/graph/test_graph.py
+++ b/tests/ops/graph/test_graph.py
@@ -5,11 +5,7 @@
from fides.api.ops.models.policy import ActionType
from fides.api.ops.task.graph_task import retry
from fides.api.ops.task.task_resources import TaskResources
-from fides.core.config import get_config
-from tests.ops.task.traversal_data import integration_db_graph
-
-CONFIG = get_config()
-
+from fides.core.config import CONFIG
from tests.ops.task.traversal_data import integration_db_graph
t1 = Collection(
diff --git a/tests/ops/integration_tests/saas/request_override/test_firebase_auth_task.py b/tests/ops/integration_tests/saas/request_override/test_firebase_auth_task.py
index 9c98a14a07..9e575ff0c3 100644
--- a/tests/ops/integration_tests/saas/request_override/test_firebase_auth_task.py
+++ b/tests/ops/integration_tests/saas/request_override/test_firebase_auth_task.py
@@ -13,11 +13,9 @@
)
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_firebase_auth
diff --git a/tests/ops/integration_tests/saas/test_adobe_campaign_task.py b/tests/ops/integration_tests/saas/test_adobe_campaign_task.py
index b5eb8b9b93..6f7422bb12 100644
--- a/tests/ops/integration_tests/saas/test_adobe_campaign_task.py
+++ b/tests/ops/integration_tests/saas/test_adobe_campaign_task.py
@@ -8,11 +8,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.skip(reason="Only staging credentials available")
@pytest.mark.integration_saas
diff --git a/tests/ops/integration_tests/saas/test_auth0_task.py b/tests/ops/integration_tests/saas/test_auth0_task.py
index e8009106d5..be9511c2f1 100644
--- a/tests/ops/integration_tests/saas/test_auth0_task.py
+++ b/tests/ops/integration_tests/saas/test_auth0_task.py
@@ -9,11 +9,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.skip(reason="Pending development of OAuth2 JWT Bearer authentication")
@pytest.mark.integration_saas
diff --git a/tests/ops/integration_tests/saas/test_braintree_task.py b/tests/ops/integration_tests/saas/test_braintree_task.py
index ef08ea6fbc..167731e678 100644
--- a/tests/ops/integration_tests/saas/test_braintree_task.py
+++ b/tests/ops/integration_tests/saas/test_braintree_task.py
@@ -9,10 +9,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
logger = logging.getLogger(__name__)
diff --git a/tests/ops/integration_tests/saas/test_braze_task.py b/tests/ops/integration_tests/saas/test_braze_task.py
index f64953d71f..74537c85cc 100644
--- a/tests/ops/integration_tests/saas/test_braze_task.py
+++ b/tests/ops/integration_tests/saas/test_braze_task.py
@@ -10,11 +10,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_braze
diff --git a/tests/ops/integration_tests/saas/test_domo_task.py b/tests/ops/integration_tests/saas/test_domo_task.py
index b28a2004f9..6505aac09d 100644
--- a/tests/ops/integration_tests/saas/test_domo_task.py
+++ b/tests/ops/integration_tests/saas/test_domo_task.py
@@ -8,11 +8,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.skip(reason="Pending account resolution")
@pytest.mark.integration_saas
diff --git a/tests/ops/integration_tests/saas/test_friendbuy_nextgen_task.py b/tests/ops/integration_tests/saas/test_friendbuy_nextgen_task.py
index 53673d1969..b22cf2c324 100644
--- a/tests/ops/integration_tests/saas/test_friendbuy_nextgen_task.py
+++ b/tests/ops/integration_tests/saas/test_friendbuy_nextgen_task.py
@@ -10,10 +10,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
logger = logging.getLogger(__name__)
diff --git a/tests/ops/integration_tests/saas/test_friendbuy_task.py b/tests/ops/integration_tests/saas/test_friendbuy_task.py
index 788e0bf982..04497b01de 100644
--- a/tests/ops/integration_tests/saas/test_friendbuy_task.py
+++ b/tests/ops/integration_tests/saas/test_friendbuy_task.py
@@ -11,10 +11,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
logger = logging.getLogger(__name__)
diff --git a/tests/ops/integration_tests/saas/test_fullstory_task.py b/tests/ops/integration_tests/saas/test_fullstory_task.py
index 6e2a928a1f..fccfebf6e8 100644
--- a/tests/ops/integration_tests/saas/test_fullstory_task.py
+++ b/tests/ops/integration_tests/saas/test_fullstory_task.py
@@ -8,13 +8,11 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.fixtures.saas.fullstory_fixtures import FullstoryTestClient, user_updated
from tests.ops.graph.graph_test_util import assert_rows_match
from tests.ops.test_helpers.saas_test_utils import poll_for_existence
-CONFIG = get_config()
-
@pytest.mark.skip("API keys are temporary for free accounts")
@pytest.mark.integration_saas
diff --git a/tests/ops/integration_tests/saas/test_hubspot_task.py b/tests/ops/integration_tests/saas/test_hubspot_task.py
index f038901dcd..a477c728c3 100644
--- a/tests/ops/integration_tests/saas/test_hubspot_task.py
+++ b/tests/ops/integration_tests/saas/test_hubspot_task.py
@@ -9,13 +9,11 @@
from fides.api.ops.task import graph_task
from fides.api.ops.task.filter_results import filter_data_categories
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.fixtures.saas.hubspot_fixtures import HubspotTestClient, user_exists
from tests.ops.graph.graph_test_util import assert_rows_match
from tests.ops.test_helpers.saas_test_utils import poll_for_existence
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_hubspot
diff --git a/tests/ops/integration_tests/saas/test_kustomer_task.py b/tests/ops/integration_tests/saas/test_kustomer_task.py
new file mode 100644
index 0000000000..a8fea75f14
--- /dev/null
+++ b/tests/ops/integration_tests/saas/test_kustomer_task.py
@@ -0,0 +1,179 @@
+import random
+
+import pytest
+import requests
+
+from fides.api.ops.graph.graph import DatasetGraph
+from fides.api.ops.models.privacy_request import PrivacyRequest
+from fides.api.ops.schemas.redis_cache import Identity
+from fides.api.ops.service.connectors import get_connector
+from fides.api.ops.task import graph_task
+from fides.api.ops.task.graph_task import get_cached_data_for_erasures
+from fides.core.config import get_config
+from tests.ops.graph.graph_test_util import assert_rows_match
+
+CONFIG = get_config()
+
+
+@pytest.mark.integration_saas
+@pytest.mark.integration_kustomer
+def test_kustomer_connection_test(kustomer_connection_config) -> None:
+ get_connector(kustomer_connection_config).test_connection()
+
+
+@pytest.mark.integration_saas
+@pytest.mark.integration_kustomer
+@pytest.mark.asyncio
+async def test_kustomer_access_request_task_with_email(
+ db,
+ policy,
+ kustomer_connection_config,
+ kustomer_dataset_config,
+ kustomer_identity_email,
+) -> None:
+ """Full access request based on the Kustomer SaaS config"""
+
+ privacy_request = PrivacyRequest(
+ id=f"test_kustomer_access_request_task_{random.randint(0, 1000)}"
+ )
+ identity = Identity(**{"email": kustomer_identity_email})
+ privacy_request.cache_identity(identity)
+
+ dataset_name = kustomer_connection_config.get_saas_config().fides_key
+ merged_graph = kustomer_dataset_config.get_graph()
+ graph = DatasetGraph(merged_graph)
+
+ v = await graph_task.run_access_request(
+ privacy_request,
+ policy,
+ graph,
+ [kustomer_connection_config],
+ {"email": kustomer_identity_email},
+ db,
+ )
+
+ assert_rows_match(
+ v[f"{dataset_name}:customer"],
+ min_size=1,
+ keys=["type", "id", "attributes", "relationships", "links"],
+ )
+
+ # verify we only returned data for our identity email
+ assert (
+ v[f"{dataset_name}:customer"][0]["attributes"]["emails"][0]["email"]
+ == kustomer_identity_email
+ )
+
+
+@pytest.mark.integration_saas
+@pytest.mark.integration_kustomer
+@pytest.mark.asyncio
+async def test_kustomer_access_request_task_with_phone_number(
+ db,
+ policy,
+ kustomer_connection_config,
+ kustomer_dataset_config,
+ kustomer_identity_phone_number,
+) -> None:
+ """Full access request based on the Kustomer SaaS config"""
+
+ privacy_request = PrivacyRequest(
+ id=f"test_kustomer_access_request_task_{random.randint(0, 1000)}"
+ )
+ identity = Identity(**{"phone_number": kustomer_identity_phone_number})
+ privacy_request.cache_identity(identity)
+
+ dataset_name = kustomer_connection_config.get_saas_config().fides_key
+ merged_graph = kustomer_dataset_config.get_graph()
+ graph = DatasetGraph(merged_graph)
+
+ v = await graph_task.run_access_request(
+ privacy_request,
+ policy,
+ graph,
+ [kustomer_connection_config],
+ {"phone_number": kustomer_identity_phone_number},
+ db,
+ )
+
+ assert_rows_match(
+ v[f"{dataset_name}:customer"],
+ min_size=1,
+ keys=["type", "id", "attributes", "relationships", "links"],
+ )
+
+ # verify we only returned data for our identity phone number
+ assert (
+ v[f"{dataset_name}:customer"][0]["attributes"]["phones"][0]["phone"]
+ == kustomer_identity_phone_number
+ )
+
+
+@pytest.mark.integration_saas
+@pytest.mark.integration_kustomer
+@pytest.mark.asyncio
+async def test_kustomer_erasure_request_task(
+ db,
+ policy,
+ erasure_policy_string_rewrite,
+ kustomer_connection_config,
+ kustomer_dataset_config,
+ kustomer_erasure_identity_email,
+ kustomer_create_erasure_data,
+) -> None:
+ """Full erasure request based on the Kustomer SaaS config"""
+
+ masking_strict = CONFIG.execution.masking_strict
+ CONFIG.execution.masking_strict = False # Allow Delete
+
+ privacy_request = PrivacyRequest(
+ id=f"test_kustomer_erasure_request_task_{random.randint(0, 1000)}"
+ )
+ identity = Identity(**{"email": kustomer_erasure_identity_email})
+ privacy_request.cache_identity(identity)
+
+ dataset_name = kustomer_connection_config.get_saas_config().fides_key
+ merged_graph = kustomer_dataset_config.get_graph()
+ graph = DatasetGraph(merged_graph)
+
+ v = await graph_task.run_access_request(
+ privacy_request,
+ policy,
+ graph,
+ [kustomer_connection_config],
+ {"email": kustomer_erasure_identity_email},
+ db,
+ )
+
+ assert_rows_match(
+ v[f"{dataset_name}:customer"],
+ min_size=1,
+ keys=["type", "id", "attributes", "relationships", "links"],
+ )
+
+ x = await graph_task.run_erasure(
+ privacy_request,
+ erasure_policy_string_rewrite,
+ graph,
+ [kustomer_connection_config],
+ {"email": kustomer_erasure_identity_email},
+ get_cached_data_for_erasures(privacy_request.id),
+ db,
+ )
+
+ assert x == {f"{dataset_name}:customer": 1}
+
+ kustomer_secrets = kustomer_connection_config.secrets
+ headers = {
+ "Authorization": f"Bearer {kustomer_secrets['api_key']}",
+ }
+ base_url = f"https://{kustomer_secrets['domain']}"
+
+ response = requests.get(
+ url=f"{base_url}/v1/customers/email={kustomer_erasure_identity_email}",
+ headers=headers,
+ )
+
+ assert response.status_code == 404
+
+ CONFIG.execution.masking_strict = masking_strict
diff --git a/tests/ops/integration_tests/saas/test_outreach_task.py b/tests/ops/integration_tests/saas/test_outreach_task.py
index 34da0c45a8..60054c87be 100644
--- a/tests/ops/integration_tests/saas/test_outreach_task.py
+++ b/tests/ops/integration_tests/saas/test_outreach_task.py
@@ -9,11 +9,9 @@
from fides.api.ops.task import graph_task
from fides.api.ops.task.filter_results import filter_data_categories
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors")
@pytest.mark.integration_saas
diff --git a/tests/ops/integration_tests/saas/test_recharge_tasks.py b/tests/ops/integration_tests/saas/test_recharge_tasks.py
index 226e74948e..9ba6c109ca 100644
--- a/tests/ops/integration_tests/saas/test_recharge_tasks.py
+++ b/tests/ops/integration_tests/saas/test_recharge_tasks.py
@@ -8,11 +8,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_recharge
diff --git a/tests/ops/integration_tests/saas/test_rollbar_task.py b/tests/ops/integration_tests/saas/test_rollbar_task.py
index 8cbc2f6498..f71522da04 100644
--- a/tests/ops/integration_tests/saas/test_rollbar_task.py
+++ b/tests/ops/integration_tests/saas/test_rollbar_task.py
@@ -8,11 +8,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.skip(reason="Pending account resolution")
@pytest.mark.integration_saas
diff --git a/tests/ops/integration_tests/saas/test_salesforce_task.py b/tests/ops/integration_tests/saas/test_salesforce_task.py
index f6df93bbba..cadeec362c 100644
--- a/tests/ops/integration_tests/saas/test_salesforce_task.py
+++ b/tests/ops/integration_tests/saas/test_salesforce_task.py
@@ -9,11 +9,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors")
@pytest.mark.integration_saas
diff --git a/tests/ops/integration_tests/saas/test_segment_task.py b/tests/ops/integration_tests/saas/test_segment_task.py
index 2708b90e85..6e64c3e8b7 100644
--- a/tests/ops/integration_tests/saas/test_segment_task.py
+++ b/tests/ops/integration_tests/saas/test_segment_task.py
@@ -9,11 +9,9 @@
from fides.api.ops.task import graph_task
from fides.api.ops.task.filter_results import filter_data_categories
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.skip(reason="Pending account resolution")
@pytest.mark.integration_saas
diff --git a/tests/ops/integration_tests/saas/test_sendgrid_task.py b/tests/ops/integration_tests/saas/test_sendgrid_task.py
index da946a13a8..24809d1935 100644
--- a/tests/ops/integration_tests/saas/test_sendgrid_task.py
+++ b/tests/ops/integration_tests/saas/test_sendgrid_task.py
@@ -8,13 +8,11 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.fixtures.saas.sendgrid_fixtures import contact_exists
from tests.ops.graph.graph_test_util import assert_rows_match
from tests.ops.test_helpers.saas_test_utils import poll_for_existence
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_sendgrid
diff --git a/tests/ops/integration_tests/saas/test_shopify_task.py b/tests/ops/integration_tests/saas/test_shopify_task.py
index 1f215b654f..0d346061c0 100644
--- a/tests/ops/integration_tests/saas/test_shopify_task.py
+++ b/tests/ops/integration_tests/saas/test_shopify_task.py
@@ -10,11 +10,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_shopify
diff --git a/tests/ops/integration_tests/saas/test_slack_enterprise_task.py b/tests/ops/integration_tests/saas/test_slack_enterprise_task.py
index 33898608f8..d3ea0bb771 100644
--- a/tests/ops/integration_tests/saas/test_slack_enterprise_task.py
+++ b/tests/ops/integration_tests/saas/test_slack_enterprise_task.py
@@ -7,12 +7,10 @@
from fides.api.ops.schemas.redis_cache import Identity
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
from tests.ops.test_helpers.dataset_utils import update_dataset
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_slack
diff --git a/tests/ops/integration_tests/saas/test_square_task.py b/tests/ops/integration_tests/saas/test_square_task.py
index a0196235e3..926a38189f 100644
--- a/tests/ops/integration_tests/saas/test_square_task.py
+++ b/tests/ops/integration_tests/saas/test_square_task.py
@@ -9,11 +9,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_square
diff --git a/tests/ops/integration_tests/saas/test_stripe_task.py b/tests/ops/integration_tests/saas/test_stripe_task.py
index afe28e510d..8a9cc5aadb 100644
--- a/tests/ops/integration_tests/saas/test_stripe_task.py
+++ b/tests/ops/integration_tests/saas/test_stripe_task.py
@@ -11,11 +11,9 @@
from fides.api.ops.task import graph_task
from fides.api.ops.task.filter_results import filter_data_categories
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_stripe
diff --git a/tests/ops/integration_tests/saas/test_twilio_conversations_task.py b/tests/ops/integration_tests/saas/test_twilio_conversations_task.py
index a0a94d5d2b..8397c2fe98 100644
--- a/tests/ops/integration_tests/saas/test_twilio_conversations_task.py
+++ b/tests/ops/integration_tests/saas/test_twilio_conversations_task.py
@@ -9,11 +9,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_twilio_conversations
diff --git a/tests/ops/integration_tests/saas/test_zendesk_task.py b/tests/ops/integration_tests/saas/test_zendesk_task.py
index b1cc681fa6..be67754e4a 100644
--- a/tests/ops/integration_tests/saas/test_zendesk_task.py
+++ b/tests/ops/integration_tests/saas/test_zendesk_task.py
@@ -10,11 +10,9 @@
from fides.api.ops.service.connectors import get_connector
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import assert_rows_match
-CONFIG = get_config()
-
@pytest.mark.integration_saas
@pytest.mark.integration_zendesk
diff --git a/tests/ops/integration_tests/setup_scripts/mariadb_setup.py b/tests/ops/integration_tests/setup_scripts/mariadb_setup.py
index 814eea37be..34ff363d58 100644
--- a/tests/ops/integration_tests/setup_scripts/mariadb_setup.py
+++ b/tests/ops/integration_tests/setup_scripts/mariadb_setup.py
@@ -10,10 +10,9 @@
ConnectionType,
)
from fides.api.ops.service.connectors.sql_connector import MariaDBConnector
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.session import get_db_engine, get_db_session
-CONFIG = get_config()
integration_config = load_toml("tests/ops/integration_test_config.toml")
diff --git a/tests/ops/integration_tests/setup_scripts/postgres_setup.py b/tests/ops/integration_tests/setup_scripts/postgres_setup.py
index 69b317dd68..8873bf9713 100644
--- a/tests/ops/integration_tests/setup_scripts/postgres_setup.py
+++ b/tests/ops/integration_tests/setup_scripts/postgres_setup.py
@@ -12,10 +12,9 @@
ConnectionType,
)
from fides.api.ops.service.connectors.sql_connector import PostgreSQLConnector
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.session import get_db_engine, get_db_session
-CONFIG = get_config()
integration_config = load_toml("tests/ops/integration_test_config.toml")
diff --git a/tests/ops/integration_tests/setup_scripts/timescale_setup.py b/tests/ops/integration_tests/setup_scripts/timescale_setup.py
index 1c35139497..5cb95f42f3 100644
--- a/tests/ops/integration_tests/setup_scripts/timescale_setup.py
+++ b/tests/ops/integration_tests/setup_scripts/timescale_setup.py
@@ -12,10 +12,9 @@
ConnectionType,
)
from fides.api.ops.service.connectors import TimescaleConnector
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.session import get_db_engine, get_db_session
-CONFIG = get_config()
integration_config = load_toml("tests/ops/integration_test_config.toml")
diff --git a/tests/ops/integration_tests/test_execution.py b/tests/ops/integration_tests/test_execution.py
index f1831ef7f7..af3f73a833 100644
--- a/tests/ops/integration_tests/test_execution.py
+++ b/tests/ops/integration_tests/test_execution.py
@@ -22,7 +22,7 @@
)
from fides.api.ops.task import graph_task
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.db.session import get_db_session
from ..fixtures.application_fixtures import integration_secrets
@@ -30,8 +30,6 @@
get_privacy_request_results,
)
-CONFIG = get_config()
-
def get_sorted_execution_logs(db, privacy_request: PrivacyRequest):
return [
diff --git a/tests/ops/integration_tests/test_integration_erasure_order.py b/tests/ops/integration_tests/test_integration_erasure_order.py
new file mode 100644
index 0000000000..52bb04cb37
--- /dev/null
+++ b/tests/ops/integration_tests/test_integration_erasure_order.py
@@ -0,0 +1,350 @@
+import random
+from typing import Any, Dict, List
+from unittest import mock
+
+import pytest
+from sqlalchemy.orm import Session
+
+from fides.api.ops.common_exceptions import TraversalError
+from fides.api.ops.graph.graph import DatasetGraph
+from fides.api.ops.graph.traversal import TraversalNode
+from fides.api.ops.models.policy import ActionType, Policy
+from fides.api.ops.models.privacy_request import ExecutionLog, PrivacyRequest
+from fides.api.ops.schemas.redis_cache import Identity
+from fides.api.ops.service.connectors.saas.authenticated_client import (
+ AuthenticatedClient,
+)
+from fides.api.ops.service.saas_request.saas_request_override_factory import (
+ SaaSRequestType,
+ register,
+)
+from fides.api.ops.task import graph_task
+from fides.api.ops.task.graph_task import get_cached_data_for_erasures
+from fides.api.ops.util.collection_util import Row
+from fides.core.config import get_config
+from tests.ops.graph.graph_test_util import assert_rows_match
+
+CONFIG = get_config()
+
+
+def erasure_execution_logs(
+ db: Session, privacy_request: PrivacyRequest
+) -> List[ExecutionLog]:
+ """Returns the erasure execution logs for the given privacy request ordered by created_at"""
+ return (
+ db.query(ExecutionLog)
+ .filter_by(
+ privacy_request_id=privacy_request.id, action_type=ActionType.erasure
+ )
+ .order_by("created_at")
+ .all()
+ )
+
+
+# custom override functions to facilitate testing
+@register("read_no_op", [SaaSRequestType.READ])
+def read_no_op(
+ client: AuthenticatedClient,
+ node: TraversalNode,
+ policy: Policy,
+ privacy_request: PrivacyRequest,
+ input_data: Dict[str, List[Any]],
+ secrets: Dict[str, Any],
+) -> List[Row]:
+ return [{f"{node.address.collection}_id": 1}]
+
+
+@register("delete_no_op", [SaaSRequestType.DELETE])
+def delete_no_op(
+ client: AuthenticatedClient,
+ param_values_per_row: List[Dict[str, Any]],
+ policy: Policy,
+ privacy_request: PrivacyRequest,
+ secrets: Dict[str, Any],
+) -> int:
+ return 1
+
+
+@pytest.mark.integration_saas
+@pytest.mark.asyncio
+async def test_saas_erasure_order_request_task(
+ db,
+ policy,
+ erasure_policy_complete_mask,
+ saas_erasure_order_connection_config,
+ saas_erasure_order_dataset_config,
+) -> None:
+ privacy_request = PrivacyRequest(
+ id=f"test_saas_erasure_order_request_task_{random.randint(0, 1000)}"
+ )
+ identity_attribute = "email"
+ identity_value = "test@ethyca.com"
+ identity_kwargs = {identity_attribute: identity_value}
+ identity = Identity(**identity_kwargs)
+ privacy_request.cache_identity(identity)
+
+ dataset_name = saas_erasure_order_connection_config.get_saas_config().fides_key
+ merged_graph = saas_erasure_order_dataset_config.get_graph()
+ graph = DatasetGraph(merged_graph)
+
+ v = await graph_task.run_access_request(
+ privacy_request,
+ policy,
+ graph,
+ [saas_erasure_order_connection_config],
+ {"email": "test@ethyca.com"},
+ db,
+ )
+
+ assert_rows_match(v[f"{dataset_name}:orders"], min_size=1, keys=["orders_id"])
+ assert_rows_match(v[f"{dataset_name}:refunds"], min_size=1, keys=["refunds_id"])
+ assert_rows_match(v[f"{dataset_name}:labels"], min_size=1, keys=["labels_id"])
+ assert_rows_match(v[f"{dataset_name}:products"], min_size=1, keys=["products_id"])
+ assert_rows_match(
+ v[f"{dataset_name}:orders_to_refunds"],
+ min_size=1,
+ keys=["orders_to_refunds_id"],
+ )
+ assert_rows_match(
+ v[f"{dataset_name}:refunds_to_orders"],
+ min_size=1,
+ keys=["refunds_to_orders_id"],
+ )
+
+ temp_masking = CONFIG.execution.masking_strict
+ CONFIG.execution.masking_strict = False
+
+ x = await graph_task.run_erasure(
+ privacy_request,
+ erasure_policy_complete_mask,
+ graph,
+ [saas_erasure_order_connection_config],
+ identity_kwargs,
+ get_cached_data_for_erasures(privacy_request.id),
+ db,
+ )
+
+ assert x == {
+ f"{dataset_name}:orders": 1,
+ f"{dataset_name}:refunds": 1,
+ f"{dataset_name}:labels": 1,
+ f"{dataset_name}:products": 0,
+ f"{dataset_name}:orders_to_refunds": 1,
+ f"{dataset_name}:refunds_to_orders": 1,
+ }
+
+ # Retrieve the erasure logs ordered by `created_at` and verify that the erasures started and ended in the expected order (no overlaps)
+ assert [
+ (log.collection_name, log.status.value)
+ for log in erasure_execution_logs(db, privacy_request)
+ ] == [
+ ("products", "in_processing"),
+ ("products", "complete"),
+ ("orders_to_refunds", "in_processing"),
+ ("orders_to_refunds", "complete"),
+ ("refunds_to_orders", "in_processing"),
+ ("refunds_to_orders", "complete"),
+ ("orders", "in_processing"),
+ ("orders", "complete"),
+ ("refunds", "in_processing"),
+ ("refunds", "complete"),
+ ("labels", "in_processing"),
+ ("labels", "complete"),
+ ]
+
+ CONFIG.execution.masking_strict = temp_masking
+
+
+@pytest.mark.integration_saas
+@pytest.mark.asyncio
+async def test_saas_erasure_order_request_task_with_cycle(
+ db,
+ policy,
+ erasure_policy_complete_mask,
+ saas_erasure_order_config,
+ saas_erasure_order_connection_config,
+ saas_erasure_order_dataset_config,
+) -> None:
+ privacy_request = PrivacyRequest(
+ id=f"test_saas_erasure_order_request_task_with_cycle_{random.randint(0, 1000)}"
+ )
+ identity_attribute = "email"
+ identity_value = "test@ethyca.com"
+ identity_kwargs = {identity_attribute: identity_value}
+ identity = Identity(**identity_kwargs)
+ privacy_request.cache_identity(identity)
+
+ # add a dependency on labels to be erased before orders to create a non-traversable cycle
+ # this won't affect the access traversal
+ dataset_name = saas_erasure_order_connection_config.get_saas_config().fides_key
+ saas_erasure_order_config["endpoints"][0]["erase_after"].append(
+ f"{dataset_name}.labels"
+ )
+ saas_erasure_order_connection_config.update(
+ db, data={"saas_config": saas_erasure_order_config}
+ )
+ merged_graph = saas_erasure_order_dataset_config.get_graph()
+ graph = DatasetGraph(merged_graph)
+
+ v = await graph_task.run_access_request(
+ privacy_request,
+ policy,
+ graph,
+ [saas_erasure_order_connection_config],
+ {"email": "test@ethyca.com"},
+ db,
+ )
+
+ assert_rows_match(v[f"{dataset_name}:orders"], min_size=1, keys=["orders_id"])
+ assert_rows_match(v[f"{dataset_name}:refunds"], min_size=1, keys=["refunds_id"])
+ assert_rows_match(v[f"{dataset_name}:labels"], min_size=1, keys=["labels_id"])
+ assert_rows_match(v[f"{dataset_name}:products"], min_size=1, keys=["products_id"])
+ assert_rows_match(
+ v[f"{dataset_name}:orders_to_refunds"],
+ min_size=1,
+ keys=["orders_to_refunds_id"],
+ )
+ assert_rows_match(
+ v[f"{dataset_name}:refunds_to_orders"],
+ min_size=1,
+ keys=["refunds_to_orders_id"],
+ )
+
+ temp_masking = CONFIG.execution.masking_strict
+ CONFIG.execution.masking_strict = False
+
+ with pytest.raises(TraversalError) as exc:
+ await graph_task.run_erasure(
+ privacy_request,
+ erasure_policy_complete_mask,
+ graph,
+ [saas_erasure_order_connection_config],
+ identity_kwargs,
+ get_cached_data_for_erasures(privacy_request.id),
+ db,
+ )
+
+ assert (
+ f"The values for the `erase_after` fields caused a cycle in the following collections"
+ in str(exc.value)
+ )
+
+ CONFIG.execution.masking_strict = temp_masking
+
+
+@pytest.mark.integration_saas
+@pytest.mark.asyncio
+@mock.patch("fides.api.ops.service.connectors.saas_connector.SaaSConnector.mask_data")
+async def test_saas_erasure_order_request_task_resume_from_error(
+ mock_mask_data,
+ db,
+ policy,
+ erasure_policy_complete_mask,
+ saas_erasure_order_connection_config,
+ saas_erasure_order_dataset_config,
+) -> None:
+ privacy_request = PrivacyRequest(
+ id=f"test_saas_erasure_order_request_task_resume_from_error_{random.randint(0, 1000)}"
+ )
+ identity_attribute = "email"
+ identity_value = "test@ethyca.com"
+ identity_kwargs = {identity_attribute: identity_value}
+ identity = Identity(**identity_kwargs)
+ privacy_request.cache_identity(identity)
+
+ dataset_name = saas_erasure_order_connection_config.get_saas_config().fides_key
+ merged_graph = saas_erasure_order_dataset_config.get_graph()
+ graph = DatasetGraph(merged_graph)
+
+ v = await graph_task.run_access_request(
+ privacy_request,
+ policy,
+ graph,
+ [saas_erasure_order_connection_config],
+ {"email": "test@ethyca.com"},
+ db,
+ )
+
+ assert_rows_match(v[f"{dataset_name}:orders"], min_size=1, keys=["orders_id"])
+ assert_rows_match(v[f"{dataset_name}:refunds"], min_size=1, keys=["refunds_id"])
+ assert_rows_match(v[f"{dataset_name}:labels"], min_size=1, keys=["labels_id"])
+ assert_rows_match(v[f"{dataset_name}:products"], min_size=1, keys=["products_id"])
+ assert_rows_match(
+ v[f"{dataset_name}:orders_to_refunds"],
+ min_size=1,
+ keys=["orders_to_refunds_id"],
+ )
+ assert_rows_match(
+ v[f"{dataset_name}:refunds_to_orders"],
+ min_size=1,
+ keys=["refunds_to_orders_id"],
+ )
+
+ temp_masking = CONFIG.execution.masking_strict
+ CONFIG.execution.masking_strict = False
+
+ # mock the mask_data function so we can force an exception on the "refunds_to_orders"
+ # collection to simulate resuming from error
+ def side_effect(node, policy, privacy_request, rows, input_data):
+ if node.address.collection == "refunds_to_orders":
+ raise Exception("Error executing refunds_to_orders task")
+ return 1
+
+ mock_mask_data.side_effect = side_effect
+
+ with pytest.raises(Exception):
+ await graph_task.run_erasure(
+ privacy_request,
+ erasure_policy_complete_mask,
+ graph,
+ [saas_erasure_order_connection_config],
+ identity_kwargs,
+ get_cached_data_for_erasures(privacy_request.id),
+ db,
+ )
+
+ # "fix" the refunds_to_orders collection and resume the erasure
+ mock_mask_data.side_effect = (
+ lambda node, policy, privacy_request, rows, input_data: 1
+ )
+
+ x = await graph_task.run_erasure(
+ privacy_request,
+ erasure_policy_complete_mask,
+ graph,
+ [saas_erasure_order_connection_config],
+ identity_kwargs,
+ get_cached_data_for_erasures(privacy_request.id),
+ db,
+ )
+
+ assert x == {
+ f"{dataset_name}:orders": 1,
+ f"{dataset_name}:refunds": 1,
+ f"{dataset_name}:labels": 1,
+ f"{dataset_name}:products": 0,
+ f"{dataset_name}:orders_to_refunds": 1,
+ f"{dataset_name}:refunds_to_orders": 1,
+ }
+
+ assert [
+ (log.collection_name, log.status.value)
+ for log in erasure_execution_logs(db, privacy_request)
+ ] == [
+ ("products", "in_processing"),
+ ("products", "complete"),
+ ("orders_to_refunds", "in_processing"),
+ ("orders_to_refunds", "complete"),
+ ("refunds_to_orders", "in_processing"),
+ ("refunds_to_orders", "error"),
+ ("refunds_to_orders", "in_processing"),
+ ("refunds_to_orders", "complete"),
+ ("orders", "in_processing"),
+ ("orders", "complete"),
+ ("refunds", "in_processing"),
+ ("refunds", "complete"),
+ ("labels", "in_processing"),
+ ("labels", "complete"),
+ ], "Cached collections were not re-executed after resuming the privacy request from errored state"
+
+ CONFIG.execution.masking_strict = temp_masking
diff --git a/tests/ops/integration_tests/test_manual_task.py b/tests/ops/integration_tests/test_manual_task.py
index a63668b2ba..a8c26ab2c8 100644
--- a/tests/ops/integration_tests/test_manual_task.py
+++ b/tests/ops/integration_tests/test_manual_task.py
@@ -11,7 +11,7 @@
PrivacyRequest,
)
from fides.api.ops.task import graph_task
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from ..graph.graph_test_util import assert_rows_match
from ..task.traversal_data import postgres_and_manual_nodes
diff --git a/tests/ops/integration_tests/test_sql_task.py b/tests/ops/integration_tests/test_sql_task.py
index 92e2e493dc..fab49b8a77 100644
--- a/tests/ops/integration_tests/test_sql_task.py
+++ b/tests/ops/integration_tests/test_sql_task.py
@@ -26,7 +26,7 @@
from fides.api.ops.task import graph_task
from fides.api.ops.task.filter_results import filter_data_categories
from fides.api.ops.task.graph_task import get_cached_data_for_erasures
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from ..graph.graph_test_util import (
assert_rows_match,
@@ -40,8 +40,6 @@
str_converter,
)
-CONFIG = get_config()
-
sample_postgres_configuration_policy = erasure_policy(
"system.operations",
"user.unique_id",
diff --git a/tests/ops/models/test_client.py b/tests/ops/models/test_client.py
index a3cd00c7f3..9081602e65 100644
--- a/tests/ops/models/test_client.py
+++ b/tests/ops/models/test_client.py
@@ -1,12 +1,10 @@
from sqlalchemy.orm import Session
from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.cryptographic_util import hash_with_salt
from fides.lib.models.client import ClientDetail
-CONFIG = get_config()
-
class TestClientModel:
def test_create_client_and_secret(self, db: Session) -> None:
diff --git a/tests/ops/models/test_consent_request.py b/tests/ops/models/test_consent_request.py
index d7c9c2ccf2..a11d3db336 100644
--- a/tests/ops/models/test_consent_request.py
+++ b/tests/ops/models/test_consent_request.py
@@ -1,10 +1,4 @@
-from datetime import datetime, timedelta, timezone
-from time import sleep
-from typing import List
from unittest import mock
-from uuid import uuid4
-
-import pytest
from fides.api.ctl.database.seed import DEFAULT_CONSENT_POLICY
from fides.api.ops.api.v1.endpoints.consent_request_endpoints import (
@@ -25,12 +19,10 @@
PrivacyRequestResponse,
)
from fides.api.ops.schemas.redis_cache import Identity
-from fides.core.config import get_config
+from fides.core.config import CONFIG
paused_location = CollectionAddress("test_dataset", "test_collection")
-CONFIG = get_config()
-
def test_consent(db):
provided_identity_data = {
@@ -64,6 +56,40 @@ def test_consent(db):
assert Consent.get(db, object_id=consent_2.id) is None
+def test_consent_with_gpc(db):
+ provided_identity_data = {
+ "privacy_request_id": None,
+ "field_name": "email",
+ "encrypted_value": {"value": "test@email.com"},
+ }
+ provided_identity = ProvidedIdentity.create(db, data=provided_identity_data)
+
+ consent_data_1 = {
+ "provided_identity_id": provided_identity.id,
+ "data_use": "user.biometric_health",
+ "opt_in": True,
+ "has_gpc_flag": True,
+ "conflicts_with_gpc": True,
+ }
+ consent_1 = Consent.create(db, data=consent_data_1)
+
+ consent_data_2 = {
+ "provided_identity_id": provided_identity.id,
+ "data_use": "user.browsing_history",
+ "opt_in": False,
+ }
+ consent_2 = Consent.create(db, data=consent_data_2)
+ data_uses = [x.data_use for x in provided_identity.consent]
+
+ assert consent_data_1["data_use"] in data_uses
+ assert consent_data_2["data_use"] in data_uses
+
+ provided_identity.delete(db)
+
+ assert Consent.get(db, object_id=consent_1.id) is None
+ assert Consent.get(db, object_id=consent_2.id) is None
+
+
def test_consent_request(db):
provided_identity_data = {
"privacy_request_id": None,
diff --git a/tests/ops/models/test_privacy_request.py b/tests/ops/models/test_privacy_request.py
index c65356d996..94617f8d92 100644
--- a/tests/ops/models/test_privacy_request.py
+++ b/tests/ops/models/test_privacy_request.py
@@ -28,12 +28,10 @@
from fides.api.ops.service.connectors.manual_connector import ManualAction
from fides.api.ops.util.cache import FidesopsRedis, get_identity_cache_key
from fides.api.ops.util.constants import API_DATE_FORMAT
-from fides.core.config import get_config
+from fides.core.config import CONFIG
paused_location = CollectionAddress("test_dataset", "test_collection")
-CONFIG = get_config()
-
def test_privacy_request(
db: Session, policy: Policy, privacy_request: PrivacyRequest
diff --git a/tests/ops/service/connectors/test_consent_email_connector.py b/tests/ops/service/connectors/test_consent_email_connector.py
index e343446871..866503bba1 100644
--- a/tests/ops/service/connectors/test_consent_email_connector.py
+++ b/tests/ops/service/connectors/test_consent_email_connector.py
@@ -296,11 +296,15 @@ def test_test_connection_call(
"data_use": "Advertising, Marketing or Promotion",
"data_use_description": None,
"opt_in": False,
+ "has_gpc_flag": False,
+ "conflicts_with_gpc": False,
},
{
"data_use": "Improve the capability",
"data_use_description": None,
"opt_in": True,
+ "has_gpc_flag": False,
+ "conflicts_with_gpc": False,
},
],
}
diff --git a/tests/ops/service/connectors/test_saas_queryconfig.py b/tests/ops/service/connectors/test_saas_queryconfig.py
index aaab70c9ed..813ce900d2 100644
--- a/tests/ops/service/connectors/test_saas_queryconfig.py
+++ b/tests/ops/service/connectors/test_saas_queryconfig.py
@@ -14,10 +14,9 @@
from fides.api.ops.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest
from fides.api.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams
from fides.api.ops.service.connectors.saas_query_config import SaaSQueryConfig
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from tests.ops.graph.graph_test_util import generate_node
-CONFIG = get_config()
privacy_request = PrivacyRequest(id="234544")
diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py
index d09f962b0e..d2506ff7c8 100644
--- a/tests/ops/service/messaging/message_dispatch_service_test.py
+++ b/tests/ops/service/messaging/message_dispatch_service_test.py
@@ -9,11 +9,7 @@
from fides.api.ops.graph.config import CollectionAddress
from fides.api.ops.models.messaging import MessagingConfig
from fides.api.ops.models.policy import CurrentStep
-from fides.api.ops.models.privacy_request import (
- CheckpointActionRequired,
- Consent,
- ManualAction,
-)
+from fides.api.ops.models.privacy_request import CheckpointActionRequired, ManualAction
from fides.api.ops.schemas.messaging.messaging import (
ConsentEmailFulfillmentBodyParams,
ConsentPreferencesByUser,
@@ -25,6 +21,7 @@
MessagingServiceType,
SubjectIdentityVerificationBodyParams,
)
+from fides.api.ops.schemas.privacy_request import Consent
from fides.api.ops.schemas.redis_cache import Identity
from fides.api.ops.service.messaging.message_dispatch_service import (
_get_dispatcher_from_config_type,
@@ -32,9 +29,7 @@
_twilio_sms_dispatcher,
dispatch_message,
)
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
@pytest.mark.unit
diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py
index 47c9868202..f0342efb02 100644
--- a/tests/ops/service/privacy_request/request_runner_service_test.py
+++ b/tests/ops/service/privacy_request/request_runner_service_test.py
@@ -60,10 +60,9 @@
upload_access_results,
)
from fides.api.ops.util.data_category import DataCategory
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.models.audit_log import AuditLog, AuditLogAction
-CONFIG = get_config()
PRIVACY_REQUEST_TASK_TIMEOUT = 5
# External services take much longer to return
PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL = 30
diff --git a/tests/ops/service/privacy_request/test_consent_email_batch_send.py b/tests/ops/service/privacy_request/test_consent_email_batch_send.py
index 45fcc73e20..ec6ddb1f1c 100644
--- a/tests/ops/service/privacy_request/test_consent_email_batch_send.py
+++ b/tests/ops/service/privacy_request/test_consent_email_batch_send.py
@@ -277,7 +277,11 @@ def test_send_consent_email_multiple_users(
identities={"ljt_readerID": "12345"},
consent_preferences=[
Consent(
- data_use="advertising", data_use_description=None, opt_in=False
+ data_use="advertising",
+ data_use_description=None,
+ opt_in=False,
+ conflicts_with_gpc=False,
+ has_gpc_flag=False,
)
],
),
@@ -285,7 +289,11 @@ def test_send_consent_email_multiple_users(
identities={"ljt_readerID": "abcde"},
consent_preferences=[
Consent(
- data_use="advertising", data_use_description=None, opt_in=False
+ data_use="advertising",
+ data_use_description=None,
+ opt_in=False,
+ conflicts_with_gpc=False,
+ has_gpc_flag=False,
)
],
),
@@ -387,6 +395,8 @@ def test_add_user_preferences_to_email_data(
"data_use": "Advertising, Marketing or Promotion",
"data_use_description": None,
"opt_in": False,
+ "has_gpc_flag": False,
+ "conflicts_with_gpc": False,
},
],
}
diff --git a/tests/ops/service/privacy_request/test_request_service.py b/tests/ops/service/privacy_request/test_request_service.py
index 22d6f83640..991cbe9631 100644
--- a/tests/ops/service/privacy_request/test_request_service.py
+++ b/tests/ops/service/privacy_request/test_request_service.py
@@ -7,11 +7,9 @@
from fides.api.ops.service.privacy_request.request_service import (
poll_server_for_completion,
)
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.cryptographic_util import str_to_b64_str
-CONFIG = get_config()
-
@pytest.fixture
def parent_user_config():
diff --git a/tests/ops/service/storage_uploader_service_test.py b/tests/ops/service/storage_uploader_service_test.py
index 1cb5a820e4..2de4e73636 100644
--- a/tests/ops/service/storage_uploader_service_test.py
+++ b/tests/ops/service/storage_uploader_service_test.py
@@ -32,9 +32,7 @@
decrypt,
decrypt_combined_nonce_and_message,
)
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
@mock.patch("fides.api.ops.service.storage.storage_uploader_service.upload_to_s3")
diff --git a/tests/ops/task/test_graph_task.py b/tests/ops/task/test_graph_task.py
index 30fe6c5765..1580ed891f 100644
--- a/tests/ops/task/test_graph_task.py
+++ b/tests/ops/task/test_graph_task.py
@@ -1,20 +1,39 @@
-import dask
+from typing import Any, Dict
+
import pytest
from bson import ObjectId
-from fides.api.ops.graph.config import CollectionAddress, FieldPath
+from fides.api.ops.graph.config import (
+ ROOT_COLLECTION_ADDRESS,
+ TERMINATOR_ADDRESS,
+ Collection,
+ CollectionAddress,
+ FieldPath,
+ GraphDataset,
+)
from fides.api.ops.graph.graph import DatasetGraph
-from fides.api.ops.graph.traversal import Traversal
+from fides.api.ops.graph.traversal import Traversal, TraversalNode
from fides.api.ops.models.connectionconfig import ConnectionConfig, ConnectionType
from fides.api.ops.models.policy import ActionType, Policy, Rule, RuleTarget
from fides.api.ops.task.graph_task import (
EMPTY_REQUEST,
+ GraphTask,
TaskResources,
+ _evaluate_erasure_dependencies,
build_affected_field_logs,
collect_queries,
+ start_function,
+ update_erasure_mapping_from_cache,
)
-from ..graph.graph_test_util import MockMongoTask, MockSqlTask, erasure_policy, field
+from ..graph.graph_test_util import (
+ MockMongoTask,
+ MockSqlTask,
+ collection,
+ erasure_policy,
+ field,
+ generate_field_list,
+)
from .traversal_data import (
combined_mongo_postgresql_graph,
sample_traversal,
@@ -555,3 +574,88 @@ def test_multiple_rules_targeting_same_field(self, node_fixture):
"data_categories": ["A"],
}
]
+
+
+class TestUpdateErasureMappingFromCache:
+ @pytest.fixture(scope="function")
+ def task_resource(self, privacy_request, policy, db):
+ tr = TaskResources(privacy_request, policy, [], db)
+ tr.get_connector = lambda x: True
+ return tr
+
+ @pytest.fixture(scope="function")
+ def collect_tasks_fn(self, task_resource):
+ def collect_tasks_fn(
+ tn: TraversalNode, data: Dict[CollectionAddress, GraphTask]
+ ) -> None:
+ """Run the traversal, as an action creating a GraphTask for each traversal_node."""
+ if not tn.is_root_node():
+ data[tn.address] = GraphTask(tn, task_resource)
+
+ return collect_tasks_fn
+
+ @pytest.fixture(scope="function")
+ def dsk(self, collect_tasks_fn) -> Dict[str, Any]:
+ """
+ Creates a Dask graph representing a dataset containing three collections (ds_1, ds_2, ds_3)
+ where the erasure order is ds_3 -> ds_2 -> ds_1
+ """
+ t = [
+ GraphDataset(
+ name=f"dr_1",
+ collections=[
+ Collection(name=f"ds_{i}", fields=generate_field_list(1))
+ for i in range(1, 4)
+ ],
+ connection_key="mock_connection_config_key",
+ )
+ ]
+
+ # the collections are not dependent on each other for access
+ field(t, "dr_1", "ds_1", "f1").identity = "email"
+ field(t, "dr_1", "ds_2", "f1").identity = "email"
+ field(t, "dr_1", "ds_3", "f1").identity = "email"
+ collection(t, CollectionAddress("dr_1", "ds_2")).erase_after = [
+ CollectionAddress("dr_1", "ds_1")
+ ]
+ collection(t, CollectionAddress("dr_1", "ds_3")).erase_after = [
+ CollectionAddress("dr_1", "ds_2")
+ ]
+ graph: DatasetGraph = DatasetGraph(*t)
+ traversal: Traversal = Traversal(graph, {"email": {"test_user@example.com"}})
+ env: Dict[CollectionAddress, Any] = {}
+ traversal.traverse(env, collect_tasks_fn)
+ erasure_end_nodes = list(graph.nodes.keys())
+
+ # the [] and [[]] values don't matter for this test, we just need to verify that they are not modified
+ dsk: Dict[CollectionAddress, Any] = {
+ k: (
+ t.erasure_request,
+ [],
+ [[]],
+ *_evaluate_erasure_dependencies(t, erasure_end_nodes),
+ )
+ for k, t in env.items()
+ }
+ dsk[TERMINATOR_ADDRESS] = (lambda x: x, *erasure_end_nodes)
+ dsk[ROOT_COLLECTION_ADDRESS] = 0
+ return dsk
+
+ def test_update_erasure_mapping_from_cache_without_data(self, dsk, task_resource):
+ task_resource.get_all_cached_erasures = lambda: {} # represents an empty cache
+ update_erasure_mapping_from_cache(dsk, task_resource)
+ (task, retrieved_data, input_list, *erasure_prereqs) = dsk[
+ CollectionAddress("dr_1", "ds_1")
+ ]
+ assert callable(task)
+ assert task.__name__ == "erasure_request"
+ assert retrieved_data == []
+ assert input_list == [[]]
+ assert erasure_prereqs == [ROOT_COLLECTION_ADDRESS]
+
+ def test_update_erasure_mapping_from_cache_with_data(self, dsk, task_resource):
+ task_resource.get_all_cached_erasures = lambda: {
+ "dr_1:ds_1": 1
+ } # a cache with the results of the ds_1 collection erasure
+ update_erasure_mapping_from_cache(dsk, task_resource)
+ assert dsk[CollectionAddress("dr_1", "ds_1")] == 1
diff --git a/tests/ops/tasks/test_celery.py b/tests/ops/tasks/test_celery.py
index 06bcd0c4f0..d672f6a12f 100644
--- a/tests/ops/tasks/test_celery.py
+++ b/tests/ops/tasks/test_celery.py
@@ -5,9 +5,7 @@
from sqlalchemy.pool import QueuePool
from fides.api.ops.tasks import DatabaseTask, _create_celery
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG, get_config
@pytest.fixture
@@ -34,7 +32,8 @@ def multiply(x, y):
def test_celery_default_config() -> None:
- celery_app = _create_celery()
+ config = get_config()
+ celery_app = _create_celery(config)
assert celery_app.conf["broker_url"] == CONFIG.redis.connection_url
assert celery_app.conf["result_backend"] == CONFIG.redis.connection_url
assert celery_app.conf["event_queue_prefix"] == "fides_worker"
@@ -43,6 +42,7 @@ def test_celery_default_config() -> None:
def test_celery_config_override() -> None:
config = get_config()
+
config.celery["event_queue_prefix"] = "overridden_fides_worker"
config.celery["task_default_queue"] = "overridden_fides"
diff --git a/tests/ops/tasks/test_scheduled.py b/tests/ops/tasks/test_scheduled.py
index 023ad59dde..9e1461e352 100644
--- a/tests/ops/tasks/test_scheduled.py
+++ b/tests/ops/tasks/test_scheduled.py
@@ -28,7 +28,7 @@ def test_initiate_scheduled_paused_privacy_request_followup(
def test_initiate_batch_consent_email_send() -> None:
- CONFIG.is_test_mode = False
+ CONFIG.test_mode = False
initiate_scheduled_batch_consent_email_send()
assert scheduler.running
@@ -44,4 +44,4 @@ def test_initiate_batch_consent_email_send() -> None:
assert type(job.trigger.timezone).__name__ == "US/Eastern"
- CONFIG.is_test_mode = True
+ CONFIG.test_mode = True
diff --git a/tests/ops/util/test_cache.py b/tests/ops/util/test_cache.py
index 23afe89f13..33c0c8319e 100644
--- a/tests/ops/util/test_cache.py
+++ b/tests/ops/util/test_cache.py
@@ -1,7 +1,7 @@
import pickle
import random
from base64 import b64encode
-from datetime import datetime, timezone
+from datetime import datetime
from enum import Enum
from typing import Any, List
@@ -13,12 +13,10 @@
ENCODED_MONGO_OBJECT_ID_PREFIX,
FidesopsRedis,
)
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from ..fixtures.application_fixtures import faker
-CONFIG = get_config()
-
def test_get_cache(cache: FidesopsRedis) -> None:
assert cache is not None
diff --git a/tests/ops/util/test_jwt_util.py b/tests/ops/util/test_jwt_util.py
index ff41443243..330ce6a041 100644
--- a/tests/ops/util/test_jwt_util.py
+++ b/tests/ops/util/test_jwt_util.py
@@ -1,7 +1,7 @@
import json
from datetime import datetime
-from fides.core.config import get_config
+from fides.core.config import CONFIG
from fides.lib.cryptography.schemas.jwt import (
JWE_ISSUED_AT,
JWE_PAYLOAD_CLIENT_ID,
@@ -10,8 +10,6 @@
from fides.lib.oauth.jwt import generate_jwe
from fides.lib.oauth.oauth_util import extract_payload, is_token_expired
-CONFIG = get_config()
-
def test_jwe_create_and_extract() -> None:
payload = {"hello": "hi there"}
diff --git a/tests/ops/util/test_logger.py b/tests/ops/util/test_logger.py
index 3a8168a19c..65c6a58bc1 100644
--- a/tests/ops/util/test_logger.py
+++ b/tests/ops/util/test_logger.py
@@ -3,9 +3,7 @@
import pytest
from fides.api.ops.util.logger import MASKED, Pii, _log_exception, _log_warning
-from fides.core.config import get_config
-
-CONFIG = get_config()
+from fides.core.config import CONFIG
@pytest.mark.unit