Merge branch 'main' into adobe-sign-test

.fides/db_dataset.yml

@@ -1047,6 +1047,9 @@ dataset:
     - name: state
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: referer
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
     - name: updated_at
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified

CHANGELOG.md

@@ -18,10 +18,28 @@ The types of changes are:
 ## [Unreleased](https://github.com/ethyca/fides/compare/2.17.0...main)
 
 ### Added
+- Additional consent reporting calls from `fides-js` [#3845](https://github.com/ethyca/fides/pull/3845)
+- Additional consent reporting calls from privacy center [#3847](https://github.com/ethyca/fides/pull/3847)
+- Access support for Recurly [#3595](https://github.com/ethyca/fides/pull/3595)
 - Access support for Adobe Sign [#3504](https://github.com/ethyca/fides/pull/3504)
 
+### Fixed
+- Fix datamap zoom for low system counts [#3835](https://github.com/ethyca/fides/pull/3835)
+- Fixed connector forms with external dataset reference fields [#3873](https://github.com/ethyca/fides/pull/3873)
+
+### Added
+
+- HTTP Logging for the Privacy Center [#3783](https://github.com/ethyca/fides/pull/3783)
+- UI support for OAuth2 authorization flow [#3819](https://github.com/ethyca/fides/pull/3819)
+- Changes in the `data` directory now trigger a server reload (for local development) [#3874](https://github.com/ethyca/fides/pull/3874)
+
 ### Changed
+
 - Simplified the file structure for HTML DSR packages [#3848](https://github.com/ethyca/fides/pull/3848)
+- Simplified the database health check to improve `/health` performance [#3884](https://github.com/ethyca/fides/pull/3884)
+- Changed max width of form components in "system information" form tab [#3864](https://github.com/ethyca/fides/pull/3864)
+- Remove manual system selection screen [#3865](https://github.com/ethyca/fides/pull/3865)
+- System and integration identifiers are now auto-generated [#3868](https://github.com/ethyca/fides/pull/3868)
 
 ## [2.17.0](https://github.com/ethyca/fides/compare/2.16.0...2.17.0)
 

clients/README.md

@@ -2,50 +2,4 @@
 
 The clients directory houses all front-end packages and shared code amongst clients, and also includes e2e tests.
 
-## Prerequisites
-
-Dependencies within this directory are managed by Turborepo. Our root `package.json` file defines 3 workspaces (or packages) that are part of the Turbo ecosystem:
-1. admin-ui
-2. privacy-center
-3. fides-js
-
-To run any or all workspaces, you'll have to complete the following setup: 
-
-1. Install Turborepo globally so that you can use it on the command line `npm install turbo --global`
-2. In `clients` folder, run `npm install`. This will install the dev dependency `turbo`. This will also install the appropriate dependencies in `clients` and in each client within `clients` folder. You may need to first remove all `node_modules` in each of the clients (admin-ui, privacy-center, etc)
-
-## Running Locally
-
-To run the project use `turbo` instead of `npm`. 
-
-Incorrect:
-```
-npm run dev
-```
-
-Correct:
-```
-turbo run dev
-```
-
-Running this in the root `clients` folder will result in every workspace being run.
-
-Running this command within `admin-ui` will result in only admin-ui being run.
-
-Available commands that exist for every workspace are defined in the root `turbo.json` file, while commands unique to a specific workspace are defined in the `turbo.json` file within the workspace.
-
-It's important to use the turbo command because, as you see in the `turbo.json` files, we've defined some dependencies and caching details on some turbo commands. 
-
-
-
-## Adding packages
-
-To install packages in any package in `clients`, run the following from `clients`:
-```
-npm install <package> --workspace=<workspace>
-```
-Example:
-```
-npm install react --workspace=admin-ui
-```
-See https://turbo.build/repo/docs/handbook/package-installation#addingremovingupgrading-packages for more details
+See the [UI Contribution Guide](http://localhost:8000/fides/development/ui/overview/) for more information

clients/admin-ui/cypress/e2e/systems.cy.ts

@@ -74,41 +74,6 @@ describe("System management page", () => {
         }).as("getConnectionTypes");
       });
 
-      it("shows available system types and lets the user choose one", () => {
-        cy.visit(ADD_SYSTEMS_ROUTE);
-        cy.getByTestId("manual-btn").click();
-        cy.url().should("contain", ADD_SYSTEMS_MANUAL_ROUTE);
-        cy.wait("@getConnectionTypes");
-        cy.getByTestId("header").contains("Choose a type of system");
-        cy.getByTestId("bigquery-item");
-        cy.getByTestId("mariadb-item");
-        // Click into one of the connectors
-        cy.getByTestId("mongodb-item").click();
-        cy.getByTestId("header").contains("Describe your MongoDB system");
-
-        // Go back to choosing to add a new type of system
-        cy.getByTestId("breadcrumbs").contains("Choose your system").click();
-        cy.getByTestId("create-system-btn").click();
-        cy.getByTestId("header").contains("Describe your new system");
-      });
-
-      it("should allow searching", () => {
-        cy.visit(ADD_SYSTEMS_MANUAL_ROUTE);
-        cy.wait("@getConnectionTypes");
-        cy.getByTestId("bigquery-item");
-        cy.getByTestId("system-catalog-search").type("db");
-        cy.getByTestId("bigquery-item").should("not.exist");
-        cy.getByTestId("mariadb-item");
-        cy.getByTestId("mongodb-item");
-        cy.getByTestId("timescale-item");
-
-        // empty state
-        cy.getByTestId("system-catalog-search")
-          .clear()
-          .type("a very specific system that we do not have");
-        cy.getByTestId("no-systems-found");
-      });
-
       it("Can step through the flow", () => {
         cy.fixture("systems/system.json").then((system) => {
           cy.intercept("GET", "/api/v1/system/*", {
@@ -119,10 +84,7 @@ describe("System management page", () => {
           cy.getByTestId("manual-btn").click();
           cy.url().should("contain", ADD_SYSTEMS_MANUAL_ROUTE);
           cy.wait("@getSystems");
-          cy.wait("@getConnectionTypes");
-          cy.getByTestId("create-system-btn").click();
           cy.getByTestId("input-name").type(system.name);
-          cy.getByTestId("input-fides_key").type(system.fides_key);
           cy.getByTestId("input-description").type(system.description);
 
           cy.getByTestId("save-btn").click();
@@ -190,10 +152,7 @@ describe("System management page", () => {
           }).as("getDemoSystem");
           cy.visit(ADD_SYSTEMS_MANUAL_ROUTE);
           cy.wait("@getSystems");
-          cy.wait("@getConnectionTypes");
-          cy.getByTestId("create-system-btn").click();
           cy.getByTestId("input-name").type(system.name);
-          cy.getByTestId("input-fides_key").type(system.fides_key);
           cy.getByTestId("input-description").type(system.description);
           cy.getByTestId("save-btn").click();
           cy.wait("@postSystem");

clients/admin-ui/package.json

@@ -27,6 +27,7 @@
     "test:ci": "jest"
   },
   "dependencies": {
+    "@chakra-ui/icons": "1.1.7",
     "@emotion/react": "^11.10.6",
     "@emotion/styled": "^11.10.6",
     "@fidesui/components": "^0.7.1",
@@ -35,7 +36,6 @@
     "@monaco-editor/react": "^4.4.6",
     "@reduxjs/toolkit": "^1.9.3",
     "chakra-react-select": "^3.3.7",
-    "@chakra-ui/icons": "1.1.7",
     "csv-stringify": "^6.3.0",
     "cytoscape": "^3.23.0",
     "cytoscape-klay": "^3.1.4",
@@ -46,6 +46,7 @@
     "i18n-iso-countries": "^7.5.0",
     "immer": "^9.0.21",
     "js-yaml": "^4.1.0",
+    "lodash": "^4.17.21",
     "lodash.snakecase": "^4.1.1",
     "msw": "^1.2.1",
     "narrow-minded": "^1.2.1",

clients/admin-ui/src/features/datamap/CytoscapeGraph.tsx

@@ -154,6 +154,14 @@ const CytoscapeGraph = ({
     };
     if (datamapGraphRef.current) {
       datamapGraphRef.current.on("click", "node", setNode);
+      datamapGraphRef.current.on("layoutstop", () => {
+        // solution found here: https://github.com/cytoscape/cytoscape.js/issues/941#issuecomment-104501028
+        if (datamapGraphRef.current && elements.length < 5) {
+          datamapGraphRef.current.maxZoom(2.5);
+          datamapGraphRef.current.fit();
+          datamapGraphRef.current.maxZoom(100);
+        }
+      });
     }
 
     return () => {

clients/admin-ui/src/features/datastore-connections/add-connection/ConnectionTypeList.tsx

@@ -35,6 +35,7 @@ const ConnectionTypeList: React.FC<ConnectionTypeListProps> = ({ items }) => (
             alignItems="center"
             justifyContent="start"
             pl="24px"
+            pr="24px"
             color="gray.700"
             fontSize="14px"
             fontStyle="normal"

clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts

@@ -172,6 +172,13 @@ export const { reducer } = datastoreConnectionSlice;
 
 export const datastoreConnectionApi = baseApi.injectEndpoints({
   endpoints: (build) => ({
+    getAuthorizationUrl: build.query<string, string>({
+      query: (connectionKey) => ({
+        url: `${CONNECTION_ROUTE}/${connectionKey}/authorize`,
+        method: "GET",
+      }),
+      providesTags: () => ["Datastore Connection"],
+    }),
     createAccessManualWebhook: build.mutation<
       CreateAccessManualWebhookResponse,
       CreateAccessManualWebhookRequest
@@ -368,6 +375,7 @@ export const datastoreConnectionApi = baseApi.injectEndpoints({
 });
 
 export const {
+  useLazyGetAuthorizationUrlQuery,
   useCreateAccessManualWebhookMutation,
   useCreateSassConnectionConfigMutation,
   useCreateUnlinkedSassConnectionConfigMutation,

clients/admin-ui/src/features/datastore-connections/system_portal_config/DeleteConnectionModal.tsx

@@ -47,7 +47,7 @@ const DeleteConnectionModal: React.FC<DataConnectionProps> = ({
       <>
         <Spacer />
         <Flex alignItems="center">
-          <Text fontSize="sm">Delete Integration</Text>
+          <Text fontSize="sm">Delete integration</Text>
           <IconButton
             marginLeft="8px"
             aria-label="Delete integration"
@@ -63,7 +63,7 @@ const DeleteConnectionModal: React.FC<DataConnectionProps> = ({
       <Modal isCentered isOpen={isOpen} onClose={closeIfComplete}>
         <ModalOverlay />
         <ModalContent>
-          <ModalHeader>Delete Integration</ModalHeader>
+          <ModalHeader>Delete integration</ModalHeader>
           <ModalCloseButton />
           <ModalBody pb={6}>
             <Stack direction="column" spacing="15px">

clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParameters.tsx

@@ -6,6 +6,7 @@ import {
   CreateSaasConnectionConfig,
   useCreateSassConnectionConfigMutation,
   useGetConnectionConfigDatasetConfigsQuery,
+  useLazyGetAuthorizationUrlQuery,
 } from "datastore-connections/datastore-connection.slice";
 import { useDatasetConfigField } from "datastore-connections/system_portal_config/forms/fields/DatasetConfigField/DatasetConfigField";
 import {
@@ -18,7 +19,6 @@ import { useMemo, useState } from "react";
 import { useAppDispatch, useAppSelector } from "~/app/hooks";
 import { DEFAULT_TOAST_PARAMS } from "~/features/common/toast";
 import { useGetConnectionTypeSecretSchemaQuery } from "~/features/connection-type";
-import { formatKey } from "~/features/datastore-connections/system_portal_config/helpers";
 import TestConnectionMessage from "~/features/datastore-connections/system_portal_config/TestConnectionMessage";
 import TestData from "~/features/datastore-connections/TestData";
 import {
@@ -44,6 +44,23 @@ import ConnectorParametersForm, {
   TestConnectionResponse,
 } from "./ConnectorParametersForm";
 
+const generateIntegrationKey = (
+  systemFidesKey: string,
+  connectionOption: ConnectionSystemTypeMap
+): string => {
+  let integrationKey = systemFidesKey;
+
+  if (!systemFidesKey.includes(connectionOption.identifier)) {
+    integrationKey += `_${connectionOption.identifier}`;
+  }
+
+  if (connectionOption.type === SystemType.SAAS) {
+    integrationKey += "_api";
+  }
+
+  return integrationKey;
+};
+
 /**
  * Only handles creating saas connectors. The BE handler automatically
  * configures the connector using the saas config and creates the
@@ -58,7 +75,7 @@ const createSaasConnector = async (
 ) => {
   const connectionConfig: Omit<CreateSaasConnectionConfigRequest, "name"> = {
     description: values.description,
-    instance_key: formatKey(values.instance_key as string),
+    instance_key: generateIntegrationKey(systemFidesKey, connectionOption),
     saas_connector_type: connectionOption.identifier,
     secrets: {},
   };
@@ -88,12 +105,9 @@ export const patchConnectionConfig = async (
   connectionConfig: ConnectionConfigurationResponse,
   patchFunc: any
 ) => {
-  const key =
-    [SystemType.DATABASE, SystemType.EMAIL, SystemType.MANUAL].indexOf(
-      connectionOption.type
-    ) > -1
-      ? formatKey(values.instance_key as string)
-      : connectionConfig?.key;
+  const key = connectionConfig
+    ? connectionConfig.key
+    : generateIntegrationKey(systemFidesKey, connectionOption);
 
   const params1: Omit<ConnectionConfigurationResponse, "created_at" | "name"> =
     {
@@ -183,6 +197,7 @@ export const useConnectorForm = ({
   const dispatch = useAppDispatch();
 
   const [isSubmitting, setIsSubmitting] = useState(false);
+  const [isAuthorizing, setIsAuthorizing] = useState(false);
 
   const {
     dropdownOptions: datasetDropdownOptions,
@@ -194,6 +209,7 @@ export const useConnectorForm = ({
   });
 
   const [createSassConnectionConfig] = useCreateSassConnectionConfigMutation();
+  const [getAuthorizationUrl] = useLazyGetAuthorizationUrlQuery();
   const [updateSystemConnectionSecrets] =
     usePatchSystemConnectionSecretsMutation();
   const [patchDatastoreConnection] = usePatchSystemConnectionConfigsMutation();
@@ -304,9 +320,51 @@ export const useConnectorForm = ({
     }
   };
 
+  const handleAuthorization = async (values: ConnectionConfigFormValues) => {
+    const isCreatingConnectionConfig = !connectionConfig;
+    try {
+      setIsAuthorizing(true);
+      if (isCreatingConnectionConfig) {
+        const response = await createSaasConnector(
+          values, // pre-process dataset references
+          secretsSchema!,
+          connectionOption,
+          systemFidesKey,
+          createSassConnectionConfig
+        );
+        // eslint-disable-next-line no-param-reassign
+        connectionConfig = response.connection;
+      } else {
+        await upsertConnectionConfigSecrets(
+          values,
+          secretsSchema!,
+          systemFidesKey,
+          originalSecrets,
+          updateSystemConnectionSecrets
+        );
+      }
+      const authorizationUrl = (await getAuthorizationUrl(
+        connectionConfig!.key
+      ).unwrap()) as string;
+
+      setIsAuthorizing(false);
+
+      // workaround to make sure isAuthorizing is set to false before redirecting
+      setTimeout(() => {
+        window.location.href = authorizationUrl;
+      }, 0);
+    } catch (error) {
+      handleError(error);
+    } finally {
+      setIsAuthorizing(false);
+    }
+  };
+
   return {
     isSubmitting,
+    isAuthorizing,
     handleSubmit,
+    handleAuthorization,
     datasetDropdownOptions,
     selectedDatasetConfigOption,
     handleDelete,
@@ -346,7 +404,9 @@ export const ConnectorParameters: React.FC<ConnectorParametersProps> = ({
 
   const {
     isSubmitting,
+    isAuthorizing,
     handleSubmit,
+    handleAuthorization,
     datasetDropdownOptions,
     selectedDatasetConfigOption,
     handleDelete,
@@ -391,8 +451,10 @@ export const ConnectorParameters: React.FC<ConnectorParametersProps> = ({
         secretsSchema={secretsSchema}
         defaultValues={defaultValues}
         isSubmitting={isSubmitting}
+        isAuthorizing={isAuthorizing}
         onSaveClick={handleSubmit}
         onTestConnectionClick={handleTestConnectionClick}
+        onAuthorizeConnectionClick={handleAuthorization}
         connectionOption={connectionOption}
         connectionConfig={connectionConfig}
         datasetDropdownOptions={datasetDropdownOptions}