Merge branch 'main' into 3518_ajackson_remove_dataset_from_saas

CHANGELOG.md

@@ -58,7 +58,8 @@ The types of changes are:
 - Add ability to close `fides-js` banner and modal via a button or ESC [#3411](https://github.com/ethyca/fides/pull/3411)
 - Add ability to open the `fides-js` modal from a link on the host site [#3411](https://github.com/ethyca/fides/pull/3411)
 - GPC preferences are automatically applied via `fides-js` [#3411](https://github.com/ethyca/fides/pull/3411)
-- Add new dataset route that has additinonal filters [#3558](https://github.com/ethyca/fides/pull/3558)
+- Add new dataset route that has additional filters [#3558](https://github.com/ethyca/fides/pull/3558)
+- Update dataset dropdown to use new api filter [#3565](https://github.com/ethyca/fides/pull/3565)
 
 ### Fixed
 
@@ -70,6 +71,7 @@ The types of changes are:
 - Fix NextJS errors & warnings for Cookie House sample app [#3411](https://github.com/ethyca/fides/pull/3411)
 - Fix bug where `fides-js` toggles were not reflecting changes from rejecting or accepting all notices [#3522](https://github.com/ethyca/fides/pull/3522)
 - Remove the `fides-js` banner from tab order when it is hidden and move the overlay components to the top of the tab order. [#3510](https://github.com/ethyca/fides/pull/3510)
+- Fix bug where `fides-js` toggle states did not always initialize properly [#3597](https://github.com/ethyca/fides/pull/3597)
 
 ### Changed
 

clients/admin-ui/src/features/dataset/dataset.slice.ts

@@ -33,6 +33,16 @@ const datasetApi = baseApi.injectEndpoints({
       query: () => ({ url: `dataset/` }),
       providesTags: () => ["Datasets"],
     }),
+    getAllFilteredDatasets: build.query<
+      Dataset[],
+      { onlyUnlinkedDatasets: boolean }
+    >({
+      query: ({ onlyUnlinkedDatasets }) => ({
+        url: `/filter/dataset`,
+        params: { only_unlinked_datasets: onlyUnlinkedDatasets },
+      }),
+      providesTags: () => ["Datasets"],
+    }),
     getDatasetByKey: build.query<Dataset, string>({
       query: (key) => ({ url: `dataset/${key}` }),
       providesTags: () => ["Dataset"],
@@ -90,6 +100,7 @@ const datasetApi = baseApi.injectEndpoints({
 
 export const {
   useGetAllDatasetsQuery,
+  useGetAllFilteredDatasetsQuery,
   useGetDatasetByKeyQuery,
   useUpdateDatasetMutation,
   useUpsertDatasetsMutation,

clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts

@@ -332,7 +332,7 @@ export const datastoreConnectionApi = baseApi.injectEndpoints({
         method: "PATCH",
         body: params.dataset_pairs,
       }),
-      invalidatesTags: () => ["Datastore Connection"],
+      invalidatesTags: () => ["Datastore Connection", "Datasets"],
     }),
     patchDatastoreConnection: build.mutation<
       DatastoreConnectionResponse,
@@ -343,15 +343,15 @@ export const datastoreConnectionApi = baseApi.injectEndpoints({
         method: "PATCH",
         body: [params],
       }),
-      invalidatesTags: () => ["Datastore Connection"],
+      invalidatesTags: () => ["Datastore Connection", "Datasets"],
     }),
     patchDatastoreConnections: build.mutation({
       query: ({ key, name, disabled, connection_type, access }) => ({
         url: CONNECTION_ROUTE,
         method: "PATCH",
         body: [{ key, name, disabled, connection_type, access }],
       }),
-      invalidatesTags: () => ["Datastore Connection"],
+      invalidatesTags: () => ["Datastore Connection", "Datasets"],
     }),
     updateDatastoreConnectionSecrets: build.mutation<
       DatastoreConnectionSecretsResponse,

clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/fields/DatasetConfigField/DatasetConfigField.tsx

@@ -21,7 +21,7 @@ import React, { useCallback, useEffect, useMemo, useState } from "react";
 
 import { getErrorMessage } from "~/features/common/helpers";
 import {
-  useGetAllDatasetsQuery,
+  useGetAllFilteredDatasetsQuery,
   useUpsertDatasetsMutation,
 } from "~/features/dataset";
 import {
@@ -121,11 +121,19 @@ export const useDatasetConfigField = ({
   const [patchDatasetConfig] = usePatchDatasetConfigsMutation();
   const [upsertDatasets] = useUpsertDatasetsMutation();
 
-  const { data: allDatasetConfigs } = useGetConnectionConfigDatasetConfigsQuery(
-    connectionConfig?.key || ""
-  );
+  const { data: allDatasetConfigs, isLoading: isLoadingDatasetConfigs } =
+    useGetConnectionConfigDatasetConfigsQuery(connectionConfig?.key || "");
 
-  const { data: allDatasets } = useGetAllDatasetsQuery();
+  const { data: allDatasets } = useGetAllFilteredDatasetsQuery(
+    {
+      onlyUnlinkedDatasets: allDatasetConfigs
+        ? allDatasetConfigs.items.length === 0
+        : false,
+    },
+    {
+      skip: isLoadingDatasetConfigs,
+    }
+  );
 
   const [datasetConfigFidesKey, setDatasetConfigFidesKey] = useState<
     string | undefined
@@ -219,15 +227,21 @@ export const useDatasetConfigField = ({
 
 type Props = {
   dropdownOptions: Option[];
+  connectionConfig?: ConnectionConfigurationResponse;
 };
 
-const DatasetConfigField: React.FC<Props> = ({ dropdownOptions }) => {
+const DatasetConfigField: React.FC<Props> = ({
+  dropdownOptions,
+  connectionConfig,
+}) => {
   const [datasetDropdownOption] = useField<string>(fieldName);
   const [datasetYaml] = useField<Dataset>("datasetYaml");
   const { setFieldValue } = useFormikContext();
   const { isOpen, onOpen, onClose } = useDisclosure();
 
-  const { data: allDatasets, isLoading } = useGetAllDatasetsQuery();
+  const { data: allDatasets, isLoading } = useGetAllFilteredDatasetsQuery({
+    onlyUnlinkedDatasets: !connectionConfig,
+  });
 
   const setDatasetYaml = useCallback<(value: Dataset) => void>(
     (value) => {

clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/fields/DatasetConfigField/YamlEditor.tsx

@@ -40,6 +40,10 @@ const YamlEditor: React.FC<YamlEditorFormProps> = ({
   );
   const [isTouched, setIsTouched] = useState(false);
   const [isEmptyState, setIsEmptyState] = useState(!yamlData);
+  /*
+  We need to get all the datasets, including saas datasets, so that we can verify that
+  the fides_key is not already in use.
+   */
   const { data: allDatasets } = useGetAllDatasetsQuery();
   const [overWrittenKeys, setOverWrittenKeys] = useState<string[]>([]);
 

clients/fides-js/src/components/Overlay.tsx

@@ -42,12 +42,10 @@ const Overlay: FunctionComponent<OverlayProps> = ({
   const [bannerIsOpen, setBannerIsOpen] = useState(false);
 
   const initialEnabledNoticeKeys = useMemo(
-    () =>
-      Object.keys(window.Fides.consent).filter(
-        (key) => window.Fides.consent[key]
-      ),
-    []
+    () => Object.keys(cookie.consent).filter((key) => cookie.consent[key]),
+    [cookie.consent]
   );
+
   const [draftEnabledNoticeKeys, setDraftEnabledNoticeKeys] = useState<
     Array<PrivacyNotice["notice_key"]>
   >(initialEnabledNoticeKeys);

src/fides/api/api/v1/endpoints/connection_endpoints.py

@@ -35,6 +35,9 @@
     ConnectionType,
 )
 from fides.api.models.datasetconfig import DatasetConfig
+from fides.api.models.sql_models import (  # type: ignore[attr-defined]
+    Dataset as CtlDataset,
+)
 from fides.api.oauth.utils import verify_oauth_client
 from fides.api.schemas.connection_configuration import connection_secrets_schemas
 from fides.api.schemas.connection_configuration.connection_config import (
@@ -48,7 +51,6 @@
     TestStatusMessage,
 )
 from fides.api.service.connectors import get_connector
-from fides.api.models.sql_models import Dataset as CtlDataset  # type: ignore[attr-defined]
 from fides.api.util.api_router import APIRouter
 from fides.api.util.connection_util import (
     patch_connection_configs,

tests/ops/api/v1/endpoints/test_dataset_endpoints.py

@@ -19,16 +19,18 @@
     DATASET_CREATE_OR_UPDATE,
     DATASET_DELETE,
     DATASET_READ,
+    CTL_DATASET_READ,
 )
 from fides.api.api.v1.urn_registry import (
     CONNECTION_DATASETS,
     DATASET_BY_KEY,
     DATASET_CONFIGS,
     DATASET_VALIDATE,
     DATASETCONFIG_BY_KEY,
-    DATASETS,
+    CONNECTION_DATASETS,
     V1_URL_PREFIX,
     YAML_DATASETS,
+    DATASETS,
 )
 from fides.api.models.connectionconfig import ConnectionConfig
 from fides.api.models.datasetconfig import DatasetConfig