Merge branch 'main' into PROD-1243-ns-fix-undefined-custom-options-path

clients/admin-ui/src/features/datamap/Datamap.tsx

@@ -9,9 +9,11 @@ import { DatamapGraphContext } from "~/features/datamap/datamap-graph/DatamapGra
 import { useTableInstance } from "~/features/datamap/datamap-table/hooks/";
 import SettingsBar from "~/features/datamap/SettingsBar";
 
+import { useFeatures } from "../common/features";
 import { selectIsGettingStarted, selectIsMapOpen } from "./datamap.slice";
 import DatamapTable from "./datamap-table/DatamapTable";
 import GetStarted from "./GetStarted";
+import GVLDatamapNotice from "./GVLDatamapNotice";
 
 const SpatialDatamap = dynamic(
   () => import("~/features/datamap/SpatialDatamap"),
@@ -67,6 +69,7 @@ const Datamap = () => {
     selectedSystemId,
     resetSelectedSystemId,
   } = useHome();
+  const { tcf: isTcfEnabled } = useFeatures();
   const { isLoading } = useTableInstance();
   if (isLoading) {
     return (
@@ -81,35 +84,38 @@ const Datamap = () => {
   }
 
   return (
-    <Flex direction="column" height="100%">
-      <Box marginBottom={3} marginRight={10}>
-        <SettingsBar />
-      </Box>
-      <Flex
-        position="relative"
-        flex={1}
-        direction="row"
-        overflow="auto"
-        borderWidth="1px"
-        borderStyle="solid"
-        borderColor="gray.200"
-      >
-        {isMapOpen ? (
-          <Box flex={1} minWidth="50%" maxWidth="100%">
-            <SpatialDatamap setSelectedSystemId={setSelectedSystemId} />
-          </Box>
-        ) : null}
-        {!isMapOpen ? (
-          <Box flex={1} minWidth="50%" maxWidth="100%">
-            <DatamapTable setSelectedSystemId={setSelectedSystemId} />
-          </Box>
-        ) : null}
-        <DatamapDrawer
-          selectedSystemId={selectedSystemId}
-          resetSelectedSystemId={resetSelectedSystemId}
-        />
+    <>
+      {isTcfEnabled ? <GVLDatamapNotice /> : null}
+      <Flex direction="column" height="100%">
+        <Box marginBottom={3} marginRight={10}>
+          <SettingsBar />
+        </Box>
+        <Flex
+          position="relative"
+          flex={1}
+          direction="row"
+          overflow="auto"
+          borderWidth="1px"
+          borderStyle="solid"
+          borderColor="gray.200"
+        >
+          {isMapOpen ? (
+            <Box flex={1} minWidth="50%" maxWidth="100%">
+              <SpatialDatamap setSelectedSystemId={setSelectedSystemId} />
+            </Box>
+          ) : null}
+          {!isMapOpen ? (
+            <Box flex={1} minWidth="50%" maxWidth="100%">
+              <DatamapTable setSelectedSystemId={setSelectedSystemId} />
+            </Box>
+          ) : null}
+          <DatamapDrawer
+            selectedSystemId={selectedSystemId}
+            resetSelectedSystemId={resetSelectedSystemId}
+          />
+        </Flex>
       </Flex>
-    </Flex>
+    </>
   );
 };
 

clients/admin-ui/src/features/datamap/GVLDatamapNotice.tsx

@@ -0,0 +1,14 @@
+import { Box } from "@fidesui/react";
+
+import EmptyTableState from "~/features/common/table/EmptyTableState";
+
+const GVLDatamapNotice = () => (
+  <Box mb="6" maxW="720px" data-testid="GVL-datamap-notice">
+    <EmptyTableState
+      title="GVL and AC vendors are not displayed"
+      description="Global Vendor List and Google Additional Consent vendors are not displayed on the data map for performance reasons. An enhancement is in the works!"
+    />
+  </Box>
+);
+
+export default GVLDatamapNotice;

src/fides/api/db/crud.py

@@ -15,6 +15,7 @@
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
+from sqlalchemy.sql import Select
 from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
 
 from fides.api.db.base import Base  # type: ignore[attr-defined]
@@ -200,11 +201,22 @@ async def list_resource(sql_model: Base, async_session: AsyncSession) -> List[Ba
 
     Returns a list of SQLAlchemy models of that resource type.
     """
+    query = select(sql_model)
+    return await list_resource_query(async_session, query, sql_model)
+
+
+async def list_resource_query(
+    async_session: AsyncSession, query: Select, sql_model: Base = Base
+) -> List[Base]:
+    """
+    Utility function to wrap a select query in generic "list_resource" execution handling.
+    Wrapping includes execution against the DB session, logging and error handling.
+    """
+
     with log.contextualize(sql_model=sql_model.__name__):
         async with async_session.begin():
             try:
                 log.debug("Fetching resources")
-                query = select(sql_model)
                 result = await async_session.execute(query)
                 sql_resources = result.scalars().all()
             except SQLAlchemyError:

src/fides/api/util/tcf/__init__.py

@@ -0,0 +1,42 @@
+from typing import List
+
+from loguru import logger
+from sqlalchemy import not_, or_
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.future import select
+from sqlalchemy.sql import Select
+
+from fides.api.models.sql_models import System  # type: ignore[attr-defined]
+from fides.api.util import errors
+
+AC_PREFIX = "gacp."
+GVL_PREFIX = "gvl."
+
+
+def exclude_gvl_systems(query: Select) -> Select:
+    """Utility function to add a query clause that excludes GVL systems"""
+    return query.where(
+        or_(System.vendor_id.is_(None), not_(System.vendor_id.startswith(GVL_PREFIX)))
+    )
+
+
+def exclude_ac_systems(query: Select) -> Select:
+    """Utility function to add a query clause that excludes AC systems"""
+    return query.where(
+        or_(System.vendor_id.is_(None), not_(System.vendor_id.startswith(AC_PREFIX)))
+    )
+
+
+def list_non_tcf_systems(exclude_gvl: bool = True, exclude_ac: bool = True) -> Select:
+    """
+    Utility to retrieve all Systems that are not GVL systems using
+    the provided (async) DB session.
+    """
+
+    query = select(System)
+    if exclude_gvl:
+        query = exclude_gvl_systems(query)
+    if exclude_ac:
+        query = exclude_ac_systems(query)
+    return query

src/fides/api/util/tcf/tcf_experience_contents.py

@@ -31,11 +31,9 @@
     TCFVendorLegitimateInterestsRecord,
     TCFVendorRelationships,
 )
+from fides.api.util.tcf import AC_PREFIX
 from fides.config import CONFIG
 
-AC_PREFIX = "gacp."
-GVL_PREFIX = "gvl."
-
 PURPOSE_DATA_USES: List[str] = []
 for purpose in MAPPED_PURPOSES.values():
     PURPOSE_DATA_USES.extend(purpose.data_uses)