Updates for consent signal processing (#5200)

.fides/db_dataset.yml

@@ -126,6 +126,9 @@ dataset:
     - name: systems
       data_categories:
         - system.operations
+    - name: connections
+      data_categories:
+        - system.operations
     - name: updated_at
       data_categories:
       - system.operations

CHANGELOG.md

@@ -17,14 +17,14 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fides/compare/2.43.1...main)
 
-
 ### Added
 - Added Gzip Middleware for responses [#5225](https://github.com/ethyca/fides/pull/5225)
 - Adding source and submitted_by fields to privacy requests (Fidesplus) [#5206](https://github.com/ethyca/fides/pull/5206)
 
 ### Changed
 - Removed unused `username` parameter from the Delighted integration configuration [#5220](https://github.com/ethyca/fides/pull/5220)
 - Removed unused `ad_account_id` parameter from the Snap integration configuration [#5229](https://github.com/ethyca/fides/pull/5220)
+- Updates to support consent signal processing (Fidesplus) [#5200](https://github.com/ethyca/fides/pull/5200)
 
 ### Developer Experience
 - Sourcemaps are now working for fides-js in debug mode [#5222](https://github.com/ethyca/fides/pull/5222)

clients/admin-ui/src/features/datastore-connections/system_portal_config/ConsentAutomationForm.tsx

@@ -250,7 +250,6 @@ export const ConsentAutomationForm = ({
                     color="white"
                     isDisabled={isSubmitting}
                     isLoading={isSubmitting}
-                    loadingText="Submitting"
                     size="sm"
                     variant="solid"
                     type="submit"

src/fides/api/alembic/migrations/versions/d9064e71f69d_add_connections_to_client.py

@@ -0,0 +1,29 @@
+"""add connections to client
+
+Revision ID: d9064e71f69d
+Revises: 896ea3803770
+Create Date: 2024-08-20 22:11:34.351186
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d9064e71f69d"
+down_revision = "896ea3803770"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "client",
+        sa.Column(
+            "connections", sa.ARRAY(sa.String()), server_default="{}", nullable=False
+        ),
+    )
+
+
+def downgrade():
+    op.drop_column("client", "connections")

src/fides/api/cryptography/schemas/jwt.py

@@ -3,3 +3,4 @@
 JWE_ISSUED_AT = "iat"
 JWE_PAYLOAD_ROLES = "roles"
 JWE_PAYLOAD_SYSTEMS = "systems"
+JWE_PAYLOAD_CONNECTIONS = "connections"

src/fides/api/models/client.py

@@ -16,6 +16,7 @@
 from fides.api.cryptography.schemas.jwt import (
     JWE_ISSUED_AT,
     JWE_PAYLOAD_CLIENT_ID,
+    JWE_PAYLOAD_CONNECTIONS,
     JWE_PAYLOAD_ROLES,
     JWE_PAYLOAD_SCOPES,
     JWE_PAYLOAD_SYSTEMS,
@@ -28,6 +29,7 @@
 DEFAULT_SCOPES: list[str] = []
 DEFAULT_ROLES: list[str] = []
 DEFAULT_SYSTEMS: list[str] = []
+DEFAULT_CONNECTIONS: list[str] = []
 
 
 class ClientDetail(Base):
@@ -42,6 +44,9 @@ def __tablename__(self) -> str:
     scopes = Column(ARRAY(String), nullable=False, server_default="{}", default=dict)
     roles = Column(ARRAY(String), nullable=False, server_default="{}", default=dict)
     systems = Column(ARRAY(String), nullable=False, server_default="{}", default=dict)
+    connections = Column(
+        ARRAY(String), nullable=False, server_default="{}", default=dict
+    )
     fides_key = Column(String, index=True, unique=True, nullable=True)
     user_id = Column(
         String, ForeignKey(FidesUser.id_field_path), nullable=True, unique=True
@@ -60,6 +65,7 @@ def create_client_and_secret(
         encoding: str = "UTF-8",
         roles: list[str] | None = None,
         systems: list[str] | None = None,
+        connections: list[str] | None = None,
     ) -> tuple["ClientDetail", str]:
         """Creates a ClientDetail and returns that along with the unhashed secret
         so it can be returned to the user on create
@@ -77,6 +83,9 @@ def create_client_and_secret(
         if not systems:
             systems = DEFAULT_SYSTEMS
 
+        if not connections:
+            connections = DEFAULT_CONNECTIONS
+
         salt = generate_salt()
         hashed_secret = hash_with_salt(
             secret.encode(encoding),
@@ -94,6 +103,7 @@ def create_client_and_secret(
                 "user_id": user_id,
                 "roles": roles,
                 "systems": systems,
+                "connections": connections,
             },
         )
         return client, secret  # type: ignore
@@ -122,6 +132,7 @@ def create_access_code_jwe(self, encryption_key: str) -> str:
             JWE_ISSUED_AT: datetime.now().isoformat(),
             JWE_PAYLOAD_ROLES: self.roles,
             JWE_PAYLOAD_SYSTEMS: self.systems,
+            JWE_PAYLOAD_CONNECTIONS: self.connections,
         }
         return generate_jwe(json.dumps(payload), encryption_key)
 
@@ -155,6 +166,7 @@ def _get_root_client_detail(
             scopes=scopes,
             roles=roles,
             systems=[],
+            connections=[],
         )
 
     return ClientDetail(
@@ -164,4 +176,5 @@ def _get_root_client_detail(
         scopes=DEFAULT_SCOPES,
         roles=DEFAULT_ROLES,
         systems=DEFAULT_SYSTEMS,
+        connections=DEFAULT_CONNECTIONS,
     )

src/fides/api/oauth/utils.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 from functools import update_wrapper
 from types import FunctionType
-from typing import Any, Callable, Dict, List, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 from fastapi import Depends, HTTPException, Security
 from fastapi.security import SecurityScopes
@@ -276,7 +276,10 @@ async def verify_oauth_client(
 
 
 def extract_token_and_load_client(
-    authorization: str = Security(oauth2_scheme), db: Session = Depends(get_db)
+    authorization: str = Security(oauth2_scheme),
+    db: Session = Depends(get_db),
+    *,
+    token_duration_override: Optional[int] = None,
 ) -> Tuple[Dict, ClientDetail]:
     """Extract the token, verify it's valid, and likewise load the client as part of authorization"""
     if authorization is None:
@@ -298,7 +301,7 @@ def extract_token_and_load_client(
 
     if is_token_expired(
         datetime.fromisoformat(issued_at),
-        CONFIG.security.oauth_access_token_expire_minutes,
+        token_duration_override or CONFIG.security.oauth_access_token_expire_minutes,
     ):
         raise AuthorizationError(detail="Not Authorized for this action")
 

src/fides/api/schemas/consentable_item.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Dict, List, Literal, Optional
 
 from pydantic import BaseModel, Field
 
@@ -83,7 +83,9 @@ class ConsentWebhookResult(BaseModel):
     A wrapper class for the identity map and notice map values returned from a `PROCESS_CONSENT_WEBHOOK` function.
     """
 
-    identity_map: Dict[str, Any] = {}
+    identity_map: Dict[
+        Literal["email", "phone_number", "fides_user_device", "external_id"], str
+    ] = {}
     notice_map: Dict[str, UserConsentPreference] = {}
 
     @property

src/fides/api/schemas/saas/shared_schemas.py

@@ -39,3 +39,13 @@ class IdentityParamRef(BaseModel):
     """A reference to the identity type in the filter Post Processor Config"""
 
     identity: str
+
+
+class ConsentPropagationStatus(Enum):
+    """
+    An enum for the different statuses that can be returned from a consent propagation request.
+    """
+
+    executed = "executed"
+    no_update_needed = "no_update_needed"
+    missing_data = "missing_data"

src/fides/api/service/connectors/saas_connector.py

@@ -17,7 +17,6 @@
 )
 from fides.api.graph.execution import ExecutionNode
 from fides.api.models.connectionconfig import ConnectionConfig, ConnectionTestStatus
-from fides.api.models.consent_automation import ConsentAutomation
 from fides.api.models.policy import Policy
 from fides.api.models.privacy_notice import UserConsentPreference
 from fides.api.models.privacy_request import PrivacyRequest, RequestTask
@@ -35,7 +34,10 @@
     ReadSaaSRequest,
     SaaSRequest,
 )
-from fides.api.schemas.saas.shared_schemas import SaaSRequestParams
+from fides.api.schemas.saas.shared_schemas import (
+    ConsentPropagationStatus,
+    SaaSRequestParams,
+)
 from fides.api.service.connectors.base_connector import BaseConnector
 from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient
 from fides.api.service.connectors.saas_query_config import SaaSQueryConfig
@@ -609,15 +611,15 @@ def relevant_consent_identities(
 
     @staticmethod
     def build_notice_based_consentable_item_hierarchy(
-        session: Session, connection_config_id: str
-    ) -> Optional[List[ConsentableItem]]:
-        """Helper function to construct list of consentable items to later pass into update consent function"""
-        consent_automation: Optional[ConsentAutomation] = ConsentAutomation.get_by(
-            session, field="connection_config_id", value=connection_config_id
-        )
-        if consent_automation:
+        connection_config: ConnectionConfig,
+    ) -> List[ConsentableItem]:
+        """
+        Helper function to construct list of consentable items to later pass into update consent function.
+        """
+
+        if consent_automation := connection_config.consent_automation:
             return build_consent_item_hierarchy(consent_automation.consentable_items)
-        return None
+        return []
 
     @staticmethod
     def obtain_notice_based_update_consent_function_or_none(
@@ -654,21 +656,23 @@ def run_consent_request(
         identity_data: Dict[str, Any],
         session: Session,
     ) -> bool:
-        """Execute a consent request. Return whether the consent request to the third party succeeded.
+        # pylint: disable=too-many-branches, too-many-statements
+        """
+        Execute a consent request. Return whether the consent request to the third party succeeded.
         Should only propagate either the entire set of opt in or opt out requests.
         Return True if 200 OK. Raises a SkippingConsentPropagation exception if no action is taken
         against the service.
         """
+
         logger.info(
             "Starting consent request for node: '{}'",
             node.address.value,
         )
         self.set_privacy_request_state(privacy_request, node, request_task)
         query_config = self.query_config(node)
         saas_config = self.saas_config
-        fired: bool = (
-            False  # True if the SaaS connector was successfully called / completed
-        )
+
+        consent_propagation_status: Optional[ConsentPropagationStatus] = None
 
         notice_based_override_function: Optional[RequestOverrideFunction] = (
             self.obtain_notice_based_update_consent_function_or_none(saas_config.type)
@@ -699,10 +703,8 @@ def run_consent_request(
                 relevant_preferences=filtered_preferences,
                 relevant_user_identities=identity_data,
             )
-            notice_based_consentable_item_hierarchy: Optional[List[ConsentableItem]] = (
-                self.build_notice_based_consentable_item_hierarchy(
-                    session, self.configuration.id
-                )
+            notice_based_consentable_item_hierarchy: List[ConsentableItem] = (
+                self.build_notice_based_consentable_item_hierarchy(self.configuration)
             )
             if not notice_based_consentable_item_hierarchy:
                 logger.info(
@@ -712,7 +714,7 @@ def run_consent_request(
                 raise SkippingConsentPropagation(
                     f"Skipping consent propagation for node {node.address.value} - no actionable consent preferences to propagate"
                 )
-            fired = self._invoke_consent_request_override(
+            consent_propagation_status = self._invoke_consent_request_override(
                 notice_based_override_function,
                 self.create_client(),
                 policy,
@@ -722,6 +724,10 @@ def run_consent_request(
                 notice_id_to_preference_map,  # type: ignore[arg-type]
                 notice_based_consentable_item_hierarchy,
             )
+            if consent_propagation_status == ConsentPropagationStatus.no_update_needed:
+                raise SkippingConsentPropagation(
+                    "Consent preferences are already up-to-date"
+                )
 
         else:
             # follow the basic (global opt-in/out) SaaS consent flow
@@ -785,7 +791,7 @@ def run_consent_request(
                             SaaSRequestType(query_config.action),
                         )
                     )
-                    fired = self._invoke_consent_request_override(
+                    consent_propagation_status = self._invoke_consent_request_override(
                         override_function,
                         self.create_client(),
                         policy,
@@ -806,17 +812,22 @@ def run_consent_request(
                                 node.address.value,
                                 exc,
                             )
+                            consent_propagation_status = (
+                                ConsentPropagationStatus.missing_data
+                            )
                             continue
                         raise exc
                     client: AuthenticatedClient = self.create_client()
                     client.send(prepared_request)
-                    fired = True
+                    consent_propagation_status = ConsentPropagationStatus.executed
 
         self.unset_connector_state()
-        if not fired:
+
+        if consent_propagation_status == ConsentPropagationStatus.missing_data:
             raise SkippingConsentPropagation(
                 "Missing needed values to propagate request."
             )
+
         add_complete_system_status_for_consent_reporting(
             session, privacy_request, self.configuration
         )
@@ -986,7 +997,7 @@ def _invoke_consent_request_override(
         identity_data: Optional[Dict[str, Any]] = None,
         notice_id_to_preference_map: Optional[Dict[str, UserConsentPreference]] = None,
         consentable_items_hierarchy: Optional[List[ConsentableItem]] = None,
-    ) -> bool:
+    ) -> ConsentPropagationStatus:
         """
         Invokes the appropriate user-defined SaaS request override for consent requests
         and performs error handling for uncaught exceptions coming out of the override.