diff --git a/CHANGELOG.md b/CHANGELOG.md index 64652fff67..371bb05a70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The types of changes are: ### Changed - Allow hiding systems via a `hidden` parameter and add two flags on the `/system` api endpoint; `show_hidden` and `dnd_relevant`, to display only systems with integrations [#5484](https://github.com/ethyca/fides/pull/5484) +- Updated POST taxonomy endpoints to handle creating resources without specifying fides_key [#5468](https://github.com/ethyca/fides/pull/5468) ### Developer Experience - Fixing BigQuery integration tests [#5491](https://github.com/ethyca/fides/pull/5491) diff --git a/src/fides/api/api/v1/endpoints/generic_overrides.py b/src/fides/api/api/v1/endpoints/generic_overrides.py index 52e9b95982..9fbde4b3e9 100644 --- a/src/fides/api/api/v1/endpoints/generic_overrides.py +++ b/src/fides/api/api/v1/endpoints/generic_overrides.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Union +from typing import Dict, List, Optional, Type, Union from fastapi import APIRouter, Depends, Query, Security from fastapi_pagination import Page, Params @@ -7,15 +7,23 @@ from sqlalchemy import not_ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql.expression import select +from starlette import status +from fides.api.db.base_class import get_key_from_data from fides.api.db.crud import list_resource_query from fides.api.db.ctl_session import get_async_db from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.datasetconfig import DatasetConfig from fides.api.oauth.utils import verify_oauth_client from fides.api.schemas.filter_params import FilterParams +from fides.api.schemas.taxonomy_extensions import DataCategory, DataSubject, DataUse from fides.api.util.filter_utils import apply_filters_to_query -from fides.common.api.scope_registry import DATASET_READ +from fides.common.api.scope_registry import ( + DATA_CATEGORY_CREATE, + DATA_SUBJECT_CREATE, + DATA_USE_CREATE, + DATASET_READ, +) from fides.common.api.v1.urn_registry import V1_URL_PREFIX from fides.api.models.sql_models import ( # type: ignore[attr-defined] # isort: skip @@ -26,6 +34,9 @@ # when we need more custom implementations for only some of the methods in a router. dataset_router = APIRouter(tags=["Dataset"], prefix=V1_URL_PREFIX) +data_use_router = APIRouter(tags=["DataUse"], prefix=V1_URL_PREFIX) +data_category_router = APIRouter(tags=["DataCategory"], prefix=V1_URL_PREFIX) +data_subject_router = APIRouter(tags=["DataSubject"], prefix=V1_URL_PREFIX) @dataset_router.get( @@ -85,5 +96,92 @@ async def list_dataset_paginated( return await async_paginate(db, filtered_query, pagination_params) +async def create_with_key( + data: Union[DataUse, DataCategory, DataSubject], + model: Type[Union[DataUse, DataCategory, DataSubject]], + db: AsyncSession, +) -> Dict: + """ + Helper to create taxonomy resource when not given a fides_key. + Automatically re-enables disabled resources with the same name. + """ + # If data with same name exists but is disabled, re-enable it + disabled_resource_with_name = db.query(model).filter( + model.key == data.name, # type: ignore[union-attr] + model.active is False, + ) + if disabled_resource_with_name: + return model.update(db=db, data=data, active=True) # type: ignore[union-attr] + data.fides_key = get_key_from_data( + {"key": data.fides_key, "name": data.name}, model.__name__ + ) + return model.create(db=db, data=data.model_dump(mode="json")) # type: ignore[union-attr] + + +@data_use_router.post( + "/data_use", + dependencies=[Security(verify_oauth_client, scopes=[DATA_USE_CREATE])], + response_model=DataUse, + status_code=status.HTTP_201_CREATED, + name="Create", +) +async def create_data_use( + data_use: DataUse, + db: AsyncSession = Depends(get_async_db), +) -> Dict: + """ + Create a data use. Updates existing data use if data use with name already exists and is disabled. + """ + if data_use.fides_key is None: + await create_with_key(data_use, DataUse, db) + + return await DataUse.create(db=db, data=data_use.model_dump(mode="json")) # type: ignore[attr-defined] + + +@data_category_router.post( + "/data_category", + dependencies=[Security(verify_oauth_client, scopes=[DATA_CATEGORY_CREATE])], + response_model=DataCategory, + status_code=status.HTTP_201_CREATED, + name="Create", +) +async def create_data_category( + data_category: DataCategory, + db: AsyncSession = Depends(get_async_db), +) -> Dict: + """ + Create a data category + """ + + if data_category.fides_key is None: + await create_with_key(data_category, DataCategory, db) + + return await DataCategory.create(db=db, data=data_category.model_dump(mode="json")) # type: ignore[attr-defined] + + +@data_subject_router.post( + "/data_subject", + dependencies=[Security(verify_oauth_client, scopes=[DATA_SUBJECT_CREATE])], + response_model=DataSubject, + status_code=status.HTTP_201_CREATED, + name="Create", +) +async def create_data_subject( + data_subject: DataSubject, + db: AsyncSession = Depends(get_async_db), +) -> Dict: + """ + Create a data subject + """ + + if data_subject.fides_key is None: + await create_with_key(data_subject, DataSubject, db) + + return await DataSubject.create(db=db, data=data_subject.model_dump(mode="json")) # type: ignore[attr-defined] + + GENERIC_OVERRIDES_ROUTER = APIRouter() GENERIC_OVERRIDES_ROUTER.include_router(dataset_router) +GENERIC_OVERRIDES_ROUTER.include_router(data_use_router) +GENERIC_OVERRIDES_ROUTER.include_router(data_category_router) +GENERIC_OVERRIDES_ROUTER.include_router(data_subject_router) diff --git a/src/fides/api/api/v1/endpoints/router_factory.py b/src/fides/api/api/v1/endpoints/router_factory.py index b7be7047e7..9a6814c70c 100644 --- a/src/fides/api/api/v1/endpoints/router_factory.py +++ b/src/fides/api/api/v1/endpoints/router_factory.py @@ -162,6 +162,7 @@ async def create( raise errors.ForbiddenIsDefaultTaxonomyError( model_type, resource.fides_key, action="create" ) + return await create_resource(sql_model, resource.model_dump(mode="json"), db) return router diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index 0cdb9e2419..f63c9d037f 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -3022,6 +3022,133 @@ def test_system_manager_gets_403_if_system_not_found( assert result.status_code == HTTP_403_FORBIDDEN +@pytest.mark.integration +class TestDefaultTaxonomyCrudOverrides: + @pytest.mark.parametrize("endpoint", TAXONOMY_ENDPOINTS) + def test_api_cannot_create_if_generated_fides_key_conflicts_with_existing( + self, + test_config: FidesConfig, + generate_auth_header, + endpoint: str, + ) -> None: + """Ensure we cannot create taxonomy elements if fides key conflicts with existing key""" + # get a default taxonomy element as a sample resource + resource = getattr(DEFAULT_TAXONOMY, endpoint)[0] + resource = TAXONOMY_EXTENSIONS[endpoint]( + **resource.model_dump(mode="json") + ) # cast resource to extended model + # This name will conflict with existing resource + resource.name = resource.fides_key + resource.fides_key = None + json_resource = resource.json(exclude_none=True) + token_scopes: List[str] = [f"{CLI_SCOPE_PREFIX_MAPPING[endpoint]}:{CREATE}"] + auth_header = generate_auth_header(scopes=token_scopes) + result = _api.create( + url=test_config.cli.server_url, + headers=auth_header, + resource_type=endpoint, + json_resource=json_resource, + ) + assert result.status_code == HTTP_409_CONFLICT + + @pytest.mark.parametrize("endpoint", TAXONOMY_ENDPOINTS) + def test_api_can_create_without_explicit_fides_key( + self, + test_config: FidesConfig, + generate_auth_header, + endpoint: str, + ) -> None: + """Ensure we can create taxonomy elements without specifying a fides_key""" + # get a default taxonomy element as a sample resource + resource = getattr(DEFAULT_TAXONOMY, endpoint)[0] + resource = TAXONOMY_EXTENSIONS[endpoint]( + **resource.model_dump(mode="json") + ) # cast resource to extended model + # Build unique name based on sample name + resource.name = resource.name + "my new resource" + resource.fides_key = None + json_resource = resource.json(exclude_none=True) + token_scopes: List[str] = [f"{CLI_SCOPE_PREFIX_MAPPING[endpoint]}:{CREATE}"] + auth_header = generate_auth_header(scopes=token_scopes) + result = _api.create( + url=test_config.cli.server_url, + headers=auth_header, + resource_type=endpoint, + json_resource=json_resource, + ) + assert result.status_code == 201 + assert result.json()["active"] is True + new_key = result.json()["fides_key"] + assert "my_new_resource" in new_key + + result = _api.get( + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + resource_type=endpoint, + resource_id=new_key, + ) + assert result.json()["active"] is True + + @pytest.mark.parametrize("endpoint", TAXONOMY_ENDPOINTS) + def test_api_can_update_active_when_creating_with_same_name( + self, + test_config: FidesConfig, + endpoint: str, + ) -> None: + """ + If we attempt to create a new resource with the same name as an existing inactive resource, + but with no explicit fides_key, we should update the existing resource to be active + """ + resource = getattr(DEFAULT_TAXONOMY, endpoint)[0] + resource = TAXONOMY_EXTENSIONS[endpoint]( + **resource.model_dump(mode="json") + ) # cast resource to extended model + resource.active = False + json_resource = resource.json(exclude_none=True) + # First, update the existing resource as inactive so we can use it to test + result = _api.update( + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + resource_type=endpoint, + json_resource=json_resource, + ) + assert result.status_code == 200 + assert result.json()["active"] is False + + # Confirm it was updated to inactive + result = _api.get( + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + resource_type=endpoint, + resource_id=resource.fides_key, + ) + assert result.json()["active"] is False + + # Now attempt to create another resource with a name that will generate the same fides_key + # as the inactive resource + + resource.name = resource.name # explicitly using the same name + resource.fides_key = None + json_resource = resource.json(exclude_none=True) + result = _api.create( + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + resource_type=endpoint, + json_resource=json_resource, + ) + assert result.status_code == 200 + assert result.json()["active"] is True + + # Confirm the existing resource was updated to active + result = _api.get( + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + resource_type=endpoint, + resource_id=resource.fides_key, + ) + assert result.json()["active"] is True + + @pytest.mark.integration class TestDefaultTaxonomyCrud: @pytest.mark.parametrize("endpoint", TAXONOMY_ENDPOINTS)