Remove fideslang module in favor of fideslang package

CHANGELOG.md

@@ -25,6 +25,7 @@ The types of changes are:
 ### Developer Experience
 
 * Replaced `make` with `nox`
+* Removed usage of `fideslang` module in favor of new [external package](https://github.com/ethyca/fideslang) shared across projects
 
 ### Docs
 

requirements.txt

@@ -3,6 +3,7 @@ click>=8.1
 colorama>=0.4.3
 cryptography>=36
 deepdiff==5.8.0
+fideslang==0.9.0
 fideslog==1.1.5
 loguru>=0.5,<0.6
 openpyxl==3.0.9

src/fidesapi/database/database.py

@@ -6,13 +6,13 @@
 from alembic import command
 from alembic.config import Config
 from alembic.migration import MigrationContext
+from fideslang import DEFAULT_TAXONOMY
 from loguru import logger as log
 from sqlalchemy_utils.functions import create_database, database_exists
 
 from fidesapi.sql_models import SqlAlchemyBase, sql_model_map
 from fidesapi.utils.errors import AlreadyExistsError, QueryError
 from fidesctl.core.utils import get_db_engine
-from fideslang import DEFAULT_TAXONOMY
 
 from .crud import create_resource, upsert_resources
 

src/fidesapi/routes/crud.py

@@ -9,6 +9,7 @@
 from typing import Dict, List
 
 from fastapi import APIRouter, Response, status
+from fideslang import model_map
 
 from fidesapi.database.crud import (
     create_resource,
@@ -19,7 +20,6 @@
     upsert_resources,
 )
 from fidesapi.sql_models import sql_model_map
-from fideslang import model_map
 
 
 def get_resource_type(router: APIRouter) -> str:

src/fidesapi/routes/visualize.py

@@ -6,11 +6,11 @@
 
 from fastapi import APIRouter, HTTPException
 from fastapi.responses import HTMLResponse
+from fideslang import model_map
 
 from fidesapi.routes.crud import list_resource
 from fidesapi.sql_models import sql_model_map
 from fidesctl.core import visualize
-from fideslang import model_map
 
 # pylint: disable=redefined-outer-name,cell-var-from-loop
 

src/fidesctl/cli/options.py

@@ -5,7 +5,6 @@
 from typing import Callable
 
 import click
-
 from fideslang import model_list
 
 

src/fidesctl/connectors/okta.py

@@ -1,11 +1,11 @@
 """Module that adds interactions with okta"""
 from typing import List
 
+from fideslang.models import Dataset, DatasetMetadata
 from okta.client import Client as OktaClient
 from okta.models import Application as OktaApplication
 
 from fidesctl.core.utils import echo_red
-from fideslang.models import Dataset, DatasetMetadata
 
 
 def get_okta_client(org_url: str) -> OktaClient:

src/fidesctl/core/annotate_dataset.py

@@ -5,14 +5,14 @@
 from typing import List, Union
 
 import click
+from fideslang import manifests
+from fideslang.models import Dataset, DatasetCollection, DatasetField, FidesKey
+from fideslang.validation import FidesValidationError
 
 from fidesctl.core import api_helpers
 from fidesctl.core import parse as core_parse
 from fidesctl.core.config import FidesctlConfig
 from fidesctl.core.utils import echo_green, get_all_level_fields
-from fideslang import manifests
-from fideslang.models import Dataset, DatasetCollection, DatasetField, FidesKey
-from fideslang.validation import FidesValidationError
 
 
 class AnnotationAbortError(Exception):

src/fidesctl/core/api_helpers.py

@@ -4,12 +4,12 @@
 
 from typing import Dict, List, Optional
 
-from requests import Response
-
-from fidesctl.core import api
 from fideslang import FidesModel
 from fideslang.parse import parse_dict
 from fideslang.validation import FidesKey
+from requests import Response
+
+from fidesctl.core import api
 
 
 def get_server_resources(

src/fidesctl/core/apply.py

@@ -4,12 +4,12 @@
 from typing import Dict, List, Tuple
 
 from deepdiff import DeepDiff
+from fideslang import FidesModel, Taxonomy
 
 from fidesctl.cli.utils import handle_cli_response
 from fidesctl.core import api
 from fidesctl.core.api_helpers import get_server_resources
 from fidesctl.core.utils import echo_green, echo_red
-from fideslang import FidesModel, Taxonomy
 
 
 def sort_create_update(

src/fidesctl/core/dataset.py

@@ -3,13 +3,13 @@
 from typing import Dict, List, Optional, Tuple
 
 import sqlalchemy
+from fideslang import manifests
+from fideslang.models import Dataset, DatasetCollection, DatasetField
 from pydantic import AnyHttpUrl
 from sqlalchemy.engine import Engine
 
 from fidesctl.core.api_helpers import list_server_resources
 from fidesctl.core.parse import parse
-from fideslang import manifests
-from fideslang.models import Dataset, DatasetCollection, DatasetField
 
 from .utils import echo_green, echo_red, get_db_engine
 

src/fidesctl/core/evaluate.py

@@ -2,13 +2,6 @@
 import uuid
 from typing import Callable, Dict, List, Optional, Set, cast
 
-from pydantic import AnyHttpUrl
-
-from fidesctl.cli.utils import handle_cli_response, pretty_echo
-from fidesctl.core import api
-from fidesctl.core.api_helpers import get_server_resource, get_server_resources
-from fidesctl.core.parse import parse
-from fidesctl.core.utils import echo_green, echo_red, get_all_level_fields
 from fideslang.default_taxonomy import DEFAULT_TAXONOMY
 from fideslang.models import (
     Dataset,
@@ -23,12 +16,16 @@
     Violation,
     ViolationAttributes,
 )
-from fideslang.relationships import (
-    get_referenced_missing_keys,
-    hydrate_missing_resources,
-)
+from fideslang.relationships import get_referenced_missing_keys
 from fideslang.utils import get_resource_by_fides_key
 from fideslang.validation import FidesKey
+from pydantic import AnyHttpUrl
+
+from fidesctl.cli.utils import handle_cli_response, pretty_echo
+from fidesctl.core import api
+from fidesctl.core.api_helpers import get_server_resource, get_server_resources
+from fidesctl.core.parse import parse
+from fidesctl.core.utils import echo_green, echo_red, get_all_level_fields
 
 
 def get_evaluation_policies(
@@ -438,6 +435,31 @@ def execute_evaluation(taxonomy: Taxonomy) -> Evaluation:
     return evaluation
 
 
+def hydrate_missing_resources(
+    url: AnyHttpUrl,
+    headers: Dict[str, str],
+    missing_resource_keys: List[FidesKey],
+    dehydrated_taxonomy: Taxonomy,
+) -> Taxonomy:
+    """
+    Query the server for all of the missing resource keys and
+    hydrate a copy of the dehydrated taxonomy with them.
+    """
+
+    for resource_name in dehydrated_taxonomy.__fields__:
+        server_resources = get_server_resources(
+            url=url,
+            resource_type=resource_name,
+            headers=headers,
+            existing_keys=missing_resource_keys,
+        )
+        dehydrated_taxonomy.__setattr__(
+            resource_name,
+            getattr(dehydrated_taxonomy, resource_name) + server_resources,
+        )
+    return dehydrated_taxonomy
+
+
 def populate_referenced_keys(
     taxonomy: Taxonomy,
     url: AnyHttpUrl,

src/fidesctl/core/export.py

@@ -5,6 +5,7 @@
 from typing import Dict, List, Tuple
 
 import pandas as pd
+from fideslang.models import ContactDetails, Taxonomy
 
 from fidesctl.core.api_helpers import get_server_resources
 from fidesctl.core.export_helpers import (
@@ -19,7 +20,6 @@
     union_data_categories_in_joined_dataframe,
 )
 from fidesctl.core.utils import echo_green, get_all_level_fields
-from fideslang.models import ContactDetails, Taxonomy
 
 
 def generate_dataset_records(

src/fidesctl/core/export_helpers.py

@@ -5,10 +5,10 @@
 from typing import Dict, List, Set, Tuple
 
 import pandas as pd
+from fideslang.models import DataSubjectRightsEnum, Taxonomy
 
 from fidesctl.core.api_helpers import get_server_resource, get_server_resources
 from fidesctl.core.utils import echo_red
-from fideslang.models import DataSubjectRightsEnum, Taxonomy
 
 DATAMAP_TEMPLATE = join(
     dirname(__file__),

src/fidesctl/core/parse.py

@@ -1,9 +1,10 @@
 """This module is responsible for parsing and verifying file, either with or without a server being available."""
-from fidesctl.core.utils import echo_green
 from fideslang import Taxonomy
 from fideslang.manifests import ingest_manifests
 from fideslang.parse import load_manifests_into_taxonomy
 
+from fidesctl.core.utils import echo_green
+
 
 def parse(manifests_dir: str) -> Taxonomy:
     """

src/fidesctl/core/system.py

@@ -2,14 +2,14 @@
 from collections import defaultdict
 from typing import Dict, List, Optional, Tuple
 
+from fideslang import manifests
+from fideslang.models import Organization, System, SystemMetadata
 from pydantic import AnyHttpUrl
 
 from fidesctl.cli.utils import handle_cli_response
 from fidesctl.core import api
 from fidesctl.core.api_helpers import get_server_resource, get_server_resources
 from fidesctl.core.parse import parse
-from fideslang import manifests
-from fideslang.models import Organization, System, SystemMetadata
 
 from .filters import filter_aws_systems
 from .utils import echo_green, echo_red

src/fidesctl/core/utils.py

@@ -8,9 +8,8 @@
 import jwt
 import requests
 import sqlalchemy
-from sqlalchemy.engine import Engine
-
 from fideslang.models import DatasetField
+from sqlalchemy.engine import Engine
 
 logger = logging.getLogger("server_api")