Unnest ops API code (#3318)

CHANGELOG.md

@@ -22,6 +22,7 @@ The types of changes are:
 - Remove logging within the Celery creation function [#3303](https://github.com/ethyca/fides/pull/3303)
 - Update how generic endpoint generation works [#3304](https://github.com/ethyca/fides/pull/3304)
 - Restrict strack-trace logging when not in Dev mode [#3081](https://github.com/ethyca/fides/pull/3081)
+- Moved all of the dirs from `fides.api.ops` into `fides.api` [#3318](https://github.com/ethyca/fides/pull/3318)
 
 ### Added
 

MANIFEST.in

@@ -12,6 +12,6 @@ include src/fides/templates/fides_datamap_template.xlsx
 recursive-include src/fides/data *
 recursive-include src/fides/ui-build *
 
-graft src/fides/api/ops/email_templates/templates
+graft src/fides/api/email_templates/templates
 
 global-exclude *.pyc

data/saas/saas_connector_scaffolding/test_new_task.jinja

@@ -1,6 +1,6 @@
 import pytest
 
-from fides.api.ops.models.policy import Policy
+from fides.api.models.policy import Policy
 from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner
 
 

docs/fides/docs/development/development_tips.md

@@ -10,7 +10,7 @@ If you're looking for a general getting started guide, please see the [Overview]
 
 ### API URLs
 
-We define API URLs for specific API versions as constants within `fides.api.ops.api.v1.urn_registry` (where `v1` can be substituted for that particular API version), then import those URLs into their specific API views. Since we are on the first version, there is no clear precedent set for overriding URLs between versions yet. The most likely change is that we'll override the `APIRouter` class instantiation with a different base path (ie. `/api/v2` instead of `/api/v1`). For example:
+We define API URLs for specific API versions as constants within `fides.api.api.v1.urn_registry` (where `v1` can be substituted for that particular API version), then import those URLs into their specific API views. Since we are on the first version, there is no clear precedent set for overriding URLs between versions yet. The most likely change is that we'll override the `APIRouter` class instantiation with a different base path (ie. `/api/v2` instead of `/api/v1`). For example:
 
 ```sh
 PRIVACY_REQUEST = "/privacy-request"
@@ -29,8 +29,8 @@ SQLAlchemy is an Object Relational Mapper, allowing us to avoid writing direct d
 
 ### Adding models
 
-Database tables are defined with model classes. Model files should live in `src/fides/api/ops/models/`. Individual model classes must inherit from our custom base class at `fides.api.ops.db.base_class.Base` to ensure uniformity within the database. Multiple models per file are encouraged so long as they fit the same logical delineation within the project. An example model declaration is added below. For a comprehensive guide see: <https://docs.sqlalchemy.org/en/14/orm/mapping_styles.html#declarative-mapping>
-You should also import your model in `src/fides/api/ops/db/base.py` so it is visible for alembic.
+Database tables are defined with model classes. Model files should live in `src/fides/api/models/`. Individual model classes must inherit from our custom base class at `fides.api.db.base_class.Base` to ensure uniformity within the database. Multiple models per file are encouraged so long as they fit the same logical delineation within the project. An example model declaration is added below. For a comprehensive guide see: <https://docs.sqlalchemy.org/en/14/orm/mapping_styles.html#declarative-mapping>
+You should also import your model in `src/fides/api/db/base.py` so it is visible for alembic.
 
 ```python
 class Book(Base):
@@ -48,7 +48,7 @@ When models are added to the project, we must then add them to the database in a
 
 Once you've added database tables via project models, you're ready to read, write, and update them via Python code. Some examples of common use cases here are listed below. Official documentation is here: <https://docs.sqlalchemy.org/en/14/orm/query.html#sqlalchemy.orm.Query>.
 
-- Import our application's database session: `from fides.api.ops.db.session import get_db_session`
+- Import our application's database session: `from fides.api.db.session import get_db_session`
 - Instantiate the database interaction object:
 
 ```python

noxfiles/ci_nox.py

@@ -74,7 +74,7 @@ def mypy(session: nox.Session) -> None:
 def pylint(session: nox.Session) -> None:
     """Run the 'pylint' code linter."""
     install_requirements(session)
-    command = ("pylint", "src", "noxfiles", "noxfile.py")
+    command = ("pylint", "src", "noxfiles", "noxfile.py", "--jobs", "0")
     session.run(*command)
 
 

scripts/create_test_data.py

@@ -7,31 +7,31 @@
 from sqlalchemy import orm
 
 from fides.api.ctl.database.database import init_db
-from fides.api.ops.db.session import get_db_session
-from fides.api.ops.models.audit_log import AuditLog, AuditLogAction
-from fides.api.ops.models.client import ClientDetail
-from fides.api.ops.models.connectionconfig import (
+from fides.api.db.session import get_db_session
+from fides.api.models.audit_log import AuditLog, AuditLogAction
+from fides.api.models.client import ClientDetail
+from fides.api.models.connectionconfig import (
     AccessLevel,
     ConnectionConfig,
     ConnectionType,
 )
-from fides.api.ops.models.fides_user import FidesUser
-from fides.api.ops.models.policy import ActionType, Policy, Rule, RuleTarget
-from fides.api.ops.models.privacy_request import (
+from fides.api.models.fides_user import FidesUser
+from fides.api.models.policy import ActionType, Policy, Rule, RuleTarget
+from fides.api.models.privacy_request import (
     ExecutionLog,
     ExecutionLogStatus,
     PrivacyRequest,
     PrivacyRequestStatus,
 )
-from fides.api.ops.models.storage import StorageConfig
-from fides.api.ops.schemas.redis_cache import Identity
-from fides.api.ops.schemas.storage.storage import (
+from fides.api.models.storage import StorageConfig
+from fides.api.schemas.redis_cache import Identity
+from fides.api.schemas.storage.storage import (
     FileNaming,
     ResponseFormat,
     StorageDetails,
     StorageType,
 )
-from fides.api.ops.util.data_category import DataCategory
+from fides.api.util.data_category import DataCategory
 from fides.core.config import get_config
 
 CONFIG = get_config()

scripts/quickstart.py

@@ -13,10 +13,10 @@
 import yaml
 from loguru import logger
 
-from fides.api.ops.api.v1 import urn_registry as ops_urls
-from fides.api.ops.api.v1.scope_registry import SCOPE_REGISTRY
-from fides.api.ops.models.connectionconfig import ConnectionType
-from fides.api.ops.models.policy import ActionType
+from fides.api.api.v1 import urn_registry as ops_urls
+from fides.api.api.v1.scope_registry import SCOPE_REGISTRY
+from fides.api.models.connectionconfig import ConnectionType
+from fides.api.models.policy import ActionType
 from fides.core.config import get_config
 
 CONFIG = get_config()

src/fides/api/ops/analytics.py → src/fides/api/analytics.py

@@ -8,7 +8,7 @@
 from loguru import logger as log
 
 from fides import __version__ as fides_version
-from fides.api.ops.models.registration import UserRegistration
+from fides.api.models.registration import UserRegistration
 from fides.core.config import CONFIG
 
 

src/fides/api/ops/api/deps.py → src/fides/api/api/deps.py

@@ -3,9 +3,9 @@
 from fastapi import Depends
 from sqlalchemy.orm import Session
 
-from fides.api.ops.common_exceptions import FunctionalityNotConfigured
-from fides.api.ops.db.session import get_db_engine, get_db_session
-from fides.api.ops.util.cache import get_cache as get_redis_connection
+from fides.api.common_exceptions import FunctionalityNotConfigured
+from fides.api.db.session import get_db_engine, get_db_session
+from fides.api.util.cache import get_cache as get_redis_connection
 from fides.core.config import CONFIG, FidesConfig
 from fides.core.config import get_config as get_app_config
 from fides.core.config.config_proxy import ConfigProxy

src/fides/api/ops/api/v1/api.py → src/fides/api/api/v1/api.py

@@ -1,4 +1,4 @@
-from fides.api.ops.api.v1.endpoints import (
+from fides.api.api.v1.endpoints import (
     config_endpoints,
     connection_endpoints,
     connection_type_endpoints,
@@ -24,7 +24,7 @@
     user_endpoints,
     user_permission_endpoints,
 )
-from fides.api.ops.util.api_router import APIRouter
+from fides.api.util.api_router import APIRouter
 
 api_router = APIRouter()
 api_router.include_router(config_endpoints.router)

src/fides/api/ops/api/v1/endpoints/config_endpoints.py → src/fides/api/api/v1/endpoints/config_endpoints.py

@@ -6,15 +6,15 @@
 from sqlalchemy.orm import Session
 from starlette.status import HTTP_200_OK
 
-from fides.api.ops.api import deps
-from fides.api.ops.api.v1 import scope_registry as scopes
-from fides.api.ops.api.v1 import urn_registry as urls
-from fides.api.ops.models.application_config import ApplicationConfig
-from fides.api.ops.oauth.utils import verify_oauth_client
-from fides.api.ops.schemas.application_config import (
+from fides.api.api import deps
+from fides.api.api.v1 import scope_registry as scopes
+from fides.api.api.v1 import urn_registry as urls
+from fides.api.models.application_config import ApplicationConfig
+from fides.api.oauth.utils import verify_oauth_client
+from fides.api.schemas.application_config import (
     ApplicationConfig as ApplicationConfigSchema,
 )
-from fides.api.ops.util.api_router import APIRouter
+from fides.api.util.api_router import APIRouter
 from fides.core.config import censor_config
 from fides.core.config import get_config as get_app_config
 

src/fides/api/ops/api/v1/endpoints/connection_endpoints.py → src/fides/api/api/v1/endpoints/connection_endpoints.py

@@ -15,48 +15,45 @@
 from sqlalchemy_utils import escape_like
 from starlette.status import HTTP_200_OK, HTTP_204_NO_CONTENT, HTTP_404_NOT_FOUND
 
-from fides.api.ops.api import deps
-from fides.api.ops.api.v1.scope_registry import (
+from fides.api.api import deps
+from fides.api.api.v1.scope_registry import (
     CONNECTION_CREATE_OR_UPDATE,
     CONNECTION_DELETE,
     CONNECTION_READ,
 )
-from fides.api.ops.api.v1.urn_registry import (
+from fides.api.api.v1.urn_registry import (
     CONNECTION_BY_KEY,
     CONNECTION_SECRETS,
     CONNECTION_TEST,
     CONNECTIONS,
     V1_URL_PREFIX,
 )
-from fides.api.ops.common_exceptions import (
-    ClientUnsuccessfulException,
-    ConnectionException,
-)
-from fides.api.ops.models.connectionconfig import (
+from fides.api.common_exceptions import ClientUnsuccessfulException, ConnectionException
+from fides.api.models.connectionconfig import (
     ConnectionConfig,
     ConnectionTestStatus,
     ConnectionType,
 )
-from fides.api.ops.oauth.utils import verify_oauth_client
-from fides.api.ops.schemas.connection_configuration import connection_secrets_schemas
-from fides.api.ops.schemas.connection_configuration.connection_config import (
+from fides.api.oauth.utils import verify_oauth_client
+from fides.api.schemas.connection_configuration import connection_secrets_schemas
+from fides.api.schemas.connection_configuration.connection_config import (
     BulkPutConnectionConfiguration,
     ConnectionConfigurationResponse,
     CreateConnectionConfigurationWithSecrets,
     SystemType,
     TestStatus,
 )
-from fides.api.ops.schemas.connection_configuration.connection_secrets import (
+from fides.api.schemas.connection_configuration.connection_secrets import (
     TestStatusMessage,
 )
-from fides.api.ops.service.connectors import get_connector
-from fides.api.ops.util.api_router import APIRouter
-from fides.api.ops.util.connection_util import (
+from fides.api.service.connectors import get_connector
+from fides.api.util.api_router import APIRouter
+from fides.api.util.connection_util import (
     patch_connection_configs,
     requeue_requires_input_requests,
     validate_secrets,
 )
-from fides.api.ops.util.logger import Pii
+from fides.api.util.logger import Pii
 
 router = APIRouter(tags=["Connections"], prefix=V1_URL_PREFIX)
 

src/fides/api/ops/api/v1/endpoints/connection_type_endpoints.py → src/fides/api/api/v1/endpoints/connection_type_endpoints.py

@@ -6,21 +6,21 @@
 from fastapi_pagination.bases import AbstractPage
 from starlette.status import HTTP_404_NOT_FOUND
 
-from fides.api.ops.api.v1.scope_registry import CONNECTION_TYPE_READ
-from fides.api.ops.api.v1.urn_registry import (
+from fides.api.api.v1.scope_registry import CONNECTION_TYPE_READ
+from fides.api.api.v1.urn_registry import (
     CONNECTION_TYPE_SECRETS,
     CONNECTION_TYPES,
     V1_URL_PREFIX,
 )
-from fides.api.ops.common_exceptions import NoSuchConnectionTypeSecretSchemaError
-from fides.api.ops.models.policy import ActionType
-from fides.api.ops.oauth.utils import verify_oauth_client
-from fides.api.ops.schemas.connection_configuration.connection_config import (
+from fides.api.common_exceptions import NoSuchConnectionTypeSecretSchemaError
+from fides.api.models.policy import ActionType
+from fides.api.oauth.utils import verify_oauth_client
+from fides.api.schemas.connection_configuration.connection_config import (
     ConnectionSystemTypeMap,
     SystemType,
 )
-from fides.api.ops.util.api_router import APIRouter
-from fides.api.ops.util.connection_type import (
+from fides.api.util.api_router import APIRouter
+from fides.api.util.connection_type import (
     connection_type_secret_schema,
     get_connection_types,
 )

src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py → src/fides/api/api/v1/endpoints/consent_request_endpoints.py

@@ -21,37 +21,37 @@
     HTTP_500_INTERNAL_SERVER_ERROR,
 )
 
-from fides.api.ctl.database.seed import DEFAULT_CONSENT_POLICY
-from fides.api.ops.api.deps import get_config_proxy, get_db
-from fides.api.ops.api.v1.endpoints.privacy_request_endpoints import (
+from fides.api.api.deps import get_config_proxy, get_db
+from fides.api.api.v1.endpoints.privacy_request_endpoints import (
     create_privacy_request_func,
 )
-from fides.api.ops.api.v1.endpoints.utils import validate_start_and_end_filters
-from fides.api.ops.api.v1.scope_registry import CONSENT_READ
-from fides.api.ops.api.v1.urn_registry import (
+from fides.api.api.v1.endpoints.utils import validate_start_and_end_filters
+from fides.api.api.v1.scope_registry import CONSENT_READ
+from fides.api.api.v1.urn_registry import (
     CONSENT_REQUEST,
     CONSENT_REQUEST_PREFERENCES,
     CONSENT_REQUEST_PREFERENCES_WITH_ID,
     CONSENT_REQUEST_VERIFY,
     V1_URL_PREFIX,
 )
-from fides.api.ops.common_exceptions import (
+from fides.api.common_exceptions import (
     FunctionalityNotConfigured,
     IdentityVerificationException,
     MessageDispatchException,
 )
-from fides.api.ops.models.messaging import get_messaging_method
-from fides.api.ops.models.privacy_request import (
+from fides.api.ctl.database.seed import DEFAULT_CONSENT_POLICY
+from fides.api.models.messaging import get_messaging_method
+from fides.api.models.privacy_request import (
     Consent,
     ConsentRequest,
     ProvidedIdentity,
     ProvidedIdentityType,
 )
-from fides.api.ops.oauth.utils import verify_oauth_client
-from fides.api.ops.schemas.messaging.messaging import MessagingMethod
-from fides.api.ops.schemas.privacy_request import BulkPostPrivacyRequests
-from fides.api.ops.schemas.privacy_request import Consent as ConsentSchema
-from fides.api.ops.schemas.privacy_request import (
+from fides.api.oauth.utils import verify_oauth_client
+from fides.api.schemas.messaging.messaging import MessagingMethod
+from fides.api.schemas.privacy_request import BulkPostPrivacyRequests
+from fides.api.schemas.privacy_request import Consent as ConsentSchema
+from fides.api.schemas.privacy_request import (
     ConsentPreferences,
     ConsentPreferencesWithVerificationCode,
     ConsentReport,
@@ -60,13 +60,13 @@
     PrivacyRequestCreate,
     VerificationCode,
 )
-from fides.api.ops.schemas.redis_cache import Identity
-from fides.api.ops.service._verification import send_verification_code_to_user
-from fides.api.ops.util.api_router import APIRouter
-from fides.api.ops.util.consent_util import (
+from fides.api.schemas.redis_cache import Identity
+from fides.api.service._verification import send_verification_code_to_user
+from fides.api.util.api_router import APIRouter
+from fides.api.util.consent_util import (
     get_or_create_fides_user_device_id_provided_identity,
 )
-from fides.api.ops.util.logger import Pii
+from fides.api.util.logger import Pii
 from fides.core.config import CONFIG
 from fides.core.config.config_proxy import ConfigProxy