Add Manual Migration Feature

CHANGELOG.md

@@ -27,6 +27,7 @@ The types of changes are:
 - A JavaScript modal to copy a script tag for `fides.js` [#3238](https://github.com/ethyca/fides/pull/3238)
 - Access and erasure support for OneSignal [#3199](https://github.com/ethyca/fides/pull/3199)
 - Add the ability to "inject" location into `/fides.js` bundles and cache responses for one hour [#3272](https://github.com/ethyca/fides/pull/3272)
+- Added an `automigrate` database setting [#3220](https://github.com/ethyca/fides/pull/3220)
 
 ### Changed
 

src/fides/api/app_setup.py

@@ -0,0 +1,200 @@
+"""
+Contains utility functions that set up the application webserver.
+"""
+from logging import DEBUG
+from typing import List, Optional, Pattern, Union
+
+from fastapi import FastAPI
+from loguru import logger
+from redis.exceptions import RedisError, ResponseError
+from slowapi.errors import RateLimitExceeded  # type: ignore
+from slowapi.extension import Limiter, _rate_limit_exceeded_handler  # type: ignore
+from slowapi.middleware import SlowAPIMiddleware  # type: ignore
+from slowapi.util import get_remote_address  # type: ignore
+from starlette.middleware.cors import CORSMiddleware
+
+import fides
+from fides.api.ctl import view
+from fides.api.ctl.database.database import configure_db
+from fides.api.ctl.database.seed import create_or_update_parent_user
+from fides.api.ctl.routes import admin, crud, generate, health, system, validate
+from fides.api.ctl.utils.errors import FidesError
+from fides.api.ctl.utils.logger import setup as setup_logging
+from fides.api.ops.api.deps import get_api_session
+from fides.api.ops.api.v1.api import api_router
+from fides.api.ops.api.v1.exception_handlers import ExceptionHandlers
+from fides.api.ops.common_exceptions import (
+    FunctionalityNotConfigured,
+    RedisConnectionError,
+)
+from fides.api.ops.models.application_config import ApplicationConfig
+from fides.api.ops.oauth.utils import get_root_client, verify_oauth_client_prod
+from fides.api.ops.service.connectors.saas.connector_registry_service import (
+    update_saas_configs,
+)
+
+# pylint: disable=wildcard-import, unused-wildcard-import
+from fides.api.ops.service.saas_request.override_implementations import *
+from fides.api.ops.util.cache import get_cache
+from fides.api.ops.util.system_manager_oauth_util import (
+    get_system_fides_key,
+    get_system_schema,
+    verify_oauth_client_for_system_from_fides_key_cli,
+    verify_oauth_client_for_system_from_request_body_cli,
+)
+from fides.core.config import CONFIG
+
+VERSION = fides.__version__
+
+ROUTERS = crud.routers + [  # type: ignore[attr-defined]
+    admin.router,
+    generate.router,
+    health.router,
+    validate.router,
+    view.router,
+    system.system_connections_router,
+    system.system_router,
+]
+
+
+def create_fides_app(
+    cors_origins: Union[str, List[str]] = CONFIG.security.cors_origins,
+    cors_origin_regex: Optional[Pattern] = CONFIG.security.cors_origin_regex,
+    routers: List = ROUTERS,
+    app_version: str = VERSION,
+    request_rate_limit: str = CONFIG.security.request_rate_limit,
+    rate_limit_prefix: str = CONFIG.security.rate_limit_prefix,
+    security_env: str = CONFIG.security.env,
+) -> FastAPI:
+    """Return a properly configured application."""
+    setup_logging(
+        CONFIG.logging.level,
+        serialize=CONFIG.logging.serialization,
+        desination=CONFIG.logging.destination,
+    )
+    logger.bind(api_config=CONFIG.logging.json()).debug(
+        "Logger configuration options in use"
+    )
+
+    fastapi_app = FastAPI(title="fides", version=app_version)
+    fastapi_app.state.limiter = Limiter(
+        default_limits=[request_rate_limit],
+        headers_enabled=True,
+        key_prefix=rate_limit_prefix,
+        key_func=get_remote_address,
+        retry_after="http-date",
+    )
+    fastapi_app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
+    for handler in ExceptionHandlers.get_handlers():
+        fastapi_app.add_exception_handler(FunctionalityNotConfigured, handler)
+    fastapi_app.add_middleware(SlowAPIMiddleware)
+
+    if cors_origins or cors_origin_regex:
+        fastapi_app.add_middleware(
+            CORSMiddleware,
+            allow_origins=[str(origin) for origin in cors_origins],
+            allow_origin_regex=cors_origin_regex,
+            allow_credentials=True,
+            allow_methods=["*"],
+            allow_headers=["*"],
+        )
+
+    for router in routers:
+        fastapi_app.include_router(router)
+    fastapi_app.include_router(api_router)
+
+    if security_env == "dev":
+        # This removes auth requirements for specific endpoints
+        fastapi_app.dependency_overrides[verify_oauth_client_prod] = get_root_client
+        fastapi_app.dependency_overrides[
+            verify_oauth_client_for_system_from_request_body_cli
+        ] = get_system_schema
+        fastapi_app.dependency_overrides[
+            verify_oauth_client_for_system_from_fides_key_cli
+        ] = get_system_fides_key
+    elif security_env == "prod":
+        # This is the most secure, so all security deps are maintained
+        pass
+
+    return fastapi_app
+
+
+def log_startup() -> None:
+    """Log application startup and other information."""
+    logger.info(f"Starting Fides - v{VERSION}")
+    logger.info(
+        "Startup configuration: reloading = {}, dev_mode = {}",
+        CONFIG.hot_reloading,
+        CONFIG.dev_mode,
+    )
+    logger.info("Startup configuration: pii logging = {}", CONFIG.logging.log_pii)
+
+    if CONFIG.logging.level == DEBUG:
+        logger.warning(
+            "WARNING: log level is DEBUG, so sensitive or personal data may be logged. "
+            "Set FIDES__LOGGING__LEVEL to INFO or higher in production."
+        )
+        CONFIG.log_all_config_values()
+
+
+async def run_database_startup() -> None:
+    """
+    Perform all relevant database startup activities/configuration for the
+    application webserver.
+    """
+
+    if not CONFIG.database.sync_database_uri:
+        raise FidesError("No database uri provided")
+
+    if CONFIG.database.automigrate:
+        await configure_db(
+            CONFIG.database.sync_database_uri, samples=CONFIG.database.load_samples
+        )
+    else:
+        logger.info("Skipping auto-migration due to 'automigrate' configuration value.")
+
+    try:
+        create_or_update_parent_user()
+    except Exception as e:
+        logger.error("Error creating parent user: {}", str(e))
+        raise FidesError(f"Error creating parent user: {str(e)}")
+
+    db = get_api_session()
+    logger.info("Loading config settings into database...")
+    try:
+        ApplicationConfig.update_config_set(db, CONFIG)
+    except Exception as e:
+        logger.error("Error occurred writing config settings to database: {}", str(e))
+        raise FidesError(
+            f"Error occurred writing config settings to database: {str(e)}"
+        )
+    finally:
+        db.close()
+
+    logger.info("Validating SaaS connector templates...")
+    try:
+        update_saas_configs(db)
+        logger.info("Finished loading SaaS templates")
+    except Exception as e:
+        logger.error(
+            "Error occurred during SaaS connector template validation: {}",
+            str(e),
+        )
+        return
+    finally:
+        db.close()
+    db.close()
+
+
+def check_redis() -> None:
+    """Check that Redis is healthy."""
+
+    logger.info("Running Cache connection test...")
+
+    try:
+        get_cache()
+    except (RedisConnectionError, RedisError, ResponseError) as e:
+        logger.error("Connection to cache failed: {}", str(e))
+        return
+    else:
+        logger.debug("Connection to cache succeeded")

src/fides/api/ctl/database/database.py

@@ -41,9 +41,11 @@ def upgrade_db(alembic_config: Config, revision: str = "head") -> None:
     command.upgrade(alembic_config, revision)
 
 
-async def init_db(database_url: str, samples: bool = False) -> None:
+async def migrate_db(database_url: str, samples: bool = False) -> None:
     """
-    Runs the migrations and creates all of the database objects.
+    Runs migrations and creates database objects if needed.
+
+    Safe to run on an existing database when upgrading Fides version.
     """
     log.info("Initializing database")
     alembic_config = get_alembic_config(database_url)
@@ -104,7 +106,7 @@ async def configure_db(database_url: str, samples: bool = False) -> None:
     """Set up the db to be used by the app."""
     try:
         create_db_if_not_exists(database_url)
-        await init_db(database_url, samples=samples)
+        await migrate_db(database_url, samples=samples)
     except InvalidCiphertextError as cipher_error:
         log.error(
             "Unable to configure database due to a decryption error! Check to ensure your `app_encryption_key` has not changed."

src/fides/api/ctl/routes/admin.py

@@ -16,7 +16,7 @@
 
 class DBActions(str, Enum):
     "The available path parameters for the `/admin/db/{action}` endpoint."
-    init = "init"
+    migrate = "migrate"
     reset = "reset"
 
 
@@ -31,15 +31,17 @@ async def db_action(action: DBActions) -> Dict:
     """
     Initiate one of the enumerated DBActions.
     """
+    action_text = "migrated"
 
-    action_text = "initialized"
     if action == DBActions.reset:
         if not CONFIG.dev_mode:
             raise errors.FunctionalityNotConfigured(
                 "unable to reset fides database outside of dev_mode."
             )
 
         database.reset_db(CONFIG.database.sync_database_uri)
-        action_text = DBActions.reset
+        action_text = "reset"
+
     await database.configure_db(CONFIG.database.sync_database_uri)
+
     return {"data": {"message": f"fides database {action_text}"}}