Setup the core repo, create helper commands to access the database.

* Add database interaction commands

* Pull down changes in orchestrator code

* Added warning statement for unexpected keyword arguments in Domain model class

* Setup database intialisation

* Changed some app settings

* Add type hinting information

* Bumped Nwa-stdlib, oauth2-lib

* Setup Migrations environment correctly

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.2rc2
+current_version = 0.0.2rc11
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?

orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "0.0.2rc2"
+__version__ = "0.0.2rc11"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings, oauth2_settings

orchestrator/app.py

@@ -12,10 +12,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Optional, Type
+from typing import Dict, Optional, Type
 
 import sentry_sdk
 import structlog
+import typer
 from fastapi.applications import FastAPI
 from fastapi_etag.dependency import add_exception_handler
 from nwastdlib.logging import initialise_logging
@@ -34,8 +35,10 @@
 
 from orchestrator.api.api_v1.api import api_router
 from orchestrator.api.error_handling import ProblemDetailException
+from orchestrator.cli.main import app as cli_app
 from orchestrator.db import db, init_database
 from orchestrator.db.database import DBSessionMiddleware
+from orchestrator.domain import SUBSCRIPTION_MODEL_REGISTRY, SubscriptionModel
 from orchestrator.exception_handlers import form_error_handler, problem_detail_handler
 from orchestrator.forms import FormException
 from orchestrator.settings import AppSettings, app_settings, tracer_provider
@@ -120,3 +123,33 @@ def add_sentry(
             integrations=[SqlalchemyIntegration(), RedisIntegration()],
         )
         self.add_middleware(SentryAsgiMiddleware)
+
+    @staticmethod
+    def register_subscription_models(product_to_subscription_model_mapping: Dict[str, Type[SubscriptionModel]]) -> None:
+        """
+        Register your subscription models.
+
+        This method is needed to register your subscription models inside the orchestrator core.
+
+        Args:
+            product_to_subscription_model_mapping: The dictionary should contain a mapping of products to SubscriptionModels.
+                The selection will be done depending on the name of the product.
+
+        Returns:
+            None:
+
+        Examples:
+            product_to_subscription_model_mapping = {
+                "Generic Product One": GenericProductModel,
+                "Generic Product Two": GenericProductModel,
+            }
+
+        """
+        SUBSCRIPTION_MODEL_REGISTRY.update(product_to_subscription_model_mapping)
+
+
+main_typer_app = typer.Typer()
+main_typer_app.add_typer(cli_app, name="orchestrator", help="The are the orchestrator cli commands")
+
+if __name__ == "__main__":
+    main_typer_app()

orchestrator/cli/database.py

@@ -0,0 +1,162 @@
+import os
+from shutil import copyfile
+from typing import List, Optional
+
+import jinja2
+import typer
+from alembic import command
+from alembic.config import Config
+from structlog import get_logger
+
+import orchestrator
+
+logger = get_logger(__name__)
+
+app: typer.Typer = typer.Typer()
+
+orchestrator_module_location = os.path.dirname(orchestrator.__file__)
+migration_dir = "migrations"
+
+loader = jinja2.FileSystemLoader(os.path.join(orchestrator_module_location, f"{migration_dir}/templates"))
+jinja_env = jinja2.Environment(
+    loader=loader, autoescape=True, lstrip_blocks=True, trim_blocks=True, undefined=jinja2.StrictUndefined
+)
+
+
+def alembic_cfg() -> Config:
+    cfg = Config("alembic.ini")
+    version_locations = cfg.get_main_option("version_locations")
+    cfg.set_main_option(
+        "version_locations", f"{version_locations} {orchestrator_module_location}/{migration_dir}/versions/schema"
+    )
+    logger.info("Version Locations", locations=cfg.get_main_option("version_locations"))
+    return cfg
+
+
+@app.command(
+    help="Initialize an empty migrations environment. This command will throw an exception when it detects conflicting files and directories."
+)
+def init() -> None:
+    """
+    Initialise the migrations directory.
+
+    This command will initialize a migration directory for the orchestrator core application and setup a correct
+    migration environment.
+
+    Returns:
+        None
+
+    """
+
+    if os.access(migration_dir, os.F_OK) and os.listdir(migration_dir):
+        raise OSError(f"Directory {migration_dir} already exists and is not empty")
+
+    logger.info("Creating directory", directory=os.path.abspath(migration_dir))
+    os.makedirs(migration_dir)
+    versions = os.path.join(migration_dir, "versions")
+    logger.info("Creating directory", directory=os.path.abspath(versions))
+    os.makedirs(versions)
+    versions_schema = os.path.join(migration_dir, "versions/schema")
+    logger.info("Creating directory", directory=os.path.abspath(versions_schema))
+    os.makedirs(versions_schema)
+
+    source_env_py = os.path.join(orchestrator_module_location, f"{migration_dir}/templates/env.py.j2")
+    env_py = os.path.join(migration_dir, "env.py")
+    logger.info("Creating file", file=os.path.abspath(env_py))
+    copyfile(source_env_py, env_py)
+
+    source_script_py_mako = os.path.join(orchestrator_module_location, f"{migration_dir}/script.py.mako")
+    script_py_mako = os.path.join(migration_dir, "script.py.mako")
+    logger.info("Creating file", file=os.path.abspath(script_py_mako))
+    copyfile(source_script_py_mako, script_py_mako)
+
+    source_helpers_py = os.path.join(orchestrator_module_location, f"{migration_dir}/templates/helpers.py.j2")
+    helpers_py = os.path.join(migration_dir, "helpers.py")
+    logger.info("Creating file", file=os.path.abspath(helpers_py))
+    copyfile(source_helpers_py, helpers_py)
+
+    template = jinja_env.get_template("alembic.ini.j2")
+
+    if not os.access(os.path.join(os.getcwd(), "alembic.ini"), os.F_OK):
+        logger.info("Creating file", file=os.path.join(os.getcwd(), "alembic.ini"))
+        with open(os.path.join(os.getcwd(), "alembic.ini"), "w") as alembic_ini:
+            alembic_ini.write(template.render(migrations_dir=migration_dir))
+    else:
+        logger.info("Skipping Alembic.ini file. It already exists")
+
+
+@app.command(help="Get the database heads")
+def heads() -> None:
+    command.heads(alembic_cfg())
+
+
+@app.command(help="Merge database revisions.")
+def merge(
+    revisions: Optional[List[str]] = typer.Argument(
+        None, help="Add the revision you would like to merge to this command."
+    ),
+    message: str = typer.Option(None, "--message", "-m", help="The revision message"),
+) -> None:
+    """
+    Merge database revisions.
+
+    Args:
+        revisions: List of revisions to merge
+        message: Optional message for the revision.
+
+    Returns:
+        None
+
+    """
+    command.merge(alembic_cfg(), revisions, message=message)
+
+
+@app.command()
+def upgrade(revision: Optional[str] = typer.Argument(None, help="Rev id to upgrade to")) -> None:
+    """
+    Upgrade the database.
+
+    Args:
+        revision: Optional argument to indicate where to upgrade to.
+
+    Returns:
+        None
+
+    """
+    command.upgrade(alembic_cfg(), revision)
+
+
+@app.command()
+def downgrade(revision: Optional[str] = typer.Argument(None, help="Rev id to upgrade to")) -> None:
+    """
+    Downgrade the database.
+
+    Args:
+        revision: Optional argument to indicate where to downgrade to.
+
+    Returns:
+        None
+
+    """
+    command.downgrade(alembic_cfg(), revision)
+
+
+@app.command()
+def revision(
+    message: str = typer.Option(None, "--message", "-m", help="The revision message"),
+    autogenerate: bool = typer.Option(False, help="Detect schema changes and add migrations"),
+    head: str = typer.Option(None, help="Determine the head the head you need to add your migration to."),
+) -> None:
+    """
+    Create a new revision file.
+
+    Args:
+        message: The revision message
+        autogenerate: Whether to detect schema changes.
+        head: To which head the migration applies
+
+    Returns:
+        None
+
+    """
+    command.revision(alembic_cfg(), message, autogenerate=autogenerate, head=head)

orchestrator/cli/main.py

@@ -11,16 +11,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import typer
-from surf import load_surf_cli
 
-from orchestrator.cli import scheduler
+from orchestrator.cli import database, scheduler
 
 app = typer.Typer()
-app.add_typer(scheduler.app, name="scheduler")
+app.add_typer(scheduler.app, name="scheduler", help="Access all the scheduler functions")
+app.add_typer(database.app, name="db", help="Interact with the application database")
 
-load_surf_cli(app)
 
 if __name__ == "__main__":
     app()

orchestrator/config/__init__.py

@@ -37,5 +37,6 @@
 NODE_SUBSCRIPTION_ID = "node_subscription_id"
 PORT_MODE = "port_mode"
 INTERNETPINNEN_PREFIX_SUBSCRIPTION_ID = "internetpinnen_prefix_subscription_id"
+PEER_GROUP_SUBSCRIPTION_ID = "peer_group_subscription_id"
 
 MAX_SPEED_POSSIBLE = 1000_000

orchestrator/db/__init__.py

@@ -86,4 +86,5 @@ def init_database(settings: AppSettings) -> Database:
     "UtcTimestamp",
     "UtcTimestampException",
     "db",
+    "init_database",
 ]

orchestrator/domain/__init__.py

@@ -20,4 +20,4 @@
 
 __doc__ = make_product_type_index_doc(SUBSCRIPTION_MODEL_REGISTRY)
 
-__all__ = ["SubscriptionModel"]
+__all__ = ["SubscriptionModel", "SUBSCRIPTION_MODEL_REGISTRY"]

orchestrator/domain/base.py

@@ -83,6 +83,11 @@ def __init_subclass__(
         super().__init_subclass__()
         cls._find_special_fields()
 
+        if kwargs.keys():
+            logger.warning(
+                "Unexpected keyword arguments in domain model class", class_name=cls.__name__, kwargs=kwargs.keys()
+            )
+
         # Check if child subscription instance models conform to the same lifecycle
         for product_block_field_name, product_block_field_type in cls._product_block_fields_.items():
             if is_list_type(product_block_field_type) or is_optional_type(product_block_field_type):
@@ -327,6 +332,8 @@ def __call__(self, *args: Any, **kwargs: Any) -> B:
             self.description = product_block.description
             self.tag = product_block.tag
 
+        kwargs["name"] = self.name
+
         return super().__call__(*args, **kwargs)
 
 
@@ -366,11 +373,11 @@ class ProductBlockModel(DomainModel, metaclass=ProductBlockModelMeta):
     """
 
     __names__: ClassVar[List[str]]
-    name: ClassVar[str]
     product_block_id: ClassVar[UUID]
     description: ClassVar[str]
     tag: ClassVar[str]
 
+    name: str  # Product block name. This needs to be an instance var because its part of the API (we expose it to the frontend)
     subscription_instance_id: UUID = Field(default_factory=uuid4)
     label: Optional[str] = None
 
@@ -389,7 +396,8 @@ def __init_subclass__(
             cls.__base_type__ = cls
             cls.__names__ = product_block_names or [cls.name]
 
-        register_specialized_type(cls, lifecycle)
+        if product_block_name is not None or lifecycle is not None:
+            register_specialized_type(cls, lifecycle)
 
         cls.__doc__ = make_product_block_docstring(cls, lifecycle)
 
@@ -724,7 +732,8 @@ def __init_subclass__(
         if is_base:
             cls.__base_type__ = cls
 
-        register_specialized_type(cls, lifecycle)
+        if is_base or lifecycle:
+            register_specialized_type(cls, lifecycle)
 
         cls.__doc__ = make_subscription_model_docstring(cls, lifecycle)
 

orchestrator/forms/__init__.py

@@ -26,7 +26,7 @@
 logger = structlog.get_logger(__name__)
 
 
-__all__ = ("generate_form", "post_process", "FormNotCompleteError", "FormValidationError")
+__all__ = ("generate_form", "post_process", "FormNotCompleteError", "FormValidationError", "FormException")
 
 
 class FormException(Exception):

orchestrator/migrations/templates/alembic.ini.j2

@@ -0,0 +1,45 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+file_template = %%(year)d-%%(month).2d-%%(day).2d_%%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+script_location = {{ migrations_dir }}
+version_locations = {{ migrations_dir }}/versions/general
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S