Rework Gafaelfawr schema management

Make Gafaelfawr's schema management match the new recommended
practices in Safir, including a stack of schema migrations that
create the database from scratch. This means that direct upgrades
from versions earlier than 10.0.0 are no longer supported, since
they will attempt to recreate the schema unless one carefully uses
`alembic stamp` first. Document that.

Change the name of the SQLAlchemy declarative base to `SchemaBase`,
matching the Safir documentation.

Switch to the Safir method of checking for unexpected schema changes,
which is much simpler than the previous approach and doesn't require
keeping around old copies of the schema.

Point to the Safir documentation for how to make schema migrations
rather than duplicating that documentation here.

CHANGELOG.md

@@ -6,6 +6,8 @@ Dependencies are updated to the latest available version during each release. Th
 
 Find changes for the upcoming release in the project's [changelog.d directory](https://github.com/lsst-sqre/gafaelfawr/tree/main/changelog.d/).
 
+Gafaelfawr does not support direct upgrades from versions older than 10.0.0. When upgrading from an older version, first upgrade to a version of Gafaelfawr between 10.0.0 and 12.1.0, inclusive, and complete the schema migration. Then you can safely upgrade to the latest version.
+
 <!-- scriv-insert-here -->
 
 <a id='changelog-12.1.0'></a>

alembic/env.py

@@ -5,7 +5,7 @@
 from safir.logging import configure_alembic_logging
 
 from gafaelfawr.dependencies.config import config_dependency
-from gafaelfawr.schema import Base
+from gafaelfawr.schema import SchemaBase
 
 # Load the Gafaelfawr configuration, which as a side effect also configures
 # logging using structlog.
@@ -14,10 +14,10 @@
 # Run the migrations.
 configure_alembic_logging()
 if context.is_offline_mode():
-    run_migrations_offline(Base.metadata, config.database_url)
+    run_migrations_offline(SchemaBase.metadata, config.database_url)
 else:
     run_migrations_online(
-        Base.metadata,
+        SchemaBase.metadata,
         config.database_url,
         config.database_password,
     )

alembic/versions/20240209_2309_5c28ed7092c2_initial_schema.py

@@ -0,0 +1,231 @@
+"""Initial schema.
+
+Revision ID: 5c28ed7092c2
+Revises:
+Create Date: 2024-11-19 22:40:16.309715+00:00
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "5c28ed7092c2"
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "admin",
+        sa.Column("username", sa.String(length=64), nullable=False),
+        sa.PrimaryKeyConstraint("username"),
+    )
+    op.create_table(
+        "admin_history",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("username", sa.String(length=64), nullable=False),
+        sa.Column(
+            "action",
+            sa.Enum("add", "remove", name="adminchange"),
+            nullable=False,
+        ),
+        sa.Column("actor", sa.String(length=64), nullable=False),
+        sa.Column(
+            "ip_address",
+            sa.String(length=64).with_variant(postgresql.INET(), "postgresql"),
+            nullable=False,
+        ),
+        sa.Column("event_time", sa.DateTime(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "admin_history_by_time",
+        "admin_history",
+        ["event_time", "id"],
+        unique=False,
+    )
+    op.create_table(
+        "token",
+        sa.Column(
+            "token", sa.String(length=64, collation="C"), nullable=False
+        ),
+        sa.Column("username", sa.String(length=64), nullable=False),
+        sa.Column(
+            "token_type",
+            sa.Enum(
+                "session",
+                "user",
+                "notebook",
+                "internal",
+                "service",
+                name="tokentype",
+            ),
+            nullable=False,
+        ),
+        sa.Column("token_name", sa.String(length=64), nullable=True),
+        sa.Column("scopes", sa.String(length=512), nullable=False),
+        sa.Column("service", sa.String(length=64), nullable=True),
+        sa.Column("created", sa.DateTime(), nullable=False),
+        sa.Column("last_used", sa.DateTime(), nullable=True),
+        sa.Column("expires", sa.DateTime(), nullable=True),
+        sa.PrimaryKeyConstraint("token"),
+        sa.UniqueConstraint("username", "token_name"),
+    )
+    op.create_index(
+        "token_by_username", "token", ["username", "token_type"], unique=False
+    )
+    op.create_table(
+        "token_auth_history",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("token", sa.String(length=64), nullable=False),
+        sa.Column("username", sa.String(length=64), nullable=False),
+        sa.Column(
+            "token_type",
+            sa.Enum(
+                "session",
+                "user",
+                "notebook",
+                "internal",
+                "service",
+                name="tokentype",
+            ),
+            nullable=False,
+        ),
+        sa.Column("token_name", sa.String(length=64), nullable=True),
+        sa.Column("parent", sa.String(length=64), nullable=True),
+        sa.Column("scopes", sa.String(length=512), nullable=True),
+        sa.Column("service", sa.String(length=64), nullable=True),
+        sa.Column(
+            "ip_address",
+            sa.String(length=64).with_variant(postgresql.INET(), "postgresql"),
+            nullable=True,
+        ),
+        sa.Column("event_time", sa.DateTime(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "token_auth_history_by_time",
+        "token_auth_history",
+        ["event_time", "id"],
+        unique=False,
+    )
+    op.create_index(
+        "token_auth_history_by_token",
+        "token_auth_history",
+        ["token", "event_time", "id"],
+        unique=False,
+    )
+    op.create_index(
+        "token_auth_history_by_username",
+        "token_auth_history",
+        ["username", "event_time", "id"],
+        unique=False,
+    )
+    op.create_table(
+        "token_change_history",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("token", sa.String(length=64), nullable=False),
+        sa.Column("username", sa.String(length=64), nullable=False),
+        sa.Column(
+            "token_type",
+            sa.Enum(
+                "session",
+                "user",
+                "notebook",
+                "internal",
+                "service",
+                name="tokentype",
+            ),
+            nullable=False,
+        ),
+        sa.Column("token_name", sa.String(length=64), nullable=True),
+        sa.Column("parent", sa.String(length=64), nullable=True),
+        sa.Column("scopes", sa.String(length=512), nullable=False),
+        sa.Column("service", sa.String(length=64), nullable=True),
+        sa.Column("expires", sa.DateTime(), nullable=True),
+        sa.Column("actor", sa.String(length=64), nullable=True),
+        sa.Column(
+            "action",
+            sa.Enum("create", "revoke", "expire", "edit", name="tokenchange"),
+            nullable=False,
+        ),
+        sa.Column("old_token_name", sa.String(length=64), nullable=True),
+        sa.Column("old_scopes", sa.String(length=512), nullable=True),
+        sa.Column("old_expires", sa.DateTime(), nullable=True),
+        sa.Column(
+            "ip_address",
+            sa.String(length=64).with_variant(postgresql.INET(), "postgresql"),
+            nullable=True,
+        ),
+        sa.Column("event_time", sa.DateTime(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "token_change_history_by_time",
+        "token_change_history",
+        ["event_time", "id"],
+        unique=False,
+    )
+    op.create_index(
+        "token_change_history_by_token",
+        "token_change_history",
+        ["token", "event_time", "id"],
+        unique=False,
+    )
+    op.create_index(
+        "token_change_history_by_username",
+        "token_change_history",
+        ["username", "event_time", "id"],
+        unique=False,
+    )
+    op.create_table(
+        "subtoken",
+        sa.Column("child", sa.String(length=64), nullable=False),
+        sa.Column("parent", sa.String(length=64), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["child"], ["token.token"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["parent"], ["token.token"], ondelete="SET NULL"
+        ),
+        sa.PrimaryKeyConstraint("child"),
+    )
+    op.create_index("subtoken_by_parent", "subtoken", ["parent"], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("subtoken_by_parent", table_name="subtoken")
+    op.drop_table("subtoken")
+    op.drop_index(
+        "token_change_history_by_username", table_name="token_change_history"
+    )
+    op.drop_index(
+        "token_change_history_by_token", table_name="token_change_history"
+    )
+    op.drop_index(
+        "token_change_history_by_time", table_name="token_change_history"
+    )
+    op.drop_table("token_change_history")
+    op.drop_index(
+        "token_auth_history_by_username", table_name="token_auth_history"
+    )
+    op.drop_index(
+        "token_auth_history_by_token", table_name="token_auth_history"
+    )
+    op.drop_index(
+        "token_auth_history_by_time", table_name="token_auth_history"
+    )
+    op.drop_table("token_auth_history")
+    op.drop_index("token_by_username", table_name="token")
+    op.drop_table("token")
+    op.drop_index("admin_history_by_time", table_name="admin_history")
+    op.drop_table("admin_history")
+    op.drop_table("admin")
+    # ### end Alembic commands ###

alembic/versions/20240209_2310_2feb306dd1ee_add_oidc_token_type.py

@@ -1,7 +1,7 @@
 """Add oidc token type.
 
 Revision ID: 2feb306dd1ee
-Revises: d2a7f04de565
+Revises: 5c28ed7092c2
 Create Date: 2024-02-09 23:10:43.229238+00:00
 """
 
@@ -11,7 +11,7 @@
 
 # revision identifiers, used by Alembic.
 revision: str = "2feb306dd1ee"
-down_revision: str | None = None
+down_revision: str | None = "5c28ed7092c2"
 branch_labels: str | Sequence[str] | None = None
 depends_on: str | Sequence[str] | None = None
 

changelog.d/20241119_153825_rra_DM_47646.md

@@ -0,0 +1,3 @@
+### Other changes
+
+- Gafaelfawr no longer supports direct upgrades from versions older than 10.0.0. When upgrading from an older version, upgrade to 12.1.0 or earlier first and complete the database schema migration, and then upgrade to the latest version.

docs/dev/development.rst

@@ -229,69 +229,7 @@ Gafaelfawr uses Alembic_ to manage and perform database migrations.
 Alembic is invoked automatically when the Gafaelfawr server is started.
 
 Whenever the database schema changes, you will need to create an Alembic migration.
-To do this, take the following steps.
-You must have Docker running locally on your system and have the :command:`docker-compose` command installed.
-
-#. Start a PostgreSQL server into which the current database schema can be created.
-
-   .. prompt:: bash
-
-      docker-compose -f alembic/docker-compose.yaml up
-
-#. Install the *current* database schema into that PostgreSQL server.
-   This must be done with a Gafaelfawr working tree that does not contain any changes to the database schema.
-   If you have already made changes that would change the database schema, use :command:`git stash`, switch to another branch, or otherwise temporarily revert those changes before running this command.
-
-   .. prompt:: bash
-
-      tox run -e gafaelfawr -- init
-
-#. Apply the code changes that will change the database schema.
-
-#. Ask Alembic to autogenerate a database migration to the new schema.
-
-   .. prompt:: bash
-
-      tox run -e alembic -- revision --autogenerate -m "<message>"
-
-   Replace ``<message>`` with a short human-readable summary of the change, ending in a period.
-   This will create a new file in :file:`alembic/versions`.
-
-#. Edit the created file in :file:`alembic/versions` and adjust it as necessary.
-   See the `Alembic documentation <https://alembic.sqlalchemy.org/en/latest/autogenerate.html>`__ for details about what Alembic can and cannot autodetect.
-
-   One common change that Alembic cannot autodetect is changes to the valid values of enum types.
-   You will need to add Alembic code to the ``upgrade`` function of the migration such as:
-
-   .. code-block:: python
-
-      op.execute("ALTER TYPE tokentype ADD VALUE 'oidc' IF NOT EXISTS")
-
-   You may want to connect to the PostgreSQL database with the :command:`psql` command-line tool so that you can examine the schema to understand what the migration needs to do.
-   For example, you can see a description of a table with :samp:`\d {table}`, which will tell you the name of an enum type that you may need to modify.
-   To do this, run:
-
-   .. prompt:: bash
-
-      psql <uri>
-
-   where ``<uri>`` is the URI to the local PostgreSQL database, which you can find in the ``databaseUrl`` configuration parameter in :file:`alembic/gafaelfawr.yaml`.
-
-#. Stop the running PostgreSQL container.
-
-   .. prompt:: bash
-
-      docker-compose -f alembic/docker-compose.yaml down
-
-#. Generate and save the new schema:
-
-   .. prompt:: bash
-
-      tox run -e gafaelfawr -- generate-schema -o tests/data/schemas/<version>
-
-   Replace ``<version>`` with the version of the Gafaelfawr release that will contain this schema version.
-   Then update the version in :file:`tests/support/constants.py` to match that new schema version.
-   This will update the test that ensures that there are no changes to the Gafaelfawr schema definition that would affect the SQL schema.
+To do this, follow the `Safir schema migration documentation <https://safir.lsst.io/user-guide/database/schema.html#creating-database-migrations>`__.
 
 Building documentation
 ======================

src/gafaelfawr/cli.py

@@ -33,7 +33,7 @@
 from .keypair import RSAKeyPair
 from .main import create_openapi
 from .models.token import Token
-from .schema import Base
+from .schema import SchemaBase
 
 __all__ = [
     "audit",
@@ -134,7 +134,7 @@ async def delete_all_data(*, config_path: Path | None) -> None:
     engine = create_database_engine(
         config.database_url, config.database_password
     )
-    tables = (t.name for t in Base.metadata.sorted_tables)
+    tables = (t.name for t in SchemaBase.metadata.sorted_tables)
     async with Factory.standalone(config, engine) as factory:
         admin_service = factory.create_admin_service()
         async with factory.session.begin():