Merge pull request #15811 from jdavcs/dev_migrations_cleanup1

Fix revision scripts, run migrations in CI, add repair option, improve migrations utils

.github/workflows/unit-postgres.yaml

@@ -56,3 +56,6 @@ jobs:
       - name: Run migration tests
         run: ./run_tests.sh -unit test/unit/data/model/migrations/test_migrations.py
         working-directory: 'galaxy root'
+      - name: Run test migrate database
+        run: ./run_tests.sh -unit test/unit/app/test_migrate_database.py
+        working-directory: 'galaxy root'

lib/galaxy/model/__init__.py

@@ -3512,7 +3512,7 @@ class DefaultQuotaAssociation(Base, Dictifiable, RepresentById):
     id = Column(Integer, primary_key=True)
     create_time = Column(DateTime, default=now)
     update_time = Column(DateTime, default=now, onupdate=now)
-    type = Column(String(32), index=True)
+    type = Column(String(32))
     quota_id = Column(Integer, ForeignKey("quota.id"), index=True)
     quota = relationship("Quota", back_populates="default")
 

lib/galaxy/model/migrations/alembic/versions_gxy/186d4835587b_drop_job_state_history_update_time_.py

@@ -5,13 +5,15 @@
 Create Date: 2022-06-01 17:50:22.629894
 
 """
-from alembic import op
 from sqlalchemy import (
     Column,
     DateTime,
 )
 
-from galaxy.model.migrations.util import drop_column
+from galaxy.model.migrations.util import (
+    add_column,
+    drop_column,
+)
 from galaxy.model.orm.now import now
 
 # revision identifiers, used by Alembic.
@@ -29,4 +31,4 @@ def upgrade():
 
 
 def downgrade():
-    op.add_column(table_name, Column("update_time", DateTime, default=now, onupdate=now))
+    add_column(table_name, Column(column_name, DateTime, default=now, onupdate=now))

lib/galaxy/model/migrations/alembic/versions_gxy/3100452fa030_add_workflow_invocation_message_table.py

@@ -5,7 +5,6 @@
 Create Date: 2023-01-13 16:13:09.578391
 
 """
-from alembic import op
 from sqlalchemy import (
     Column,
     ForeignKey,
@@ -14,6 +13,10 @@
 )
 
 from galaxy.model.custom_types import TrimmedString
+from galaxy.model.migrations.util import (
+    create_table,
+    drop_table,
+)
 
 # revision identifiers, used by Alembic.
 revision = "3100452fa030"
@@ -27,7 +30,7 @@
 
 
 def upgrade():
-    op.create_table(
+    create_table(
         table_name,
         Column("id", Integer, primary_key=True),
         Column("reason", String(32)),
@@ -43,4 +46,4 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_table(table_name)
+    drop_table(table_name)

lib/galaxy/model/migrations/alembic/versions_gxy/3a2914d703ca_add_index_wf_r_s_s__workflow_invocation_.py

@@ -26,4 +26,4 @@ def upgrade():
 
 
 def downgrade():
-    drop_index(index_name, table_name, columns)
+    drop_index(index_name, table_name)

lib/galaxy/model/migrations/alembic/versions_gxy/518c8438a91b_add_when_expression_column.py

@@ -5,12 +5,11 @@
 Create Date: 2022-10-24 16:43:39.565871
 
 """
-import sqlalchemy as sa
-from alembic import op
+from sqlalchemy import Column
 
 from galaxy.model.custom_types import JSONType
 from galaxy.model.migrations.util import (
-    column_exists,
+    add_column,
     drop_column,
 )
 
@@ -26,8 +25,7 @@
 
 
 def upgrade():
-    if not column_exists(table_name, column_name):
-        op.add_column(table_name, sa.Column(column_name, JSONType))
+    add_column(table_name, Column(column_name, JSONType))
 
 
 def downgrade():

lib/galaxy/model/migrations/alembic/versions_gxy/59e024ceaca1_add_export_association_table.py

@@ -5,7 +5,6 @@
 Create Date: 2022-10-12 18:02:34.659770
 
 """
-from alembic import op
 from sqlalchemy import (
     Column,
     DateTime,
@@ -17,6 +16,10 @@
     TrimmedString,
     UUIDType,
 )
+from galaxy.model.migrations.util import (
+    create_table,
+    drop_table,
+)
 
 # revision identifiers, used by Alembic.
 revision = "59e024ceaca1"
@@ -29,7 +32,7 @@
 
 
 def upgrade():
-    op.create_table(
+    create_table(
         table_name,
         Column("id", Integer, primary_key=True),
         Column("task_uuid", UUIDType(), index=True, unique=True),
@@ -41,4 +44,4 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_table(table_name)
+    drop_table(table_name)

lib/galaxy/model/migrations/alembic/versions_gxy/6a67bf27e6a6_deferred_data_tables.py

@@ -5,13 +5,16 @@
 Create Date: 2022-03-14 12:17:55.313830
 
 """
-from alembic import op
 from sqlalchemy import (
     Boolean,
     Column,
 )
 
-from galaxy.model.migrations.util import drop_column
+from galaxy.model.migrations.util import (
+    add_column,
+    drop_column,
+    transaction,
+)
 
 # revision identifiers, used by Alembic.
 revision = "6a67bf27e6a6"
@@ -21,10 +24,12 @@
 
 
 def upgrade():
-    op.add_column("history_dataset_association", Column("metadata_deferred", Boolean(), default=False))
-    op.add_column("library_dataset_dataset_association", Column("metadata_deferred", Boolean(), default=False))
+    with transaction():
+        add_column("history_dataset_association", Column("metadata_deferred", Boolean(), default=False))
+        add_column("library_dataset_dataset_association", Column("metadata_deferred", Boolean(), default=False))
 
 
 def downgrade():
-    drop_column("history_dataset_association", "metadata_deferred")
-    drop_column("library_dataset_dataset_association", "metadata_deferred")
+    with transaction():
+        drop_column("history_dataset_association", "metadata_deferred")
+        drop_column("library_dataset_dataset_association", "metadata_deferred")

lib/galaxy/model/migrations/alembic/versions_gxy/9540a051226e_preferred_object_store_ids.py

@@ -5,14 +5,17 @@
 Create Date: 2022-06-10 10:38:25.212102
 
 """
-from alembic import op
 from sqlalchemy import (
     Column,
     String,
 )
 
 from galaxy.model.custom_types import JSONType
-from galaxy.model.migrations.util import drop_column
+from galaxy.model.migrations.util import (
+    add_column,
+    drop_column,
+    transaction,
+)
 
 # revision identifiers, used by Alembic.
 revision = "9540a051226e"
@@ -22,15 +25,17 @@
 
 
 def upgrade():
-    preferred_object_store_type = String(255)
-    op.add_column("galaxy_user", Column("preferred_object_store_id", preferred_object_store_type, default=None))
-    op.add_column("history", Column("preferred_object_store_id", preferred_object_store_type, default=None))
-    op.add_column("job", Column("preferred_object_store_id", preferred_object_store_type, default=None))
-    op.add_column("job", Column("object_store_id_overrides", JSONType))
+    with transaction():
+        preferred_object_store_type = String(255)
+        add_column("galaxy_user", Column("preferred_object_store_id", preferred_object_store_type, default=None))
+        add_column("history", Column("preferred_object_store_id", preferred_object_store_type, default=None))
+        add_column("job", Column("preferred_object_store_id", preferred_object_store_type, default=None))
+        add_column("job", Column("object_store_id_overrides", JSONType))
 
 
 def downgrade():
-    drop_column("galaxy_user", "preferred_object_store_id")
-    drop_column("history", "preferred_object_store_id")
-    drop_column("job", "preferred_object_store_id")
-    drop_column("job", "object_store_id_overrides")
+    with transaction():
+        drop_column("galaxy_user", "preferred_object_store_id")
+        drop_column("history", "preferred_object_store_id")
+        drop_column("job", "preferred_object_store_id")
+        drop_column("job", "object_store_id_overrides")

lib/galaxy/model/migrations/alembic/versions_gxy/b182f655505f_add_workflow_source_metadata_column.py

@@ -5,12 +5,11 @@
 Create Date: 2022-03-14 12:56:57.067748
 
 """
-from alembic import op
 from sqlalchemy import Column
 
 from galaxy.model.custom_types import JSONType
 from galaxy.model.migrations.util import (
-    column_exists,
+    add_column,
     drop_column,
 )
 
@@ -26,8 +25,7 @@
 
 
 def upgrade():
-    if not column_exists(table_name, column_name):
-        op.add_column(table_name, Column(column_name, JSONType))
+    add_column(table_name, Column(column_name, JSONType))
 
 
 def downgrade():

lib/galaxy/model/migrations/alembic/versions_gxy/c39f1de47a04_add_skipped_state_to_collection_job_.py

@@ -7,6 +7,8 @@
 """
 from alembic import op
 
+from galaxy.model.migrations.util import transaction
+
 # revision identifiers, used by Alembic.
 revision = "c39f1de47a04"
 down_revision = "3100452fa030"
@@ -88,14 +90,12 @@
 
 
 def upgrade():
-    op.execute("BEGIN")
-    op.execute(f"DROP VIEW IF EXISTS {view_name}")
-    op.execute(f"CREATE VIEW {view_name} AS {new_aggregate_query}")
-    op.execute("END")
+    with transaction():
+        op.execute(f"DROP VIEW IF EXISTS {view_name}")
+        op.execute(f"CREATE VIEW {view_name} AS {new_aggregate_query}")
 
 
 def downgrade():
-    op.execute("BEGIN")
-    op.execute(f"DROP VIEW IF EXISTS {view_name}")
-    op.execute(f"CREATE VIEW {view_name} AS {previous_aggregate_query}")
-    op.execute("END")
+    with transaction():
+        op.execute(f"DROP VIEW IF EXISTS {view_name}")
+        op.execute(f"CREATE VIEW {view_name} AS {previous_aggregate_query}")

lib/galaxy/model/migrations/alembic/versions_gxy/caa7742f7bca_add_index_wf_r_i_p__workflow_invocation_.py

@@ -27,4 +27,4 @@ def upgrade():
 
 
 def downgrade():
-    drop_index(index_name, table_name, columns)
+    drop_index(index_name, table_name)

lib/galaxy/model/migrations/alembic/versions_gxy/d0583094c8cd_add_quota_source_labels.py

@@ -5,7 +5,6 @@
 Create Date: 2022-06-09 12:24:44.329038
 
 """
-from alembic import op
 from sqlalchemy import (
     Column,
     ForeignKey,
@@ -15,8 +14,15 @@
 )
 
 from galaxy.model.migrations.util import (
-    add_unique_constraint,
+    add_column,
+    create_index,
+    create_table,
+    create_unique_constraint,
     drop_column,
+    drop_constraint,
+    drop_index,
+    drop_table,
+    transaction,
 )
 
 # revision identifiers, used by Alembic.
@@ -27,23 +33,27 @@
 
 
 def upgrade():
-    op.add_column("quota", Column("quota_source_label", String(32), default=None))
-
-    op.create_table(
-        "user_quota_source_usage",
-        Column("id", Integer, primary_key=True),
-        Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
-        Column("quota_source_label", String(32), index=True),
-        # user had an index on disk_usage - does that make any sense? -John
-        Column("disk_usage", Numeric(15, 0)),
-    )
-    add_unique_constraint("uqsu_unique_label_per_user", "user_quota_source_usage", ["user_id", "quota_source_label"])
-    op.drop_index("ix_default_quota_association_type", "default_quota_association")
-    op.create_index("ix_quota_quota_source_label", "quota", ["quota_source_label"])
+    with transaction():
+        add_column("quota", Column("quota_source_label", String(32), default=None))
+        create_table(
+            "user_quota_source_usage",
+            Column("id", Integer, primary_key=True),
+            Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
+            Column("quota_source_label", String(32), index=True),
+            # user had an index on disk_usage - does that make any sense? -John
+            Column("disk_usage", Numeric(15, 0)),
+        )
+        create_unique_constraint(
+            "uqsu_unique_label_per_user", "user_quota_source_usage", ["user_id", "quota_source_label"]
+        )
+        drop_index("ix_default_quota_association_type", "default_quota_association")
+        create_index("ix_quota_quota_source_label", "quota", ["quota_source_label"])
 
 
 def downgrade():
-    op.create_index("ix_default_quota_association_type", "default_quota_association", ["type"], unique=True)
-    op.drop_table("user_quota_source_usage")
-    op.drop_index("ix_quota_quota_source_label", "quota")
-    drop_column("quota", "quota_source_label")
+    with transaction():
+        drop_index("ix_quota_quota_source_label", "quota")
+        create_index("ix_default_quota_association_type", "default_quota_association", ["type"], unique=True)
+        drop_constraint("uqsu_unique_label_per_user", "user_quota_source_usage")
+        drop_table("user_quota_source_usage")
+        drop_column("quota", "quota_source_label")

lib/galaxy/model/migrations/alembic/versions_gxy/e0e3bb173ee6_add_column_deleted_to_api_keys.py

@@ -5,15 +5,16 @@
 Create Date: 2022-09-27 14:09:05.890227
 
 """
-from alembic import op
 from sqlalchemy import (
     Boolean,
     Column,
 )
 
 from galaxy.model.migrations.util import (
-    column_exists,
+    add_column,
     drop_column,
+    drop_index,
+    transaction,
 )
 
 # revision identifiers, used by Alembic.
@@ -29,9 +30,10 @@
 
 
 def upgrade():
-    if not column_exists(table_name, column_name):
-        op.add_column(table_name, Column(column_name, Boolean(), default=False))
+    add_column(table_name, Column(column_name, Boolean(), default=False, index=True))
 
 
 def downgrade():
-    drop_column(table_name, column_name)
+    with transaction():
+        drop_index("ix_api_keys_deleted", table_name)
+        drop_column(table_name, column_name)