Replace sqlalchemy-utilities with local implementation

lib/galaxy/config/__init__.py

@@ -29,6 +29,7 @@
 from galaxy.containers import parse_containers_config
 from galaxy.exceptions import ConfigurationError
 from galaxy.model import mapping
+from galaxy.model.database_utils import database_exists
 from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database
 from galaxy.util import (
     ExecutionTimer,
@@ -1270,7 +1271,6 @@ def _configure_signal_handlers(self, handlers):
             signal.signal(sig, handler)
 
     def _wait_for_database(self, url):
-        from sqlalchemy_utils import database_exists
         attempts = self.config.database_wait_attempts
         pause = self.config.database_wait_sleep
         for i in range(1, attempts):

lib/galaxy/dependencies/dev-requirements.txt

@@ -214,7 +214,6 @@ sphinxcontrib-jsmath==1.0.1; python_version >= "3.5"
 sphinxcontrib-qthelp==1.0.3; python_version >= "3.5"
 sphinxcontrib-serializinghtml==1.1.4; python_version >= "3.5"
 sqlalchemy-migrate==0.13.0
-sqlalchemy-utils==0.36.7
 sqlalchemy==1.3.23; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 sqlitedict==1.7.0
 sqlparse==0.4.1; python_version >= "3.5"

lib/galaxy/dependencies/pinned-requirements.txt

@@ -169,7 +169,6 @@ six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" and pytho
 social-auth-core==3.3.0
 sortedcontainers==2.3.0
 sqlalchemy-migrate==0.13.0
-sqlalchemy-utils==0.36.7
 sqlalchemy==1.3.23; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 sqlitedict==1.7.0
 sqlparse==0.4.1; python_version >= "3.5"

lib/galaxy/model/database_utils.py

@@ -0,0 +1,95 @@
+import sqlite3
+from contextlib import contextmanager
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine.url import make_url
+from sqlalchemy.sql.compiler import IdentifierPreparer
+from sqlalchemy.sql.expression import text
+
+from galaxy.exceptions import ConfigurationError
+
+
+def database_exists(db_url, database=None):
+    """Check if database exists; connect with db_url.
+
+    If database is None, use the database name from db_url.
+    """
+    dbm = DatabaseManager.make_manager(db_url, database)
+    return dbm.exists()
+
+
+def create_database(db_url, database=None, encoding='utf8', template=None):
+    """Create database; connect with db_url.
+
+    If database is None, use the database name from db_url.
+    """
+    dbm = DatabaseManager.make_manager(db_url, database)
+    dbm.create(encoding, template)
+
+
+@contextmanager
+def sqlalchemy_engine(url):
+    engine = create_engine(url)
+    try:
+        yield engine
+    finally:
+        engine.dispose()
+
+
+class DatabaseManager:
+
+    @staticmethod
+    def make_manager(db_url, database):
+        if db_url.startswith('postgres'):
+            return PosgresDatabaseManager(db_url, database)
+        elif db_url.startswith('sqlite'):
+            return SqliteDatabaseManager(db_url, database)
+        else:
+            raise ConfigurationError(f'Invalid database URL: {db_url}')
+
+    def __init__(self, db_url, database):
+        self.url = make_url(db_url)
+        self.database = database
+        if not database:
+            self._handle_no_database()
+
+
+class PosgresDatabaseManager(DatabaseManager):
+
+    def _handle_no_database(self):
+        self.database = self.url.database  # use database from db_url
+        self.url.database = 'postgres'
+
+    def exists(self):
+        with sqlalchemy_engine(self.url) as engine:
+            stmt = text('SELECT 1 FROM pg_database WHERE datname=:database')
+            stmt = stmt.bindparams(database=self.database)
+            with engine.connect() as conn:
+                return bool(conn.scalar(stmt))
+
+    def create(self, encoding, template):
+        with sqlalchemy_engine(self.url) as engine:
+            preparer = IdentifierPreparer(engine.dialect)
+            template = template or 'template1'
+            database, template = preparer.quote(self.database), preparer.quote(template)
+            stmt = f"CREATE DATABASE {database} ENCODING '{encoding}' TEMPLATE {template}"
+            with engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn:
+                conn.execute(stmt)
+
+
+class SqliteDatabaseManager(DatabaseManager):
+
+    def _handle_no_database(self):
+        self.database = self.url.database  # use database from db_url
+
+    def exists(self):
+        try:
+            sqlite3.connect(f'file:{self.url.database}?mode=ro', uri=True)
+        except sqlite3.OperationalError:
+            return False
+        else:
+            return True
+
+    def create(self, encoding, template):
+        # Don't use encoding and template for sqlite
+        sqlite3.connect(f'file:{self.url.database}', uri=True)

lib/galaxy/model/migrate/check.py

@@ -9,9 +9,9 @@
     Table
 )
 from sqlalchemy.exc import NoSuchTableError
-from sqlalchemy_utils import create_database, database_exists
 
 from galaxy.model import mapping
+from galaxy.model.database_utils import create_database, database_exists
 
 log = logging.getLogger(__name__)
 

lib/galaxy/model/migrate/versions/0166_job_state_summary_view.py

@@ -15,12 +15,13 @@ def upgrade(migrate_engine):
     print(__doc__)
     # drop first because sqlite does not support or_replace
     downgrade(migrate_engine)
-    create_view = CreateView(HistoryDatasetCollectionJobStateSummary)
+    view = HistoryDatasetCollectionJobStateSummary
+    create_view = CreateView(view.name, view.__view__)
     # print(str(create_view.compile(migrate_engine)))
     migrate_engine.execute(create_view)
 
 
 def downgrade(migrate_engine):
-    drop_view = DropView(HistoryDatasetCollectionJobStateSummary)
+    drop_view = DropView(HistoryDatasetCollectionJobStateSummary.name)
     # print(str(drop_view.compile(migrate_engine)))
     migrate_engine.execute(drop_view)

lib/galaxy/model/tool_shed_install/migrate/check.py

@@ -12,11 +12,8 @@
     Table
 )
 from sqlalchemy.exc import NoSuchTableError
-from sqlalchemy_utils import (
-    create_database,
-    database_exists,
-)
 
+from galaxy.model.database_utils import create_database, database_exists
 from galaxy.model.tool_shed_install import mapping
 
 

lib/galaxy/model/view/__init__.py

@@ -1,15 +1,13 @@
 """
 Galaxy sql view models
 """
-from sqlalchemy import Integer, MetaData
+from sqlalchemy import Integer
 from sqlalchemy.orm import mapper
 from sqlalchemy.sql import column, text
-from sqlalchemy_utils import create_view
 
+from galaxy.model.view.utils import create_view
 from .utils import View
 
-metadata = MetaData()
-
 AGGREGATE_STATE_QUERY = """
 SELECT
     hdca.id as hdca_id,
@@ -38,9 +36,11 @@
 
 
 class HistoryDatasetCollectionJobStateSummary(View):
+    name = 'collection_job_state_summary_view'
+    pkey = 'hdca_id'
 
     __view__ = text(AGGREGATE_STATE_QUERY).columns(
-        column('hdca_id', Integer),
+        column(pkey, Integer),
         column('new', Integer),
         column('resubmitted', Integer),
         column('waiting', Integer),
@@ -56,7 +56,7 @@ class HistoryDatasetCollectionJobStateSummary(View):
         column('all_jobs', Integer)
     )
 
-    __table__ = create_view('collection_job_state_summary_view', __view__, metadata)
+    __table__ = create_view(name, __view__, pkey)
 
 
 mapper(HistoryDatasetCollectionJobStateSummary, HistoryDatasetCollectionJobStateSummary.__table__)

lib/galaxy/model/view/utils.py

@@ -3,32 +3,40 @@
 """
 from inspect import getmembers
 
+from sqlalchemy import (
+    Column,
+    event,
+    MetaData,
+    Table,
+)
 from sqlalchemy.ext import compiler
-from sqlalchemy_utils import view
+from sqlalchemy.schema import DDLElement
 
 
 class View:
     is_view = True
 
 
-class DropView(view.DropView):
-    def __init__(self, ViewModel, **kwargs):
-        super().__init__(str(ViewModel.__table__.name), **kwargs)
+class CreateView(DDLElement):
+    def __init__(self, name, selectable):
+        self.name = name
+        self.selectable = selectable
 
 
-@compiler.compiles(DropView, "sqlite")
-def compile_drop_materialized_view(element, compiler, **kw):
-    # modified because sqlalchemy_utils adds a cascade for
-    # sqlite even though sqlite does not support cascade keyword
-    return 'DROP {}VIEW IF EXISTS {}'.format(
-        'MATERIALIZED ' if element.materialized else '',
-        element.name
-    )
+@compiler.compiles(CreateView)
+def compile_create_view(element, compiler, **kw):
+    compiled_selectable = compiler.sql_compiler.process(element.selectable, literal_binds=True)
+    return f'CREATE VIEW {element.name} AS {compiled_selectable}'
 
 
-class CreateView(view.CreateView):
-    def __init__(self, ViewModel, **kwargs):
-        super().__init__(str(ViewModel.__table__.name), ViewModel.__view__, **kwargs)
+class DropView(DDLElement):
+    def __init__(self, name):
+        self.name = name
+
+
+@compiler.compiles(DropView)
+def compile_drop_view(element, compiler, **kw):
+    return f'DROP VIEW IF EXISTS {element.name}'
 
 
 def is_view_model(o):
@@ -38,10 +46,29 @@ def is_view_model(o):
 def install_views(engine):
     import galaxy.model.view
     views = getmembers(galaxy.model.view, is_view_model)
-    for _name, ViewModel in views:
+    for _, view in views:
         # adding DropView here because our unit-testing calls this function when
         # it mocks the app and CreateView will attempt to rebuild an existing
         # view in a database that is already made, the right answer is probably
         # to change the sql that gest emitted when CreateView is rendered.
-        engine.execute(DropView(ViewModel))
-        engine.execute(CreateView(ViewModel))
+        engine.execute(DropView(view.name))
+        engine.execute(CreateView(view.name, view.__view__))
+
+
+def create_view(name, selectable, pkey):
+    metadata = MetaData()
+
+    columns = [
+        Column(
+            c.name,
+            c.type,
+            primary_key=(c.name == pkey)
+        )
+        for c in selectable.c
+    ]
+    table = Table(name, metadata, *columns)
+
+    event.listen(metadata, 'after_create', CreateView(name, selectable))
+    event.listen(metadata, 'before_drop', DropView(name))
+
+    return table

lib/galaxy_test/driver/driver_util.py

@@ -22,14 +22,11 @@
 import nose.plugins.manager
 import yaml
 from paste import httpserver
-from sqlalchemy_utils import (
-    create_database,
-    database_exists,
-)
 
 from galaxy.app import UniverseApplication as GalaxyUniverseApplication
 from galaxy.config import LOGGING_CONFIG_DEFAULT
 from galaxy.model import mapping
+from galaxy.model.database_utils import create_database, database_exists
 from galaxy.model.tool_shed_install import mapping as toolshed_mapping
 from galaxy.tool_util.verify.interactor import GalaxyInteractorApi, verify_tool
 from galaxy.util import asbool, download_to_file, galaxy_directory

lib/tool_shed/webapp/model/migrate/check.py

@@ -5,10 +5,8 @@
 from migrate.versioning import repository, schema
 from sqlalchemy import create_engine, MetaData, Table
 from sqlalchemy.exc import NoSuchTableError
-from sqlalchemy_utils import (
-    create_database,
-    database_exists,
-)
+
+from galaxy.model.database_utils import create_database, database_exists
 
 log = logging.getLogger(__name__)
 

pyproject.toml

@@ -68,7 +68,6 @@ social-auth-core = {version = "==3.3.0", extras = ["openidconnect"]}
 sortedcontainers = "*"
 SQLAlchemy = ">=1.3.22, <1.4.0"  # https://github.com/kvesteri/sqlalchemy-utils/issues/474
 sqlalchemy-migrate = "*"
-SQLAlchemy-Utils = "!=0.36.8"  # https://github.com/kvesteri/sqlalchemy-utils/issues/462
 sqlitedict = "*"
 sqlparse = "*"
 starlette = "*"

test/unit/data/test_galaxy_mapping.py

@@ -5,11 +5,11 @@
 
 import pytest
 from sqlalchemy import inspect
-from sqlalchemy_utils import create_database
 
 import galaxy.datatypes.registry
 import galaxy.model
 import galaxy.model.mapping as mapping
+from galaxy.model.database_utils import create_database
 from galaxy.model.security import GalaxyRBACAgent
 
 datatypes_registry = galaxy.datatypes.registry.Registry()