Merge pull request #11696 from ic4f/dev_no_sqlalchemy_utils

Replace sqlalchemy-utilities with local implementation

lib/galaxy/config/__init__.py

@@ -29,6 +29,7 @@
 from galaxy.containers import parse_containers_config
 from galaxy.exceptions import ConfigurationError
 from galaxy.model import mapping
+from galaxy.model.database_utils import database_exists
 from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database
 from galaxy.util import (
     ExecutionTimer,
@@ -1270,7 +1271,6 @@ def _configure_signal_handlers(self, handlers):
             signal.signal(sig, handler)
 
     def _wait_for_database(self, url):
-        from sqlalchemy_utils import database_exists
         attempts = self.config.database_wait_attempts
         pause = self.config.database_wait_sleep
         for i in range(1, attempts):

lib/galaxy/dependencies/dev-requirements.txt

@@ -214,7 +214,6 @@ sphinxcontrib-jsmath==1.0.1; python_version >= "3.5"
 sphinxcontrib-qthelp==1.0.3; python_version >= "3.5"
 sphinxcontrib-serializinghtml==1.1.4; python_version >= "3.5"
 sqlalchemy-migrate==0.13.0
-sqlalchemy-utils==0.36.7
 sqlalchemy==1.3.23; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 sqlitedict==1.7.0
 sqlparse==0.4.1; python_version >= "3.5"

lib/galaxy/dependencies/pinned-requirements.txt

@@ -169,7 +169,6 @@ six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" and pytho
 social-auth-core==3.3.0
 sortedcontainers==2.3.0
 sqlalchemy-migrate==0.13.0
-sqlalchemy-utils==0.36.7
 sqlalchemy==1.3.23; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
 sqlitedict==1.7.0
 sqlparse==0.4.1; python_version >= "3.5"

lib/galaxy/model/database_utils.py

@@ -0,0 +1,125 @@
+import sqlite3
+from contextlib import contextmanager
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine.url import make_url
+from sqlalchemy.sql.compiler import IdentifierPreparer
+from sqlalchemy.sql.expression import text
+
+from galaxy.exceptions import ConfigurationError
+
+
+def database_exists(db_url, database=None):
+    """Check if database exists; connect with db_url.
+
+    If database is None, use the database name from db_url.
+    """
+    dbm = DatabaseManager.make_manager(db_url, database)
+    return dbm.exists()
+
+
+def create_database(db_url, database=None, encoding='utf8', template=None):
+    """Create database; connect with db_url.
+
+    If database is None, use the database name from db_url.
+    """
+    dbm = DatabaseManager.make_manager(db_url, database)
+    dbm.create(encoding, template)
+
+
+@contextmanager
+def sqlalchemy_engine(url):
+    engine = create_engine(url)
+    try:
+        yield engine
+    finally:
+        engine.dispose()
+
+
+class DatabaseManager:
+
+    @staticmethod
+    def make_manager(db_url, database):
+        if db_url.startswith('postgres'):
+            return PosgresDatabaseManager(db_url, database)
+        elif db_url.startswith('sqlite'):
+            return SqliteDatabaseManager(db_url, database)
+        elif db_url.startswith('mysql'):
+            return MySQLDatabaseManager(db_url, database)
+        else:
+            raise ConfigurationError(f'Invalid database URL: {db_url}')
+
+    def __init__(self, db_url, database):
+        self.url = make_url(db_url)
+        self.database = database
+        if not database:
+            self._handle_no_database()
+
+
+class PosgresDatabaseManager(DatabaseManager):
+
+    def _handle_no_database(self):
+        self.database = self.url.database  # use database from db_url
+        self.url.database = 'postgres'
+
+    def exists(self):
+        with sqlalchemy_engine(self.url) as engine:
+            stmt = text('SELECT 1 FROM pg_database WHERE datname=:database')
+            stmt = stmt.bindparams(database=self.database)
+            with engine.connect() as conn:
+                return bool(conn.scalar(stmt))
+
+    def create(self, encoding, template):
+        with sqlalchemy_engine(self.url) as engine:
+            preparer = IdentifierPreparer(engine.dialect)
+            template = template or 'template1'
+            database, template = preparer.quote(self.database), preparer.quote(template)
+            stmt = f"CREATE DATABASE {database} ENCODING '{encoding}' TEMPLATE {template}"
+            with engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn:
+                conn.execute(stmt)
+
+
+class SqliteDatabaseManager(DatabaseManager):
+
+    def _handle_no_database(self):
+        self.database = self.url.database  # use database from db_url
+
+    def exists(self):
+
+        def can_connect_to_dbfile():
+            try:
+                sqlite3.connect(f'file:{db}?mode=ro', uri=True)
+            except sqlite3.OperationalError:
+                return False
+            else:
+                return True
+
+        db = self.url.database
+        # No database or ':memory:' creates an in-memory database
+        return not db or db == ':memory:' or can_connect_to_dbfile()
+
+    def create(self, *args):
+        # Ignore any args (encoding, template)
+        sqlite3.connect(f'file:{self.url.database}', uri=True)
+
+
+class MySQLDatabaseManager(DatabaseManager):
+
+    def _handle_no_database(self):
+        self.database = self.url.database  # use database from db_url
+
+    def exists(self):
+        with sqlalchemy_engine(self.url) as engine:
+            stmt = text("SELECT schema_name FROM information_schema.schemata WHERE schema_name=:database")
+            stmt = stmt.bindparams(database=self.database)
+            with engine.connect() as conn:
+                return bool(conn.scalar(stmt))
+
+    def create(self, encoding, *arg):
+        # Ignore any args (template)
+        with sqlalchemy_engine(self.url) as engine:
+            preparer = IdentifierPreparer(engine.dialect)
+            database = preparer.quote(self.database)
+            stmt = f"CREATE DATABASE {database} CHARACTER SET = '{encoding}'"
+            with engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn:
+                conn.execute(stmt)

lib/galaxy/model/migrate/check.py

@@ -9,9 +9,9 @@
     Table
 )
 from sqlalchemy.exc import NoSuchTableError
-from sqlalchemy_utils import create_database, database_exists
 
 from galaxy.model import mapping
+from galaxy.model.database_utils import create_database, database_exists
 
 log = logging.getLogger(__name__)
 

lib/galaxy/model/migrate/versions/0166_job_state_summary_view.py

@@ -15,12 +15,13 @@ def upgrade(migrate_engine):
     print(__doc__)
     # drop first because sqlite does not support or_replace
     downgrade(migrate_engine)
-    create_view = CreateView(HistoryDatasetCollectionJobStateSummary)
+    view = HistoryDatasetCollectionJobStateSummary
+    create_view = CreateView(view.name, view.__view__)
     # print(str(create_view.compile(migrate_engine)))
     migrate_engine.execute(create_view)
 
 
 def downgrade(migrate_engine):
-    drop_view = DropView(HistoryDatasetCollectionJobStateSummary)
+    drop_view = DropView(HistoryDatasetCollectionJobStateSummary.name)
     # print(str(drop_view.compile(migrate_engine)))
     migrate_engine.execute(drop_view)

lib/galaxy/model/tool_shed_install/migrate/check.py

@@ -12,11 +12,8 @@
     Table
 )
 from sqlalchemy.exc import NoSuchTableError
-from sqlalchemy_utils import (
-    create_database,
-    database_exists,
-)
 
+from galaxy.model.database_utils import create_database, database_exists
 from galaxy.model.tool_shed_install import mapping
 
 

lib/galaxy/model/view/__init__.py

@@ -1,14 +1,11 @@
 """
 Galaxy sql view models
 """
-from sqlalchemy import Integer, MetaData
+from sqlalchemy import Integer
 from sqlalchemy.orm import mapper
 from sqlalchemy.sql import column, text
-from sqlalchemy_utils import create_view
 
-from .utils import View
-
-metadata = MetaData()
+from galaxy.model.view.utils import View
 
 AGGREGATE_STATE_QUERY = """
 SELECT
@@ -38,6 +35,7 @@
 
 
 class HistoryDatasetCollectionJobStateSummary(View):
+    name = 'collection_job_state_summary_view'
 
     __view__ = text(AGGREGATE_STATE_QUERY).columns(
         column('hdca_id', Integer),
@@ -55,8 +53,8 @@ class HistoryDatasetCollectionJobStateSummary(View):
         column('upload', Integer),
         column('all_jobs', Integer)
     )
-
-    __table__ = create_view('collection_job_state_summary_view', __view__, metadata)
+    pkeys = {'hdca_id'}
+    __table__ = View._make_table(name, __view__, pkeys)
 
 
 mapper(HistoryDatasetCollectionJobStateSummary, HistoryDatasetCollectionJobStateSummary.__table__)

lib/galaxy/model/view/utils.py

@@ -1,34 +1,67 @@
 """
-View wrappers, currently using sqlalchemy_views
+View wrappers
 """
 from inspect import getmembers
 
+from sqlalchemy import (
+    Column,
+    MetaData,
+    Table,
+)
 from sqlalchemy.ext import compiler
-from sqlalchemy_utils import view
+from sqlalchemy.schema import DDLElement
 
 
 class View:
-    is_view = True
+    """Base class for Views."""
 
+    @staticmethod
+    def _make_table(name, selectable, pkeys):
+        """ Create a view.
 
-class DropView(view.DropView):
-    def __init__(self, ViewModel, **kwargs):
-        super().__init__(str(ViewModel.__table__.name), **kwargs)
+        :param name: The name of the view.
+        :param selectable: SQLAlchemy selectable.
+        :param pkeys: set of primary keys for the selectable.
+        """
+        columns = [
+            Column(
+                c.name,
+                c.type,
+                primary_key=(c.name in pkeys)
+            )
+            for c in selectable.c
+        ]
+        # We do not use the metadata object from model.mapping.py that contains all the Table objects
+        # because that would create a circular import (create_view is called from View objects
+        # in model.view; but those View objects are imported into model.mapping.py where the
+        # metadata object we need is defined). Thus, we do not use the after_create/before_drop
+        # hooks to automate creating/dropping views.  Instead, this is taken care of in install_views().
 
+        # The metadata object passed to Table() should be empty: this table is internal to a View
+        # object and is not intended to be created in the database.
+        return Table(name, MetaData(), *columns)
 
-@compiler.compiles(DropView, "sqlite")
-def compile_drop_materialized_view(element, compiler, **kw):
-    # modified because sqlalchemy_utils adds a cascade for
-    # sqlite even though sqlite does not support cascade keyword
-    return 'DROP {}VIEW IF EXISTS {}'.format(
-        'MATERIALIZED ' if element.materialized else '',
-        element.name
-    )
 
+class CreateView(DDLElement):
+    def __init__(self, name, selectable):
+        self.name = name
+        self.selectable = selectable
 
-class CreateView(view.CreateView):
-    def __init__(self, ViewModel, **kwargs):
-        super().__init__(str(ViewModel.__table__.name), ViewModel.__view__, **kwargs)
+
+class DropView(DDLElement):
+    def __init__(self, name):
+        self.name = name
+
+
+@compiler.compiles(CreateView)
+def compile_create_view(element, compiler, **kw):
+    compiled_selectable = compiler.sql_compiler.process(element.selectable, literal_binds=True)
+    return f'CREATE VIEW {element.name} AS {compiled_selectable}'
+
+
+@compiler.compiles(DropView)
+def compile_drop_view(element, compiler, **kw):
+    return f'DROP VIEW IF EXISTS {element.name}'
 
 
 def is_view_model(o):
@@ -38,10 +71,10 @@ def is_view_model(o):
 def install_views(engine):
     import galaxy.model.view
     views = getmembers(galaxy.model.view, is_view_model)
-    for _name, ViewModel in views:
+    for _, view in views:
         # adding DropView here because our unit-testing calls this function when
         # it mocks the app and CreateView will attempt to rebuild an existing
         # view in a database that is already made, the right answer is probably
         # to change the sql that gest emitted when CreateView is rendered.
-        engine.execute(DropView(ViewModel))
-        engine.execute(CreateView(ViewModel))
+        engine.execute(DropView(view.name))
+        engine.execute(CreateView(view.name, view.__view__))

lib/galaxy_test/driver/driver_util.py

@@ -22,14 +22,11 @@
 import nose.plugins.manager
 import yaml
 from paste import httpserver
-from sqlalchemy_utils import (
-    create_database,
-    database_exists,
-)
 
 from galaxy.app import UniverseApplication as GalaxyUniverseApplication
 from galaxy.config import LOGGING_CONFIG_DEFAULT
 from galaxy.model import mapping
+from galaxy.model.database_utils import create_database, database_exists
 from galaxy.model.tool_shed_install import mapping as toolshed_mapping
 from galaxy.tool_util.verify.interactor import GalaxyInteractorApi, verify_tool
 from galaxy.util import asbool, download_to_file, galaxy_directory

lib/tool_shed/webapp/model/migrate/check.py

@@ -5,10 +5,8 @@
 from migrate.versioning import repository, schema
 from sqlalchemy import create_engine, MetaData, Table
 from sqlalchemy.exc import NoSuchTableError
-from sqlalchemy_utils import (
-    create_database,
-    database_exists,
-)
+
+from galaxy.model.database_utils import create_database, database_exists
 
 log = logging.getLogger(__name__)
 

pyproject.toml

@@ -68,7 +68,6 @@ social-auth-core = {version = "==3.3.0", extras = ["openidconnect"]}
 sortedcontainers = "*"
 SQLAlchemy = ">=1.3.22, <1.4.0"  # https://github.com/kvesteri/sqlalchemy-utils/issues/474
 sqlalchemy-migrate = "*"
-SQLAlchemy-Utils = "!=0.36.8"  # https://github.com/kvesteri/sqlalchemy-utils/issues/462
 sqlitedict = "*"
 sqlparse = "*"
 starlette = "*"

test/unit/data/test_galaxy_mapping.py

@@ -5,11 +5,11 @@
 
 import pytest
 from sqlalchemy import inspect
-from sqlalchemy_utils import create_database
 
 import galaxy.datatypes.registry
 import galaxy.model
 import galaxy.model.mapping as mapping
+from galaxy.model.database_utils import create_database
 from galaxy.model.security import GalaxyRBACAgent
 
 datatypes_registry = galaxy.datatypes.registry.Registry()