diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 584cd993d97b..7a5be5e45682 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -29,6 +29,7 @@ from galaxy.containers import parse_containers_config from galaxy.exceptions import ConfigurationError from galaxy.model import mapping +from galaxy.model.database_utils import database_exists from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database from galaxy.util import ( ExecutionTimer, @@ -1270,7 +1271,6 @@ def _configure_signal_handlers(self, handlers): signal.signal(sig, handler) def _wait_for_database(self, url): - from sqlalchemy_utils import database_exists attempts = self.config.database_wait_attempts pause = self.config.database_wait_sleep for i in range(1, attempts): diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 80dac09686e8..9eb15390fe58 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -214,7 +214,6 @@ sphinxcontrib-jsmath==1.0.1; python_version >= "3.5" sphinxcontrib-qthelp==1.0.3; python_version >= "3.5" sphinxcontrib-serializinghtml==1.1.4; python_version >= "3.5" sqlalchemy-migrate==0.13.0 -sqlalchemy-utils==0.36.7 sqlalchemy==1.3.23; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") sqlitedict==1.7.0 sqlparse==0.4.1; python_version >= "3.5" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 3181af293ad7..cfc179cf595d 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -169,7 +169,6 @@ six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" and pytho social-auth-core==3.3.0 sortedcontainers==2.3.0 sqlalchemy-migrate==0.13.0 -sqlalchemy-utils==0.36.7 sqlalchemy==1.3.23; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") sqlitedict==1.7.0 sqlparse==0.4.1; python_version >= "3.5" diff --git a/lib/galaxy/model/database_utils.py b/lib/galaxy/model/database_utils.py new file mode 100644 index 000000000000..5a949f0a0a3f --- /dev/null +++ b/lib/galaxy/model/database_utils.py @@ -0,0 +1,125 @@ +import sqlite3 +from contextlib import contextmanager + +from sqlalchemy import create_engine +from sqlalchemy.engine.url import make_url +from sqlalchemy.sql.compiler import IdentifierPreparer +from sqlalchemy.sql.expression import text + +from galaxy.exceptions import ConfigurationError + + +def database_exists(db_url, database=None): + """Check if database exists; connect with db_url. + + If database is None, use the database name from db_url. + """ + dbm = DatabaseManager.make_manager(db_url, database) + return dbm.exists() + + +def create_database(db_url, database=None, encoding='utf8', template=None): + """Create database; connect with db_url. + + If database is None, use the database name from db_url. + """ + dbm = DatabaseManager.make_manager(db_url, database) + dbm.create(encoding, template) + + +@contextmanager +def sqlalchemy_engine(url): + engine = create_engine(url) + try: + yield engine + finally: + engine.dispose() + + +class DatabaseManager: + + @staticmethod + def make_manager(db_url, database): + if db_url.startswith('postgres'): + return PosgresDatabaseManager(db_url, database) + elif db_url.startswith('sqlite'): + return SqliteDatabaseManager(db_url, database) + elif db_url.startswith('mysql'): + return MySQLDatabaseManager(db_url, database) + else: + raise ConfigurationError(f'Invalid database URL: {db_url}') + + def __init__(self, db_url, database): + self.url = make_url(db_url) + self.database = database + if not database: + self._handle_no_database() + + +class PosgresDatabaseManager(DatabaseManager): + + def _handle_no_database(self): + self.database = self.url.database # use database from db_url + self.url.database = 'postgres' + + def exists(self): + with sqlalchemy_engine(self.url) as engine: + stmt = text('SELECT 1 FROM pg_database WHERE datname=:database') + stmt = stmt.bindparams(database=self.database) + with engine.connect() as conn: + return bool(conn.scalar(stmt)) + + def create(self, encoding, template): + with sqlalchemy_engine(self.url) as engine: + preparer = IdentifierPreparer(engine.dialect) + template = template or 'template1' + database, template = preparer.quote(self.database), preparer.quote(template) + stmt = f"CREATE DATABASE {database} ENCODING '{encoding}' TEMPLATE {template}" + with engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn: + conn.execute(stmt) + + +class SqliteDatabaseManager(DatabaseManager): + + def _handle_no_database(self): + self.database = self.url.database # use database from db_url + + def exists(self): + + def can_connect_to_dbfile(): + try: + sqlite3.connect(f'file:{db}?mode=ro', uri=True) + except sqlite3.OperationalError: + return False + else: + return True + + db = self.url.database + # No database or ':memory:' creates an in-memory database + return not db or db == ':memory:' or can_connect_to_dbfile() + + def create(self, *args): + # Ignore any args (encoding, template) + sqlite3.connect(f'file:{self.url.database}', uri=True) + + +class MySQLDatabaseManager(DatabaseManager): + + def _handle_no_database(self): + self.database = self.url.database # use database from db_url + + def exists(self): + with sqlalchemy_engine(self.url) as engine: + stmt = text("SELECT schema_name FROM information_schema.schemata WHERE schema_name=:database") + stmt = stmt.bindparams(database=self.database) + with engine.connect() as conn: + return bool(conn.scalar(stmt)) + + def create(self, encoding, *arg): + # Ignore any args (template) + with sqlalchemy_engine(self.url) as engine: + preparer = IdentifierPreparer(engine.dialect) + database = preparer.quote(self.database) + stmt = f"CREATE DATABASE {database} CHARACTER SET = '{encoding}'" + with engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn: + conn.execute(stmt) diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index fc81b7780b82..f864ad60c726 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -9,9 +9,9 @@ Table ) from sqlalchemy.exc import NoSuchTableError -from sqlalchemy_utils import create_database, database_exists from galaxy.model import mapping +from galaxy.model.database_utils import create_database, database_exists log = logging.getLogger(__name__) diff --git a/lib/galaxy/model/migrate/versions/0166_job_state_summary_view.py b/lib/galaxy/model/migrate/versions/0166_job_state_summary_view.py index 74d5b924babc..062224e7abd9 100644 --- a/lib/galaxy/model/migrate/versions/0166_job_state_summary_view.py +++ b/lib/galaxy/model/migrate/versions/0166_job_state_summary_view.py @@ -15,12 +15,13 @@ def upgrade(migrate_engine): print(__doc__) # drop first because sqlite does not support or_replace downgrade(migrate_engine) - create_view = CreateView(HistoryDatasetCollectionJobStateSummary) + view = HistoryDatasetCollectionJobStateSummary + create_view = CreateView(view.name, view.__view__) # print(str(create_view.compile(migrate_engine))) migrate_engine.execute(create_view) def downgrade(migrate_engine): - drop_view = DropView(HistoryDatasetCollectionJobStateSummary) + drop_view = DropView(HistoryDatasetCollectionJobStateSummary.name) # print(str(drop_view.compile(migrate_engine))) migrate_engine.execute(drop_view) diff --git a/lib/galaxy/model/tool_shed_install/migrate/check.py b/lib/galaxy/model/tool_shed_install/migrate/check.py index 92176c541f5c..0e1a66248fc4 100644 --- a/lib/galaxy/model/tool_shed_install/migrate/check.py +++ b/lib/galaxy/model/tool_shed_install/migrate/check.py @@ -12,11 +12,8 @@ Table ) from sqlalchemy.exc import NoSuchTableError -from sqlalchemy_utils import ( - create_database, - database_exists, -) +from galaxy.model.database_utils import create_database, database_exists from galaxy.model.tool_shed_install import mapping diff --git a/lib/galaxy/model/view/__init__.py b/lib/galaxy/model/view/__init__.py index 6546fce74b42..ae1958da9fbd 100644 --- a/lib/galaxy/model/view/__init__.py +++ b/lib/galaxy/model/view/__init__.py @@ -1,14 +1,11 @@ """ Galaxy sql view models """ -from sqlalchemy import Integer, MetaData +from sqlalchemy import Integer from sqlalchemy.orm import mapper from sqlalchemy.sql import column, text -from sqlalchemy_utils import create_view -from .utils import View - -metadata = MetaData() +from galaxy.model.view.utils import View AGGREGATE_STATE_QUERY = """ SELECT @@ -38,6 +35,7 @@ class HistoryDatasetCollectionJobStateSummary(View): + name = 'collection_job_state_summary_view' __view__ = text(AGGREGATE_STATE_QUERY).columns( column('hdca_id', Integer), @@ -55,8 +53,8 @@ class HistoryDatasetCollectionJobStateSummary(View): column('upload', Integer), column('all_jobs', Integer) ) - - __table__ = create_view('collection_job_state_summary_view', __view__, metadata) + pkeys = {'hdca_id'} + __table__ = View._make_table(name, __view__, pkeys) mapper(HistoryDatasetCollectionJobStateSummary, HistoryDatasetCollectionJobStateSummary.__table__) diff --git a/lib/galaxy/model/view/utils.py b/lib/galaxy/model/view/utils.py index cbc6f55fabbe..075be7bba083 100644 --- a/lib/galaxy/model/view/utils.py +++ b/lib/galaxy/model/view/utils.py @@ -1,34 +1,67 @@ """ -View wrappers, currently using sqlalchemy_views +View wrappers """ from inspect import getmembers +from sqlalchemy import ( + Column, + MetaData, + Table, +) from sqlalchemy.ext import compiler -from sqlalchemy_utils import view +from sqlalchemy.schema import DDLElement class View: - is_view = True + """Base class for Views.""" + @staticmethod + def _make_table(name, selectable, pkeys): + """ Create a view. -class DropView(view.DropView): - def __init__(self, ViewModel, **kwargs): - super().__init__(str(ViewModel.__table__.name), **kwargs) + :param name: The name of the view. + :param selectable: SQLAlchemy selectable. + :param pkeys: set of primary keys for the selectable. + """ + columns = [ + Column( + c.name, + c.type, + primary_key=(c.name in pkeys) + ) + for c in selectable.c + ] + # We do not use the metadata object from model.mapping.py that contains all the Table objects + # because that would create a circular import (create_view is called from View objects + # in model.view; but those View objects are imported into model.mapping.py where the + # metadata object we need is defined). Thus, we do not use the after_create/before_drop + # hooks to automate creating/dropping views. Instead, this is taken care of in install_views(). + # The metadata object passed to Table() should be empty: this table is internal to a View + # object and is not intended to be created in the database. + return Table(name, MetaData(), *columns) -@compiler.compiles(DropView, "sqlite") -def compile_drop_materialized_view(element, compiler, **kw): - # modified because sqlalchemy_utils adds a cascade for - # sqlite even though sqlite does not support cascade keyword - return 'DROP {}VIEW IF EXISTS {}'.format( - 'MATERIALIZED ' if element.materialized else '', - element.name - ) +class CreateView(DDLElement): + def __init__(self, name, selectable): + self.name = name + self.selectable = selectable -class CreateView(view.CreateView): - def __init__(self, ViewModel, **kwargs): - super().__init__(str(ViewModel.__table__.name), ViewModel.__view__, **kwargs) + +class DropView(DDLElement): + def __init__(self, name): + self.name = name + + +@compiler.compiles(CreateView) +def compile_create_view(element, compiler, **kw): + compiled_selectable = compiler.sql_compiler.process(element.selectable, literal_binds=True) + return f'CREATE VIEW {element.name} AS {compiled_selectable}' + + +@compiler.compiles(DropView) +def compile_drop_view(element, compiler, **kw): + return f'DROP VIEW IF EXISTS {element.name}' def is_view_model(o): @@ -38,10 +71,10 @@ def is_view_model(o): def install_views(engine): import galaxy.model.view views = getmembers(galaxy.model.view, is_view_model) - for _name, ViewModel in views: + for _, view in views: # adding DropView here because our unit-testing calls this function when # it mocks the app and CreateView will attempt to rebuild an existing # view in a database that is already made, the right answer is probably # to change the sql that gest emitted when CreateView is rendered. - engine.execute(DropView(ViewModel)) - engine.execute(CreateView(ViewModel)) + engine.execute(DropView(view.name)) + engine.execute(CreateView(view.name, view.__view__)) diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index 8f8681fb8cea..7f6108f56f7e 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -22,14 +22,11 @@ import nose.plugins.manager import yaml from paste import httpserver -from sqlalchemy_utils import ( - create_database, - database_exists, -) from galaxy.app import UniverseApplication as GalaxyUniverseApplication from galaxy.config import LOGGING_CONFIG_DEFAULT from galaxy.model import mapping +from galaxy.model.database_utils import create_database, database_exists from galaxy.model.tool_shed_install import mapping as toolshed_mapping from galaxy.tool_util.verify.interactor import GalaxyInteractorApi, verify_tool from galaxy.util import asbool, download_to_file, galaxy_directory diff --git a/lib/tool_shed/webapp/model/migrate/check.py b/lib/tool_shed/webapp/model/migrate/check.py index 85bd5998920d..2e37c4840b1b 100644 --- a/lib/tool_shed/webapp/model/migrate/check.py +++ b/lib/tool_shed/webapp/model/migrate/check.py @@ -5,10 +5,8 @@ from migrate.versioning import repository, schema from sqlalchemy import create_engine, MetaData, Table from sqlalchemy.exc import NoSuchTableError -from sqlalchemy_utils import ( - create_database, - database_exists, -) + +from galaxy.model.database_utils import create_database, database_exists log = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index 86b00a7b277e..9c2a45cc9c69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,6 @@ social-auth-core = {version = "==3.3.0", extras = ["openidconnect"]} sortedcontainers = "*" SQLAlchemy = ">=1.3.22, <1.4.0" # https://github.com/kvesteri/sqlalchemy-utils/issues/474 sqlalchemy-migrate = "*" -SQLAlchemy-Utils = "!=0.36.8" # https://github.com/kvesteri/sqlalchemy-utils/issues/462 sqlitedict = "*" sqlparse = "*" starlette = "*" diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py index 2db5e6fe9d64..f23808c15ae4 100644 --- a/test/unit/data/test_galaxy_mapping.py +++ b/test/unit/data/test_galaxy_mapping.py @@ -5,11 +5,11 @@ import pytest from sqlalchemy import inspect -from sqlalchemy_utils import create_database import galaxy.datatypes.registry import galaxy.model import galaxy.model.mapping as mapping +from galaxy.model.database_utils import create_database from galaxy.model.security import GalaxyRBACAgent datatypes_registry = galaxy.datatypes.registry.Registry() diff --git a/test/unit/model/__init__.py b/test/unit/model/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/unit/model/common.py b/test/unit/model/common.py new file mode 100644 index 000000000000..3f9386b5fe8d --- /dev/null +++ b/test/unit/model/common.py @@ -0,0 +1,47 @@ +import os + +import pytest +from sqlalchemy.engine.url import make_url +from sqlalchemy.sql.compiler import IdentifierPreparer + +from galaxy.model.database_utils import sqlalchemy_engine + +# GALAXY_TEST_CONNECT_POSTGRES_URI='postgresql://postgres@localhost:5432/postgres' pytest test/unit/model +skip_if_not_postgres_uri = pytest.mark.skipif( + not os.environ.get('GALAXY_TEST_CONNECT_POSTGRES_URI'), + reason="GALAXY_TEST_CONNECT_POSTGRES_URI not set" +) + +# GALAXY_TEST_CONNECT_MYSQL_URI='mysql+mysqldb://root@localhost/mysql' pytest test/unit/model +skip_if_not_mysql_uri = pytest.mark.skipif( + not os.environ.get('GALAXY_TEST_CONNECT_MYSQL_URI'), + reason="GALAXY_TEST_CONNECT_MYSQL_URI not set" +) + + +def replace_database_in_url(url, database_name): + """ + Substitute the database part of url for database_name. + + Example: replace_database_in_url('foo/db1', 'db2') returns 'foo/db2' + This will not work for unix domain connections. + """ + i = url.rfind('/') + return f'{url[:i]}/{database_name}' + + +def drop_database(db_url, database): + """Drop database; connect with db_url. + + Used only for test purposes to cleanup after creating a test database. + """ + if db_url.startswith('postgresql') or db_url.startswith('mysql'): + with sqlalchemy_engine(db_url) as engine: + preparer = IdentifierPreparer(engine.dialect) + database = preparer.quote(database) + stmt = f'DROP DATABASE IF EXISTS {database}' + with engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn: + conn.execute(stmt) + else: + url = make_url(db_url) + os.remove(url.database) diff --git a/test/unit/model/conftest.py b/test/unit/model/conftest.py new file mode 100644 index 000000000000..02f5252c7b17 --- /dev/null +++ b/test/unit/model/conftest.py @@ -0,0 +1,24 @@ +import os +import uuid + +import pytest + + +@pytest.fixture +def database_name(): + return f'galaxytest_{uuid.uuid4().hex}' + + +@pytest.fixture +def postgres_url(): + return os.environ.get('GALAXY_TEST_CONNECT_POSTGRES_URI') + + +@pytest.fixture +def mysql_url(): + return os.environ.get('GALAXY_TEST_CONNECT_MYSQL_URI') + + +@pytest.fixture +def sqlite_memory_url(): + return 'sqlite:///:memory:' diff --git a/test/unit/model/test_database_utils.py b/test/unit/model/test_database_utils.py new file mode 100644 index 000000000000..72cbf17c860c --- /dev/null +++ b/test/unit/model/test_database_utils.py @@ -0,0 +1,75 @@ +import os +import tempfile + +from galaxy.model.database_utils import ( + create_database, + database_exists, +) +from .common import ( + drop_database, + replace_database_in_url, + skip_if_not_mysql_uri, + skip_if_not_postgres_uri, +) + + +@skip_if_not_postgres_uri +def test_create_exists_postgres_database(database_name, postgres_url): + assert not database_exists(postgres_url, database_name) + create_database(postgres_url, database_name) + assert database_exists(postgres_url, database_name) + drop_database(postgres_url, database_name) + assert not database_exists(postgres_url, database_name) + + +@skip_if_not_postgres_uri +def test_create_exists_postgres_database__pass_as_url(database_name, postgres_url): + # the database in the url is the one to create/check + url = replace_database_in_url(postgres_url, database_name) + + assert not database_exists(url) + create_database(url) + assert database_exists(url) + drop_database(postgres_url, database_name) + assert not database_exists(url) + + +def test_create_exists_sqlite_database(database_name): + with tempfile.TemporaryDirectory() as tmp_dir: + url = make_sqlite_url(tmp_dir, database_name) + + assert not database_exists(url, database_name) + create_database(url, database_name) + assert database_exists(url, database_name) + drop_database(url, database_name) + assert not database_exists(url, database_name) + + +def test_create_exists_sqlite_database__pass_as_url(database_name): + # the database in the url is the one to create/check + with tempfile.TemporaryDirectory() as tmp_dir: + url = make_sqlite_url(tmp_dir, database_name) + + assert not database_exists(url) + create_database(url) + assert database_exists(url) + drop_database(url, database_name) + assert not database_exists(url) + + +def test_exists_sqlite_in_memory_database(database_name, sqlite_memory_url): + assert database_exists(sqlite_memory_url) + + +@skip_if_not_mysql_uri +def test_create_exists_mysql_database(database_name, mysql_url): + assert not database_exists(mysql_url, database_name) + create_database(mysql_url, database_name) + assert database_exists(mysql_url, database_name) + drop_database(mysql_url, database_name) + assert not database_exists(mysql_url, database_name) + + +def make_sqlite_url(tmp_dir, database_name): + path = os.path.join(tmp_dir, database_name) + return f'sqlite:///{path}' diff --git a/test/unit/model/test_views.py b/test/unit/model/test_views.py new file mode 100644 index 000000000000..990b0c6c6dbe --- /dev/null +++ b/test/unit/model/test_views.py @@ -0,0 +1,89 @@ +import pytest +from sqlalchemy import ( + Column, + Integer, + MetaData, + Table, +) +from sqlalchemy.sql import ( + column, + text, +) + +from galaxy.model.database_utils import ( + create_database, + sqlalchemy_engine, +) +from galaxy.model.view.utils import ( + CreateView, + View, +) +from .common import ( + drop_database, + replace_database_in_url, + skip_if_not_mysql_uri, + skip_if_not_postgres_uri, +) + + +@pytest.fixture +def view(): + # A View class we would add to galaxy.model.view + class TestView(View): + name = 'testview' + __view__ = text('SELECT id, foo FROM testfoo').columns( + column('id', Integer), + column('foo', Integer) + ) + pkeys = {'id'} + View._make_table(name, __view__, pkeys) + + return TestView + + +@skip_if_not_postgres_uri +def test_postgres_create_view(database_name, postgres_url, view): + metadata = MetaData() + make_table(metadata) # table from which the view will select + url = replace_database_in_url(postgres_url, database_name) + query = f"SELECT 1 FROM information_schema.views WHERE table_name = '{view.name}'" + create_database(postgres_url, database_name) + run_view_test(url, metadata, view, query) + drop_database(postgres_url, database_name) + + +def test_sqlite_create_view(sqlite_memory_url, view): + metadata = MetaData() + make_table(metadata) # table from which the view will select + url = sqlite_memory_url + query = f"SELECT 1 FROM sqlite_master WHERE type='view' AND name='{view.name}'" + run_view_test(url, metadata, view, query) + + +@skip_if_not_mysql_uri +def test_mysql_create_view(database_name, mysql_url, view): + metadata = MetaData() + make_table(metadata) # table from which the view will select + url = replace_database_in_url(mysql_url, database_name) + query = f"SELECT 1 FROM information_schema.views WHERE table_name = '{view.name}'" + create_database(mysql_url, database_name) + run_view_test(url, metadata, view, query) + drop_database(mysql_url, database_name) + + +def make_table(metadata): + users = Table('testfoo', metadata, + Column('id', Integer, primary_key=True), + Column('foo', Integer), + Column('bar', Integer) + ) + return users + + +def run_view_test(url, metadata, view, query): + with sqlalchemy_engine(url) as engine: + with engine.connect() as conn: + metadata.create_all(conn) # create table in database + conn.execute(CreateView(view.name, view.__view__)) # create view in database + result = conn.execute(query).fetchall() + assert len(result) == 1 # assert that view exists in database