Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: Clean up DB sessions consistently #702

Closed
wants to merge 24 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ env:
jobs:
test_quetz:
# timeout for the whole job
timeout-minutes: 10
timeout-minutes: 15
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
Expand Down Expand Up @@ -79,7 +79,7 @@ jobs:
- name: Testing server
shell: bash -l -eo pipefail {0}
# timeout for the step
timeout-minutes: 5
timeout-minutes: 15
env:
TEST_DB_BACKEND: ${{ matrix.test_database }}
QUETZ_TEST_DBINIT: ${{ matrix.db_init }}
Expand Down
1 change: 0 additions & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ repos:
- id: mypy
files: ^quetz/
additional_dependencies:
- sqlalchemy-stubs
- types-click
- types-Jinja2
- types-mock
Expand Down
2 changes: 1 addition & 1 deletion environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ dependencies:
- authlib=0.15.5
- psycopg2
- httpx>=0.22.0
- sqlalchemy
- sqlalchemy >= 2, <3
- sqlalchemy-utils
- sqlite
- python-multipart
Expand Down
7 changes: 2 additions & 5 deletions init_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,10 @@
def init_test_db():
config = Config()
init_db(config.sqlalchemy_database_url)
db = get_session(config.sqlalchemy_database_url)

testUsers = []
with get_session(config) as db:
testUsers = []

try:
for index, username in enumerate(["alice", "bob", "carol", "dave"]):
user = User(id=uuid.uuid4().bytes, username=username)

Expand Down Expand Up @@ -102,8 +101,6 @@ def init_test_db():

db.add(channel_member)
db.commit()
finally:
db.close()


if __name__ == "__main__":
Expand Down
4 changes: 2 additions & 2 deletions plugins/quetz_conda_suggest/quetz_conda_suggest/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import quetz
from quetz.config import Config
from quetz.database import get_db_manager
from quetz.database import get_session
from quetz.db_models import PackageVersion
from quetz.utils import add_entry_for_index

Expand Down Expand Up @@ -45,7 +45,7 @@ def post_add_package_version(version, condainfo):
if command not in suggest_map:
suggest_map[command] = package

with get_db_manager() as db:
with get_session() as db:
if not version.binfiles:
metadata = db_models.CondaSuggestMetadata(
version_id=version.id, data=json.dumps(suggest_map)
Expand Down
2 changes: 1 addition & 1 deletion plugins/quetz_conda_suggest/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def package_version(user, channel, db, dao, package):
package_format,
"linux-64",
"0.1",
"0",
0,
"0",
"test-package-0.1-0.tar.bz2",
package_info,
Expand Down
14 changes: 4 additions & 10 deletions plugins/quetz_conda_suggest/tests/test_quetz_conda_suggest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import shutil
import tarfile
import tempfile
from contextlib import contextmanager
from unittest import mock

import pytest
Expand All @@ -18,7 +17,7 @@
assert response.json() == None # noqa: E711


def test_post_add_package_version(package_version, db, config):
def test_post_add_package_version(package_version, db, config, session_maker):
filename = "test-package-0.1-0.tar.bz2"

with tempfile.SpooledTemporaryFile(mode="wb") as target:
Expand All @@ -27,13 +26,9 @@
target.seek(0)
condainfo = CondaInfo(target, filename)

@contextmanager
def get_db():
yield db

from quetz_conda_suggest import main

with mock.patch("quetz_conda_suggest.main.get_db_manager", get_db):
with mock.patch("quetz_conda_suggest.main.get_session", session_maker):
main.post_add_package_version(package_version, condainfo)

meta = db.query(db_models.CondaSuggestMetadata).first()
Expand All @@ -50,8 +45,8 @@
b"lib/libtpkg.so\n",
b"lib/pkgconfig/libtpkg.pc\n",
]
with mock.patch("quetz_conda_suggest.main.get_db_manager", get_db):
with mock.patch("quetz_conda_suggest.main.get_session", session_maker):
main.post_add_package_version(package_version, condainfo)

Check failure on line 49 in plugins/quetz_conda_suggest/tests/test_quetz_conda_suggest.py

View workflow job for this annotation

GitHub Actions / test_quetz (ubuntu-latest, sqlite, create-tables)

test_post_add_package_version sqlalchemy.exc.IntegrityError: (sqlite3.IntegrityError) UNIQUE constraint failed: quetz_conda_suggest_metadata.version_id [SQL: INSERT INTO quetz_conda_suggest_metadata (version_id, data) VALUES (?, ?)] [parameters: (<memory at 0x7f51a7426080>, '***"test-bin": "test-package"***')] (Background on this error at: https://sqlalche.me/e/20/gkpj)

meta = db.query(db_models.CondaSuggestMetadata).all()

Expand All @@ -76,7 +71,6 @@
response = client.get("/api/dummylogin/madhurt")
filename = "test-package-0.1-0.tar.bz2"

@contextmanager
def get_db():
yield db

Expand Down Expand Up @@ -114,10 +108,10 @@
tar.addfile(t, io.BytesIO(b))
tar.close()

with mock.patch("quetz_conda_suggest.main.get_db_manager", get_db):
with mock.patch("quetz_conda_suggest.main.get_session", get_db):
url = f"/api/channels/{channel.name}/files/"
files = {"files": (filename, open(filename, "rb"))}
response = client.post(url, files=files)

Check failure on line 114 in plugins/quetz_conda_suggest/tests/test_quetz_conda_suggest.py

View workflow job for this annotation

GitHub Actions / test_quetz (ubuntu-latest, sqlite, create-tables)

test_conda_suggest_endpoint_with_upload TypeError: 'generator' object does not support the context manager protocol

assert response.status_code == 201

Expand Down
8 changes: 4 additions & 4 deletions plugins/quetz_content_trust/quetz_content_trust/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from quetz import authorization
from quetz.config import Config
from quetz.database import get_db_manager
from quetz.database import get_session
from quetz.deps import get_rules

from . import db_models
Expand Down Expand Up @@ -102,7 +102,7 @@ def post_role(
):
auth.assert_channel_roles(channel, ["owner"])

with get_db_manager() as db:
with get_session() as db:
existing_role_count = (
db.query(db_models.ContentTrustRole)
.filter(
Expand Down Expand Up @@ -190,7 +190,7 @@ def get_role(
):
auth.assert_channel_roles(channel, ["owner", "maintainer", "member"])

with get_db_manager() as db:
with get_session() as db:
query = (
db.query(db_models.ContentTrustRole)
.filter(db_models.ContentTrustRole.channel == channel)
Expand All @@ -211,7 +211,7 @@ def get_new_key(secret: bool = False):
mamba_key = libmamba_api.Key.from_ed25519(key.public_key)
private_key = key.private_key

with get_db_manager() as db:
with get_session() as db:
db.add(key)
db.commit()

Expand Down
4 changes: 2 additions & 2 deletions plugins/quetz_content_trust/quetz_content_trust/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from sqlalchemy import desc

import quetz
from quetz.database import get_db_manager
from quetz.database import get_session

from . import db_models
from .api import router
Expand All @@ -21,7 +21,7 @@ def register_router():
def post_index_creation(raw_repodata: dict, channel_name, subdir):
"""Use available online keys to sign packages"""

with get_db_manager() as db:
with get_session() as db:
query = (
db.query(db_models.SigningKey)
.join(db_models.RoleDelegation.keys)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import quetz
from quetz.config import Config
from quetz.database import get_db_manager
from quetz.database import get_session
from quetz.db_models import PackageFormatEnum, PackageVersion
from quetz.utils import add_temp_static_file

Expand Down Expand Up @@ -107,7 +107,7 @@ def _load_instructions(tar, path):

@quetz.hookimpl(tryfirst=True)
def post_package_indexing(tempdir: Path, channel_name, subdirs, files, packages):
with get_db_manager() as db:
with get_session() as db:
query = (
db.query(PackageVersion)
.filter(
Expand Down
Loading
Loading