From fb1a0ff2db3859f4ba79d7670c9e2a6b922d127e Mon Sep 17 00:00:00 2001 From: Michael Puehringer Date: Thu, 12 Jan 2023 08:08:57 +0100 Subject: [PATCH 1/2] Upgrade python deps --- .gitignore | 3 +++ Makefile | 34 ++++++++++++++++++++++++++-------- coral/__init__.py | 4 +--- coral/db.py | 2 +- coral/migration/env.py | 2 +- coral/settings.py | 4 +--- coral/sql_query_mapper.py | 34 ++++++++-------------------------- coral/tests/conftest.py | 34 ++++++++++++++++++++++++++++++++++ coral/tests/test_api.py | 5 +++++ coral/tests/test_dummy.py | 2 -- requirements.txt | 2 +- requirements_dev.txt | 12 ++++-------- setup.py | 2 +- 13 files changed, 86 insertions(+), 54 deletions(-) create mode 100755 coral/tests/conftest.py create mode 100644 coral/tests/test_api.py delete mode 100644 coral/tests/test_dummy.py diff --git a/.gitignore b/.gitignore index cf8a9f2..c9762f2 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ package-lock.json !.yarn/releases !.yarn/sdks !.yarn/versions + +# venv +.venv/ diff --git a/Makefile b/Makefile index 5e14a86..c9a66c4 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,13 @@ .DEFAULT_GOAL := help pkg_src = coral -flake8 = flake8 $(pkg_src) setup.py -isort = isort $(pkg_src) setup.py black = black --line-length 140 $(pkg_src) setup.py +pyright = echo 'pyright not used' # pyright $(pkg_src) setup.py +ruff = ruff $(pkg_src) setup.py --line-length 140 --select E,W,F,N,I,C,B,UP,PT,SIM,RUF --ignore E501,C901,B008 + +.PHONY: start ## Start the development server +start: + python $(pkg_src) .PHONY: all ## Perform the most common development-time rules all: format lint test @@ -13,25 +17,25 @@ ci: check-format lint test .PHONY: format ## Auto-format the source code format: - $(isort) + $(ruff) --fix $(black) .PHONY: check-format ## Check the source code format without changes check-format: - $(isort) --check-only $(black) --check -.PHONY: lint ## Run flake8 +.PHONY: lint ## Run flake8 and pyright lint: - $(flake8) + $(ruff) --format=github + $(pyright) .PHONY: test ## Run tests test: pytest $(pkg_src) .PHONEY: documentation ## Generate docs -documentation: - mkdocs build +documentation: + echo "TODO" .PHONY: install ## Install the requirements install: @@ -41,6 +45,20 @@ install: develop: pip install -e .[develop] +.PHONY: env_encrypt ## Encrypts the current .//.env +env_encrypt: + openssl aes-256-cbc -pbkdf2 -in ./$(pkg_src)/.env -out ./$(pkg_src)/.env.enc + +.PHONY: env_decrypt ## Decrypts the .//.env.enc +env_decrypt: + @if [ -z "${ENV_PASSWORD}" ]; then \ + echo "No ENV_PASSWORD set, prompting for password..."; \ + openssl aes-256-cbc -pbkdf2 -d -in ./$(pkg_src)/.env.enc -out ./$(pkg_src)/.env; \ + else \ + echo "ENV_PASSWORD set, using it..."; \ + openssl aes-256-cbc -pbkdf2 -d -in ./$(pkg_src)/.env.enc -out ./$(pkg_src)/.env -pass env:ENV_PASSWORD; \ + fi + .PHONY: build ## Build a wheel build: python setup.py sdist bdist_wheel --dist-dir dist_python diff --git a/coral/__init__.py b/coral/__init__.py index ccdc298..9f43879 100644 --- a/coral/__init__.py +++ b/coral/__init__.py @@ -4,9 +4,7 @@ # Licensed under the new BSD license, available at http://caleydo.org/license ############################################################################### from os import path -from typing import Type -from pydantic import BaseModel from tdp_core.plugin.model import AVisynPlugin, RegHelper from .settings import CoralSettings @@ -34,5 +32,5 @@ def register(self, registry: RegHelper): ) @property - def setting_class(self) -> Type[BaseModel]: + def setting_class(self) -> type[CoralSettings]: return CoralSettings diff --git a/coral/db.py b/coral/db.py index f76a859..0e49ed8 100644 --- a/coral/db.py +++ b/coral/db.py @@ -12,7 +12,7 @@ _log.info("Setting up the db view.") # main dictionary containing all views registered for this plugin -views = dict() +views = {} schema = "cohort" # create a view + idtype for each available table diff --git a/coral/migration/env.py b/coral/migration/env.py index ea7b7dd..98295d2 100644 --- a/coral/migration/env.py +++ b/coral/migration/env.py @@ -1,3 +1,3 @@ -import tdp_core.dbmigration.env # NOQA +import tdp_core.dbmigration.env tdp_core.dbmigration.env.run_migrations_online() diff --git a/coral/settings.py b/coral/settings.py index 822d0d3..b1adcbb 100644 --- a/coral/settings.py +++ b/coral/settings.py @@ -1,5 +1,3 @@ -from typing import Dict - from pydantic import BaseModel from tdp_core import manager @@ -11,7 +9,7 @@ class CoralSettings(BaseModel): statement_timeout: str = "300000" supp_statement_timeout: str = "40000" statement_timeout_query: str = "set statement_timeout to {}" - logging: Dict = {"version": 1, "disable_existing_loggers": False, "loggers": {"coral": {"level": "DEBUG"}}} + logging: dict = {"version": 1, "disable_existing_loggers": False, "loggers": {"coral": {"level": "DEBUG"}}} def get_settings() -> CoralSettings: diff --git a/coral/sql_query_mapper.py b/coral/sql_query_mapper.py index d5e4d9b..f2dca51 100644 --- a/coral/sql_query_mapper.py +++ b/coral/sql_query_mapper.py @@ -95,7 +95,7 @@ def to_dict(self, row): try: return {c.key: getattr(row, c.key) for c in inspect(row).mapper.column_attrs} except NoInspectionAvailable: # this exception is raised when the result is not of a registered sqlalchemy type due to which the inspection fails - result = dict() + result = {} result[COLUMN_LABEL_SCORE] = row[0] result[COLUMN_LABEL_ID] = row[1] return result @@ -145,10 +145,7 @@ def get_cohorts_by_id_sql(self, args, error_msg): str_values = str_values + ("{val}, ".format(val=curr_val)) cnt += 1 - if cnt > 0: - str_values = str_values[:-2] # remove the last ', ' from the value list - else: - str_values = "-1" # returns no cohorts -> ids are only positiv + str_values = str_values[:-2] if cnt > 0 else "-1" # returns no cohorts -> ids are only positiv sql_text = "SELECT id, name, is_initial, previous_cohort, entity_database, entity_schema, entity_table FROM cohort.cohort c WHERE c.id IN ({str_values})".format( str_values=str_values @@ -173,7 +170,7 @@ def update_cohort_name_sql(self, args, error_msg): session_data.query(Cohort).filter(Cohort.id == cohort_id).update({Cohort.name: name}, synchronize_session=False) # synchronize_session - # False - don’t synchronize the session. This option is the most efficient and is reliable once the session is expired, + # False - don`t synchronize the session. This option is the most efficient and is reliable once the session is expired, # which typically occurs after a commit(), or explicitly using expire_all(). Before the expiration, # updated objects may still remain in the session with stale values on their attributes, which can lead to confusing results. @@ -341,19 +338,13 @@ def equals_filter_statement(self, prefix, attribute, values, numeric): if val.startswith("!"): add_equals_not_value = True val = val[1:] - if numeric in ["true"]: # determine if the values are strings or numbers - curr_val = "{val}".format(val=val) - else: - curr_val = "'{val}'".format(val=val) + curr_val = "{val}".format(val=val) if numeric in ["true"] else "'{val}'".format(val=val) str_not_values = str_not_values + ("{val}, ".format(val=curr_val)) else: add_equals_value = True - if numeric in ["true"]: # determine if the values are strings or numbers - curr_val = "{val}".format(val=val) - else: - curr_val = "'{val}'".format(val=val) + curr_val = "{val}".format(val=val) if numeric in ["true"] else "'{val}'".format(val=val) str_values = str_values + ("{val}, ".format(val=curr_val)) @@ -455,10 +446,7 @@ def create_cohort_treatment_filtered(self, args, cohort, error_msg): raise RuntimeError(error_msg) array_operation = "=" - if base_agent in ["true"]: # determine if the values are strings or numbers - array_operation = "@>" - else: - array_operation = "=" + array_operation = "@>" if base_agent in ["true"] else "=" entity_id_col = "" if cohort.entity_table == "tdp_tissue": @@ -508,10 +496,7 @@ def create_cohort_treatment_filtered(self, args, cohort, error_msg): # regimen is defined if regimen is not None: reg_sql = ("tmp.rn::int = {val}").format(val=regimen) - if agent is not None: - sql_where = sql_where + " AND " + reg_sql - else: - sql_where = reg_sql + sql_where = sql_where + " AND " + reg_sql if agent is not None else reg_sql # SQL with the combined filter (actual agent values, null values) sql_refiend = "" @@ -555,10 +540,7 @@ def create_cohort_treatment_filtered(self, args, cohort, error_msg): base_table=cohort.entity_table, ) - if agent_exists: - sql_refiend = sql_refiend + " UNION " + sql_null - else: - sql_refiend = sql_refiend + sql_null + sql_refiend = sql_refiend + " UNION " + sql_null if agent_exists else sql_refiend + sql_null # complete SQL statement that filters the data based on the given cohort new_sql_text = """SELECT cohort.* FROM ({entities}) cohort diff --git a/coral/tests/conftest.py b/coral/tests/conftest.py new file mode 100755 index 0000000..73deb4f --- /dev/null +++ b/coral/tests/conftest.py @@ -0,0 +1,34 @@ +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from tdp_core.security.manager import SecurityManager +from tdp_core.security.model import User +from tdp_core.server.visyn_server import create_visyn_server +from tdp_core.tests.fixtures.postgres_db import postgres_db + +assert postgres_db # silence unused import warning + + +@pytest.fixture(scope="session") +def app(postgres_db) -> FastAPI: + server = create_visyn_server( + workspace_config={ + "tdp_core": {"enabled_plugins": ["tdp_core", "coral"]}, + "coral": { + "dburl": postgres_db.url, + }, + } + ) + + return server + + +@pytest.fixture() +def client(monkeypatch, app: FastAPI): + def mock_current_user_in_manager(self): + return User(id="admin") + + monkeypatch.setattr(SecurityManager, "current_user", property(mock_current_user_in_manager)) + + with TestClient(app) as client: + yield client diff --git a/coral/tests/test_api.py b/coral/tests/test_api.py new file mode 100644 index 0000000..6ed2de4 --- /dev/null +++ b/coral/tests/test_api.py @@ -0,0 +1,5 @@ +from fastapi.testclient import TestClient + + +def test_loggedinas(client: TestClient): + assert client.get("/loggedinas").json()["name"] == "admin" diff --git a/coral/tests/test_dummy.py b/coral/tests/test_dummy.py deleted file mode 100644 index 360271b..0000000 --- a/coral/tests/test_dummy.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_dummy(): - assert "it works" diff --git a/requirements.txt b/requirements.txt index e421511..869a813 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -tdp_core@git+https://github.com/datavisyn/tdp_core.git@develop#egg=tdp_core +tdp_core@git+https://github.com/datavisyn/tdp_core.git@upgrade_python_deps#egg=tdp_core diff --git a/requirements_dev.txt b/requirements_dev.txt index 6d5e044..4193b14 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,9 +1,5 @@ -black~=22.3.0 -debugpy~=1.5.1 -flake8~=4.0.1 -isort~=5.10.1 -mkdocs-material~=8.2.8 -pep8-naming~=0.12.1 +black~=22.12.0 +pyright~=1.1.285 pytest-runner~=6.0.0 -pytest~=7.1.1 -recommonmark~=0.7.1 \ No newline at end of file +pytest~=7.2.0 +ruff==0.0.218 diff --git a/setup.py b/setup.py index 2921a2e..40c967f 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ def requirements(file): package_data={}, # Although 'package_data' is the preferred approach, in some case you may # need to place data files outside of your packages. See: - # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa + # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # In this case, 'data_file' will be installed into '/my_data' data_files=[], # [('my_data', ['data/data_file'])], ) From 8c8a94ed23fdf249e978ed182e1313f7f1ba3358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=BChringer?= <51900829+puehringer@users.noreply.github.com> Date: Thu, 12 Jan 2023 21:54:04 +0100 Subject: [PATCH 2/2] Switch back to #develop --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 869a813..e421511 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -tdp_core@git+https://github.com/datavisyn/tdp_core.git@upgrade_python_deps#egg=tdp_core +tdp_core@git+https://github.com/datavisyn/tdp_core.git@develop#egg=tdp_core