From 985c114efd9705387ec3a4515c1bbbd1d766a962 Mon Sep 17 00:00:00 2001 From: RKIMetadataExchange Date: Fri, 8 Nov 2024 13:10:10 +0000 Subject: [PATCH 1/3] Bump cookiecutter template to af56c8 --- .cruft.json | 2 +- .github/workflows/cookiecutter.yml | 11 +---------- .github/workflows/release.yml | 11 +---------- .pre-commit-config.yaml | 2 +- README.md | 23 ++++++++++++----------- 5 files changed, 16 insertions(+), 33 deletions(-) diff --git a/.cruft.json b/.cruft.json index 69b4294f..e4c76675 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/robert-koch-institut/mex-template", - "commit": "fca460e90134ef8c10f54d491aa80891509b1b6c", + "commit": "af56c8fa17144906e94518391acc9795d5f2dc89", "checkout": null, "context": { "cookiecutter": { diff --git a/.github/workflows/cookiecutter.yml b/.github/workflows/cookiecutter.yml index a06f00f4..4862c11a 100644 --- a/.github/workflows/cookiecutter.yml +++ b/.github/workflows/cookiecutter.yml @@ -52,16 +52,7 @@ jobs: SIGNING_PUB: ${{ secrets.SIGNING_PUB }} run: | eval "$(ssh-agent -s)" - install --directory ~/.ssh --mode 700 - base64 -d <<< '${{ secrets.SIGNING_KEY }}' > ~/.ssh/mex - base64 -d <<< '${{ secrets.SIGNING_PUB }}' > ~/.ssh/mex.pub - chmod 600 ~/.ssh/* - ssh-add ~/.ssh/mex - git config --local user.email ${{ vars.MEX_BOT_EMAIL }} - git config --local user.name ${{ vars.MEX_BOT_USER }} - git config --local gpg.format ssh - git config --local user.signingkey ~/.ssh/mex.pub - git config --local commit.gpgsign true + pdm setup-commit-signing - name: Update template env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b99cb575..8dd10172 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -63,16 +63,7 @@ jobs: SIGNING_PUB: ${{ secrets.SIGNING_PUB }} run: | eval "$(ssh-agent -s)" - install --directory ~/.ssh --mode 700 - base64 -d <<< '${{ secrets.SIGNING_KEY }}' > ~/.ssh/mex - base64 -d <<< '${{ secrets.SIGNING_PUB }}' > ~/.ssh/mex.pub - chmod 600 ~/.ssh/* - ssh-add ~/.ssh/mex - git config --local user.email ${{ vars.MEX_BOT_EMAIL }} - git config --local user.name ${{ vars.MEX_BOT_USER }} - git config --local gpg.format ssh - git config --local user.signingkey ~/.ssh/mex.pub - git config --local commit.gpgsign true + pdm setup-commit-signing - name: Release new version id: release diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37152b44..75548378 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: python: python3.11 repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.5 + rev: v0.7.2 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/README.md b/README.md index a6750fb3..14492c54 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Backend server for the RKI metadata exchange. [![open-code](https://github.com/robert-koch-institut/mex-backend/actions/workflows/open-code.yml/badge.svg)](https://gitlab.opencode.de/robert-koch-institut/mex/mex-backend) [![testing](https://github.com/robert-koch-institut/mex-backend/actions/workflows/testing.yml/badge.svg)](https://github.com/robert-koch-institut/mex-backend/actions/workflows/testing.yml) -## project +## Project The Metadata Exchange (MEx) project is committed to improve the retrieval of RKI research data and projects. How? By focusing on metadata: instead of providing the @@ -39,27 +39,28 @@ data Findable, Accessible, Interoperable and Reusable. **Contact** \ For more information, please feel free to email us at [mex@rki.de](mailto:mex@rki.de). -### Publisher of this document +### Publisher + **Robert Koch-Institut** \ Nordufer 20 \ 13353 Berlin \ Germany -## package +## Package The `mex-backend` package is a multi-purpose backend application with an HTTP-API. It provides endpoints to ingest data from ETL-pipelines, for a metadata editor application, and for publishing pipelines to extract standardized data for use in upstream frontend applications. -## license +## License This package is licensed under the [MIT license](/LICENSE). All other software components of the MEx project are open-sourced under the same license as well. -## development +## Development -### installation +### Installation - on unix, consider using pyenv https://github.com/pyenv/pyenv - get pyenv `curl https://pyenv.run | bash` @@ -72,13 +73,13 @@ components of the MEx project are open-sourced under the same license as well. - switch version `pyenv global 3.11` - run `.\mex.bat install` -### linting and testing +### Linting and testing - run all linters with `pdm lint` - run only unit tests with `pdm unit` - run unit and integration tests with `pdm test` -### updating dependencies +### Updating dependencies - update boilerplate files with `cruft update` - update global requirements in `requirements.txt` manually @@ -86,18 +87,18 @@ components of the MEx project are open-sourced under the same license as well. - update package dependencies using `pdm update-all` - update github actions in `.github/workflows/*.yml` manually -### creating release +### Creating release - run `pdm release RULE` to release a new version where RULE determines which part of the version to update and is one of `major`, `minor`, `patch`. -### container workflow +### Container workflow - build image with `make image` - run directly using docker `make run` - start with docker compose `make start` -## commands +## Commands - run `pdm run {command} --help` to print instructions - run `pdm run {command} --debug` for interactive debugging From 82e676287ce2cd7e48f1f8d20d9f23385d4f153c Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Fri, 8 Nov 2024 14:26:18 +0100 Subject: [PATCH 2/3] lock --- pdm.lock | 8 ++-- pyproject.toml | 114 ++++++++++++++++++++++++++----------------------- 2 files changed, 64 insertions(+), 58 deletions(-) diff --git a/pdm.lock b/pdm.lock index b01e5310..a7bafc34 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:013119e2bfcd5bea3d1983ddde73e109ab0533018200759084fbdb4e8c6762fe" +content_hash = "sha256:1e5bb851c30361443f04880fd89e191d5632eaef33a12062c647e71d7168a567" [[metadata.targets]] requires_python = "==3.11.*" @@ -1230,14 +1230,14 @@ files = [ [[package]] name = "types-pytz" -version = "2024.2.0.20241003" +version = "2024.1.0.20240417" requires_python = ">=3.8" summary = "Typing stubs for pytz" groups = ["dev"] marker = "python_version == \"3.11\"" files = [ - {file = "types-pytz-2024.2.0.20241003.tar.gz", hash = "sha256:575dc38f385a922a212bac00a7d6d2e16e141132a3c955078f4a4fd13ed6cb44"}, - {file = "types_pytz-2024.2.0.20241003-py3-none-any.whl", hash = "sha256:3e22df1336c0c6ad1d29163c8fda82736909eb977281cb823c57f8bae07118b7"}, + {file = "types-pytz-2024.1.0.20240417.tar.gz", hash = "sha256:6810c8a1f68f21fdf0f4f374a432487c77645a0ac0b31de4bf4690cf21ad3981"}, + {file = "types_pytz-2024.1.0.20240417-py3-none-any.whl", hash = "sha256:8335d443310e2db7b74e007414e74c4f53b67452c0cb0d228ca359ccfba59659"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 449ac8f4..11d4b4d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,25 +8,25 @@ license = { file = "LICENSE" } urls = { Repository = "https://github.com/robert-koch-institut/mex-backend" } requires-python = ">=3.11,<3.13" dependencies = [ - "fastapi>=0.115.4,<1", - "httpx>=0.27.2,<1", - "jinja2>=3.1.4,<4", + "fastapi>=0.115,<1", + "httpx>=0.27,<1", + "jinja2>=3,<4", "mex-common @ git+https://github.com/robert-koch-institut/mex-common.git@0.40.0", - "neo4j>=5.24.0,<6", - "pydantic>=2.9.1,<3", - "starlette>=0.41.2,<1", - "uvicorn[standard]>=0.30.6,<1", + "neo4j>=5,<6", + "pydantic>=2,<3", + "starlette>=0.41,<1", + "uvicorn[standard]>=0.30,<1", ] optional-dependencies.dev = [ "black>=24,<25", - "ipdb>=0.13.13,<1", - "mypy>=1.11.2,<2", + "ipdb>=0.13,<1", + "mypy>=1,<2", "pytest-cov>=6,<7", - "pytest-random-order>=1.1.1,<2", - "pytest>=8.3.3,<9", - "ruff>=0.6.5,<1", - "sphinx>=8.0.2,<9", - "types-pytz>=2024.1.0,<2025", + "pytest-random-order>=1,<2", + "pytest>=8,<9", + "ruff>=0.6,<1", + "sphinx>=8,<9", + "types-pytz>=2024,<2024.2", ] [project.scripts] @@ -90,51 +90,57 @@ docstring-code-format = true [tool.ruff.lint] ignore = [ - "D100", # Allow missing module docstring for brevity - "D104", # Allow missing package docstring for brevity - "D106", # Allow missing nested class docstring (eg pydantic Config) - "D203", # Disallow blank line before class docstring (inverse of D211) - "D213", # Disallow multi-line docstring starting at second line (inverse of D212) - "D406", # Allow section name ending with newline (google style compat) - "D407", # Allow missing dashed underline after section (google style compat) - "D413", # Allow missing blank line after last section (google style compat) - "N805", # Allow first argument of a method to be non-self (pydantic compat) - "N815", # Allow mixedCase variables in class scope (model compat) - "RUF012", # Allow mutable class attributes (pydantic compat) -] -select = [ - "A", # Flake8 builtin shaddow - "B", # BugBear bug and issue finder - "C90", # McCabe complexity checker - "D", # Python docstring style checker - "E", # Python code style errors - "ERA", # Commented-out code detector - "F", # Pyflakes passive python checker - "I", # Isort import utility - "N", # Pep8 naming conventions - "PERF", # Lint performance anti-patterns - "RET", # Flake8 return statement checker - "RUF", # Ruff-specific rules - "S", # Bandit automated security testing - "T10", # Flake8 debug statement checker - "T20", # Flake8 print statement checker - "UP", # PyUpgrade syntax recommender - "W", # Python code style warnings + "AIR", # Disable airflow specific rules (we are not using airflow) + "ANN", # Disable all annotations checks (handled by mypy) + "COM", # Disable flake8-commas checks (let ruff format handle that) + "CPY", # Disable copyright notice checks (we have LICENSE files) + "D100", # Allow missing module docstring (for brevity and speed) + "D104", # Allow missing package docstring (for brevity and speed) + "D203", # Disallow blank line before class docstring (inverse of D211) + "D213", # Disallow multi-line docstring starting at second line (inverse of D212) + "D406", # Allow section name ending with newline (google style compat) + "D407", # Allow missing dashed underline after section (google style compat) + "D413", # Allow missing blank line after last section (google style compat) + "DJ", # Disable django specific checks (we are not using django) + "FBT", # Disable boolean type hint checks (for more flexibility) + "FIX", # Allow committing with open TODOs (don't punish committers) + "ISC001", # Disable checks for implicitly concatenated strings (formatter compat) + "N805", # Allow first argument of a method to be non-self (pydantic compat) + "N815", # Allow mixedCase variables in class scope (model compat) + "PTH123", # Allow using builtin open method (simpler than pathlib) + "RUF012", # Allow mutable class attributes (pydantic compat) + "SIM108", # Allow explicit if-else instead of ternary (easier to read) + "TD003", # Allow TODOs without ticket link (don't punish TODO writers) + "TRY003", # Allow long exception message at the raise site (for pydantic) ] +select = ["ALL"] [tool.ruff.lint.per-file-ignores] -"tests/**" = [ - "D101", # Allow missing docstring in public class for tests - "D102", # Allow missing docstring in public method for tests - "D103", # Allow missing docstring in public function for tests - "D107", # Allow missing docstring in `__init__` for tests - "E501", # Allow line too long in tests - "N807", # Allow mocking `__init__` for tests - "S101", # Allow use of `assert` in tests +"docs/**" = [ + "INP001", # Docs folder does not need to be a package ] -"**/main.py" = [ - "N803", # Allow dromedaryCase query parameters +"scripts/**" = [ + "INP001", # Scripts folder does not need to be a package ] +"tests/**" = [ + "ARG005", # Allow unused lambda arguments for mocking + "D101", # Allow missing docstring in public class + "D102", # Allow missing docstring in public method + "D103", # Allow missing docstring in public function + "D107", # Allow missing docstring in `__init__` + "E501", # Allow longer lines with test data + "ISC", # Allow implicitly concatenated strings + "N807", # Allow mocking `__init__` + "PLR0915", # Allow functions with many statements + "PLR2004", # Allow comparing with static values + "PT004", # Allow public fixtures without returns + "PT013", # Allow more flexible pytest imports + "S101", # Allow use of `assert` in tests + "SLF", # Allow private member access +] + +[tool.ruff.lint.flake8-import-conventions.extend-aliases] +"reflex" = "rx" [tool.ruff.lint.isort] known-first-party = ["mex", "tests"] From d7fa486d8242f10f6d9fda057edb7ecee49dcf19 Mon Sep 17 00:00:00 2001 From: Nicolas Drebenstedt Date: Fri, 8 Nov 2024 14:34:59 +0100 Subject: [PATCH 3/3] lint --- pyproject.toml | 3 +++ tests/auxiliary/conftest.py | 4 ++-- tests/identity/test_provider.py | 2 +- tests/merged/test_helpers.py | 8 ++++---- tests/rules/test_helpers.py | 4 ++-- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 11d4b4d5..8ccfacec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,9 @@ select = ["ALL"] "scripts/**" = [ "INP001", # Scripts folder does not need to be a package ] +"**/main.py" = [ + "N803", # Allow dromedaryCase query parameters +] "tests/**" = [ "ARG005", # Allow unused lambda arguments for mocking "D101", # Allow missing docstring in public class diff --git a/tests/auxiliary/conftest.py b/tests/auxiliary/conftest.py index c765d317..d3487ea6 100644 --- a/tests/auxiliary/conftest.py +++ b/tests/auxiliary/conftest.py @@ -32,7 +32,7 @@ def mocked_init(self: WikidataQueryServiceConnector) -> None: # mock search_wikidata_with_query def get_data_by_query( - self: WikidataQueryServiceConnector, query: str + _self: WikidataQueryServiceConnector, _query: str ) -> list[dict[str, dict[str, str]]]: return [ { @@ -63,7 +63,7 @@ def get_data_by_query( wikidata_organization_raw = json.load(fh) def get_wikidata_item_details_by_id( - self: WikidataQueryServiceConnector, item_id: str + _self: WikidataQueryServiceConnector, _item_id: str ) -> dict[str, str]: return wikidata_organization_raw diff --git a/tests/identity/test_provider.py b/tests/identity/test_provider.py index 83802f51..b173d9b2 100644 --- a/tests/identity/test_provider.py +++ b/tests/identity/test_provider.py @@ -211,7 +211,7 @@ def test_assign_identity( ], ids=["nothing found", "one item", "two items"], ) -def test_fetch_identities_mocked( +def test_fetch_identities_mocked( # noqa: PLR0913 mocked_graph: MockedGraph, mocked_return: list[dict[str, str]], had_primary_source: MergedPrimarySourceIdentifier | None, diff --git a/tests/merged/test_helpers.py b/tests/merged/test_helpers.py index 9d6348af..a31f7c1d 100644 --- a/tests/merged/test_helpers.py +++ b/tests/merged/test_helpers.py @@ -245,8 +245,8 @@ def test_create_merged_item( extracted_items, rule_set, ) - except Exception as error: - assert str(expected) in str(error) + except Exception as error: # noqa: BLE001 + assert str(expected) in str(error) # noqa: PT017 else: assert merged_item.model_dump(exclude_defaults=True) == expected @@ -462,7 +462,7 @@ def test_search_merged_items_in_graph_mocked( try: merged_result = search_merged_items_in_graph(stable_target_id="bFQoRhcVH5DHUB") - except Exception as error: - assert str(expected) in str(error), error + except Exception as error: # noqa: BLE001 + assert str(expected) in str(error) # noqa: PT017 else: assert merged_result.model_dump(exclude_defaults=True) == expected diff --git a/tests/rules/test_helpers.py b/tests/rules/test_helpers.py index 078d5322..02c9515a 100644 --- a/tests/rules/test_helpers.py +++ b/tests/rules/test_helpers.py @@ -96,7 +96,7 @@ def test_transform_raw_rules_to_rule_set_response( ) -> None: try: rule_set = transform_raw_rules_to_rule_set_response(items) - except Exception as error: - assert str(expected) in str(error) + except Exception as error: # noqa: BLE001 + assert str(expected) in str(error) # noqa: PT017 else: assert rule_set.model_dump() == expected