diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f95fb35..afc1140 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,4 @@ repos: - - repo: https://github.com/asottile/pyupgrade - rev: v2.31.1 - hooks: - - id: pyupgrade - args: - - --py311-plus - - repo: https://github.com/charliermarsh/ruff-pre-commit - # Ruff version. - rev: "v0.0.269" - hooks: - - id: ruff - args: [--fix, --exit-non-zero-on-fix] - repo: local hooks: - id: black @@ -18,8 +6,12 @@ repos: language: system entry: black types: [python] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.3.2' + hooks: + - id: ruff - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace exclude: ^src/api/client.js$ diff --git a/docs/gen_ref_pages.py b/docs/gen_ref_pages.py index c1e6654..a943c4b 100644 --- a/docs/gen_ref_pages.py +++ b/docs/gen_ref_pages.py @@ -28,6 +28,7 @@ with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file: nav_file.writelines(nav.build_literate_nav()) -readme = Path("README.md").open("r") -with mkdocs_gen_files.open("index.md", "w") as index_file: +with Path("README.md").open("r") as readme, mkdocs_gen_files.open( + "index.md", "w", +) as index_file: index_file.writelines(readme.read()) diff --git a/main.py b/main.py index fc079a8..9d2989d 100644 --- a/main.py +++ b/main.py @@ -1,72 +1,77 @@ -""" -Stores daily data from ACRCloud's broadcast monitoring service in ownCloud. -""" +"""Stores daily data from ACRCloud's broadcast monitoring service in ownCloud.""" + +from __future__ import annotations import json -import os from datetime import datetime, timedelta from functools import cache from io import BytesIO from logging import getLogger +from pathlib import Path +from typing import Any import urllib3 from acrclient import Client as ACRClient from acrclient.models import GetBmCsProjectsResultsParams -from configargparse import ArgParser # type: ignore -from minio import Minio # type: ignore -from minio.error import S3Error # type: ignore -from owncloud import Client as OwnCloudClient # type: ignore -from owncloud.owncloud import HTTPResponseError as OCResponseError # type: ignore +from configargparse import ArgParser # type: ignore[import-untyped] +from minio import Minio # type: ignore[import-untyped] +from minio.error import S3Error # type: ignore[import-untyped] +from owncloud import Client as OwnCloudClient # type: ignore[import-untyped] +from owncloud.owncloud import HTTPResponseError # type: ignore[import-untyped] + +# type: ignore[import-untyped] from tqdm import tqdm logger = getLogger(__name__) -def daterange(start_date, end_date) -> list[datetime]: - dates = [] - for n in range(int((end_date - start_date).days)): - dates.append(start_date + timedelta(n)) - return dates +def daterange(start_date: datetime, end_date: datetime) -> list[datetime]: + """Get range to load.""" + return [start_date + timedelta(n) for n in range(int((end_date - start_date).days))] @cache def oc_mkdir(oc: OwnCloudClient, path: str) -> bool: + """Create a dir on ownCloud.""" try: return oc.mkdir(path) - except OCResponseError as ex: # pragma: no cover + except HTTPResponseError as ex: # pragma: no cover if str(ex) != "HTTP error: 405": - logger.exception(ex) + logger.exception("Failed to mkdir") return True @cache -def oc_file_exists(oc: OwnCloudClient, path: str): +def oc_file_exists(oc: OwnCloudClient, path: str) -> bool: + """Check if file exists on ownCloud.""" try: oc.file_info(path) - return True - except OCResponseError as ex: + except HTTPResponseError as ex: if str(ex) != "HTTP error: 404": # pragma: no cover - logger.exception(ex) + logger.exception("File missing") return False + else: + return True def oc_check(oc: OwnCloudClient, oc_path: str) -> list[datetime]: - """ - Checks ownCloud for missing files. - """ + """Check ownCloud for missing files.""" missing = [] - start = datetime.now() - timedelta(7) - for requested in tqdm(daterange(start, datetime.now()), desc="Checking ownCloud"): + start = datetime.now() - timedelta(7) # noqa: DTZ005 + for requested in tqdm( + daterange(start, datetime.now()), # noqa: DTZ005 + desc="Checking ownCloud", + ): oc_mkdir(oc, oc_path) - oc_mkdir(oc, os.path.join(oc_path, str(requested.year))) - oc_mkdir(oc, os.path.join(oc_path, str(requested.year), str(requested.month))) + oc_mkdir(oc, str(Path(oc_path) / str(requested.year))) + oc_mkdir(oc, str(Path(oc_path) / str(requested.year) / str(requested.month))) status = oc_file_exists( oc, - os.path.join( - oc_path, - str(requested.year), - str(requested.month), - requested.strftime("%Y-%m-%d.json"), + str( + Path(oc_path) + / str(requested.year) + / str(requested.month) + / requested.strftime("%Y-%m-%d.json"), ), ) if not status: @@ -75,15 +80,16 @@ def oc_check(oc: OwnCloudClient, oc_path: str) -> list[datetime]: def mc_check(mc: Minio, bucket: str) -> list[datetime]: - """ - Checks MinIO for missing files. - """ + """Check MinIO for missing files.""" missing = [] - start = datetime.now() - timedelta(7) - for requested in tqdm(daterange(start, datetime.now()), desc="Checking MinIO"): + start = datetime.now() - timedelta(7) # noqa: DTZ005 + for requested in tqdm( + daterange(start, datetime.now()), # noqa: DTZ005 + desc="Checking MinIO", + ): try: mc.stat_object(bucket, requested.strftime("%Y-%m-%d.json")) - except S3Error as ex: + except S3Error as ex: # noqa: PERF203 if ex.code == "NoSuchKey": missing.append(requested) return missing @@ -95,7 +101,8 @@ def fetch_one( acr_project_id: str, acr_stream_id: str, requested: str, -): +) -> Any: # noqa: ANN401 + """Fetch one "day" from ACRCloud.""" return acr.get_bm_cs_projects_results( project_id=int(acr_project_id), stream_id=acr_stream_id, @@ -109,21 +116,21 @@ def fetch_one( ) -def oc_fetch( +def oc_fetch( # noqa: PLR0913 missing: list[datetime], acr: ACRClient, oc: OwnCloudClient, acr_project_id: str, acr_stream_id: str, oc_path: str, -): - """Fetches missing data from ACRCloud and stores it in ownCloud.""" +) -> None: + """Fetch missing data from ACRCloud and stores it in ownCloud.""" for requested in tqdm(missing, desc="Loading into ownCloud from ACRCloud"): - target = os.path.join( - oc_path, - str(requested.year), - str(requested.month), - requested.strftime("%Y-%m-%d.json"), + target = str( + Path(oc_path) + / str(requested.year) + / str(requested.month) + / requested.strftime("%Y-%m-%d.json"), ) oc.put_file_contents( target, @@ -133,20 +140,20 @@ def oc_fetch( acr_project_id=acr_project_id, acr_stream_id=acr_stream_id, requested=requested.strftime("%Y%m%d"), - ) + ), ), ) -def mc_fetch( +def mc_fetch( # noqa: PLR0913 missing: list[datetime], acr: ACRClient, mc: Minio, acr_project_id: str, acr_stream_id: str, bucket: str, -): - """Fetches missing data from ACRCloud and stores it in MinIO.""" +) -> None: + """Fetch missing data from ACRCloud and stores it in MinIO.""" for requested in tqdm(missing, desc="Loading into MinIO from ACRCloud"): _as_bytes = json.dumps( fetch_one( @@ -154,7 +161,7 @@ def mc_fetch( acr_project_id=acr_project_id, acr_stream_id=acr_stream_id, requested=requested.strftime("%Y%m%d"), - ) + ), ).encode("utf-8") mc.put_object( bucket, @@ -165,7 +172,8 @@ def mc_fetch( ) -def main(): # pragma: no cover +def main() -> None: # pragma: no cover + """Fetch data from ACRCloud and stores it on-premise.""" p = ArgParser( description=__doc__, default_config_files=[ @@ -307,7 +315,8 @@ def main(): # pragma: no cover options.minio_secret_key, secure=options.minio_secure, http_client=urllib3.PoolManager( - cert_reqs=options.minio_cert_reqs, ca_certs=options.minio_ca_certs + cert_reqs=options.minio_cert_reqs, + ca_certs=options.minio_ca_certs, ), ) if not mc.bucket_exists(options.minio_bucket): diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..1fc0ef6 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,59 @@ +[lint] +select = [ + "F", # pyflakes + "E", # pycodestyle errors + "I", # isort + "C90", # mccabe + "N", # pep8-naming + "D", # pydocstyle + "UP", # pyupgrade + "ANN", # flake8-annotations + "ASYNC", # flake8-async + "S", # flake8-bandit + "BLE", # flake8-blind-exception + "FBT", # flake8-boolean-trap + "B", # flake8-bugbear + "A", # flake8-builtins + "COM", # flake8-commas + "C4", # flake8-comprehensions + "DTZ", # flake8-datetimez + "T10", # flake8-debugger + "EM", # flake8-errmsg + "EXE", # flake8-executable + "FA", # flake8-future-annotations + "ISC", # flake8-implicit-str-concat + "ICN", # flake8-import-conventions + "G", # flake8-logging-format + "INP", # flake8-no-pep420 + "PIE", # flake8-pie + "T20", # flake8-print + "PYI", # flake8-pyi + "PT", # flake8-pytest-style + "Q", # flake8-quotes + "RSE", # flake8-raise + "RET", # flake8-return + "SLF", # flake8-self + "SLOT", # flake8-slots + "SIM", # flake8-simplify + "TID", # flake8-tidy-imports + "TCH", # flake8-type-checking + "INT", # flake8-gettext + "ARG", # flake8-unused-arguments + "PTH", # flake8-use-pathlib + "TD", # flake8-todos + "ERA", # eradicate + "PGH", # pygrep-hooks + "PL", # Pylint + "TRY", # tryceratops + "PERF", # Perflint + "RUF", # ruff specific rules +] +ignore = [ + "D203", + "D213", +] + +[lint.per-file-ignores] +"tests/**/*.py" = ["S101", "D", "ANN", "INP001", "DTZ001", "S108"] +"**/__init__.py" = ["D104"] +"docs/gen_ref_pages.py" = ["INP001"] diff --git a/test_main.py b/tests/test_main.py similarity index 78% rename from test_main.py rename to tests/test_main.py index 70cf08d..d64e4d6 100644 --- a/test_main.py +++ b/tests/test_main.py @@ -1,16 +1,14 @@ from datetime import datetime from unittest.mock import ANY, call, patch -from minio.error import S3Error # type: ignore - -# from owncloud import Client as OwnCloudClient -from owncloud.owncloud import HTTPResponseError as OCResponseError # type: ignore -from pytest import mark +import pytest +from minio.error import S3Error # type: ignore[import-untyped] +from owncloud.owncloud import HTTPResponseError # type: ignore[import-untyped] import main -@mark.freeze_time("2023-01-08") +@pytest.mark.freeze_time("2023-01-08") @patch("owncloud.Client") def test_oc_check_files_exist(oc_client): oc = oc_client() @@ -24,26 +22,26 @@ def test_oc_check_files_exist(oc_client): oc.mkdir.assert_has_calls(mkdir_calls) oc.file_info.assert_has_calls( - [call(f"/tmp/test/2023/1/2023-01-0{x}.json") for x in range(1, 8)] + [call(f"/tmp/test/2023/1/2023-01-0{x}.json") for x in range(1, 8)], ) assert missing == [] -@mark.freeze_time("2023-01-08") +@pytest.mark.freeze_time("2023-01-08") @patch("owncloud.Client") def test_oc_check_files_missing(oc_client): oc = oc_client() - oc.file_info.side_effect = OCResponseError( - type("", (object,), {"status_code": 404})() + oc.file_info.side_effect = HTTPResponseError( + type("", (object,), {"status_code": 404})(), ) missing = main.oc_check(oc=oc, oc_path="/tmp/test") oc.file_info.assert_has_calls( - [call(f"/tmp/test/2023/1/2023-01-0{x}.json") for x in range(1, 8)] + [call(f"/tmp/test/2023/1/2023-01-0{x}.json") for x in range(1, 8)], ) - assert len(missing) == 7 + assert len(missing) == 7 # noqa: PLR2004 -@mark.freeze_time("2023-01-08") +@pytest.mark.freeze_time("2023-01-08") @patch("acrclient.Client") @patch("owncloud.Client") def test_oc_fetch(oc_client, acr_client): @@ -61,23 +59,24 @@ def test_oc_fetch(oc_client, acr_client): oc_path="/tmp/test", ) oc.put_file_contents.assert_called_with( - "/tmp/test/2023/1/2023-01-01.json", '"JSON"' + "/tmp/test/2023/1/2023-01-01.json", + '"JSON"', ) -@mark.freeze_time("2023-01-08") +@pytest.mark.freeze_time("2023-01-08") @patch("minio.Minio") def test_mc_check_files_exist(mc_client): mc = mc_client() missing = main.mc_check(mc=mc, bucket="acrcloud.raw") assert mc_client.called mc.stat_object.assert_has_calls( - [call("acrcloud.raw", f"2023-01-0{x}.json") for x in range(1, 8)] + [call("acrcloud.raw", f"2023-01-0{x}.json") for x in range(1, 8)], ) assert missing == [] -@mark.freeze_time("2023-01-08") +@pytest.mark.freeze_time("2023-01-08") @patch("minio.Minio") def test_mc_check_files_missing(mc_client): mc = mc_client() @@ -92,12 +91,12 @@ def test_mc_check_files_missing(mc_client): missing = main.mc_check(mc=mc, bucket="acrcloud.raw") assert mc_client.called mc.stat_object.assert_has_calls( - [call("acrcloud.raw", f"2023-01-0{x}.json") for x in range(1, 8)] + [call("acrcloud.raw", f"2023-01-0{x}.json") for x in range(1, 8)], ) - assert len(missing) == 7 + assert len(missing) == 7 # noqa: PLR2004 -@mark.freeze_time("2023-01-08") +@pytest.mark.freeze_time("2023-01-08") @patch("acrclient.Client") @patch("minio.Minio") def test_mc_fetch(mc_client, acr_client):