diff --git a/README.md b/README.md index 03fcba1..5bea1db 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # ACR Loader Loads data from ACRCloud's broadcat monitoring service and stores it -in our ownCloud instance. Runs as a cronjob and is scheduled to run -once per day. +in our ownCloud instance and/or MinIO service. Runs as a cronjob and +is scheduled to run once per day. ## Usage @@ -10,7 +10,10 @@ once per day. helm install my-acrloader oci://ghcr.io/radiorabe/helm/acrloader \ --version x.y.z \ --set acr.bearerToken=,acr.projectId=,streamId= \ - --set oc.url=,oc.user=,oc.pass=,oc.path= + --set oc.enabled=true \ + --set oc.url=,oc.user=,oc.pass=,oc.path= \ + --set minio.enabled=true \ + --set minio.url=,minio.access_key=,minio.secret_key= ``` ## Development diff --git a/charts/acrloader/templates/cm.yaml b/charts/acrloader/templates/cm.yaml index bd9c23a..966ce1b 100644 --- a/charts/acrloader/templates/cm.yaml +++ b/charts/acrloader/templates/cm.yaml @@ -7,6 +7,19 @@ metadata: data: ACR_PROJECT_ID: {{ .Values.acr.projectId | quote }} ACR_STREAM_ID: {{ .Values.acr.streamId | quote }} + {{- if .Values.oc.enabled }} + OC: "true" OC_URL: {{ .Values.oc.url | quote }} OC_USER: {{ .Values.oc.user | quote }} OC_PATH: {{ .Values.oc.path | quote }} + {{- end }} + {{- if .Values.minio.enabled }} + MINIO: "true" + MINIO_URL: {{ .Values.minio.url | quote }} + MINIO_BUCKET: {{ .Values.minio.bucket | quote }} + MINIO_SECURE: {{ .Values.minio.secure | quote }} + MINIO_CERT_REQS: {{ .Values.minio.cert_reqs | quote }} + {{- with .Values.minio.ca_certs }} + MINIO_CA_CERTS: {{ . | quote }} + {{- end }} + {{- end }} diff --git a/charts/acrloader/templates/secret.yaml b/charts/acrloader/templates/secret.yaml index 0bee6c9..2c7c7b0 100644 --- a/charts/acrloader/templates/secret.yaml +++ b/charts/acrloader/templates/secret.yaml @@ -7,5 +7,11 @@ metadata: {{- include "acrloader.labels" . | nindent 4 }} data: ACR_BEARER_TOKEN: {{ .Values.acr.bearerToken | b64enc | quote }} + {{- if .Values.oc.enabled }} OC_PASS: {{ .Values.oc.pass | b64enc | quote }} + {{- end }} + {{- if .Values.minio.enabled }} + MINIO_ACCESS_KEY: {{ .Values.minio.access_key | b64enc | quote }} + MINIO_SECRET_KEY: {{ .Values.minio.secret_key | b64enc | quote }} + {{- end }} {{- end }} diff --git a/charts/acrloader/values.yaml b/charts/acrloader/values.yaml index 9801850..c50f5ef 100644 --- a/charts/acrloader/values.yaml +++ b/charts/acrloader/values.yaml @@ -64,7 +64,18 @@ acr: streamId: "" oc: + enabled: true url: "" user: "" pass: "" path: "" + +minio: + enabled: true + url: "" + bucket: "" + access_key: "" + secret_key: "" + secure: "True" + cert_reqs: "CERT_REQUIRED" + ca_certs: "" diff --git a/main.py b/main.py index 2bd968a..4b5a715 100644 --- a/main.py +++ b/main.py @@ -1,20 +1,23 @@ """ Stores daily data from ACRCloud's broadcast monitoring service in ownCloud. """ +import json +import os from datetime import datetime, timedelta -from logging import getLogger from functools import cache -import os -import json +from io import BytesIO +from logging import getLogger +import urllib3 from acrclient import Client as ACRClient from acrclient.models import GetBmCsProjectsResultsParams from configargparse import ArgParser # type: ignore +from minio import Minio # type: ignore +from minio.error import S3Error # type: ignore from owncloud import Client as OwnCloudClient # type: ignore from owncloud.owncloud import HTTPResponseError as OCResponseError # type: ignore from tqdm import tqdm - logger = getLogger(__name__) @@ -47,7 +50,7 @@ def oc_file_exists(oc: OwnCloudClient, path: str): return False -def check(oc: OwnCloudClient, oc_path: str) -> list[datetime]: +def oc_check(oc: OwnCloudClient, oc_path: str) -> list[datetime]: """ Checks ownCloud for missing files. """ @@ -71,7 +74,42 @@ def check(oc: OwnCloudClient, oc_path: str) -> list[datetime]: return missing -def fetch( +def mc_check(mc: Minio, bucket: str) -> list[datetime]: + """ + Checks MinIO for missing files. + """ + missing = [] + start = datetime.now() - timedelta(7) + for requested in tqdm(daterange(start, datetime.now()), desc="Checking MinIO"): + try: + mc.stat_object(bucket, requested.strftime("%Y-%m-%d.json")) + except S3Error as ex: + if ex.code == "NoSuchKey": + missing.append(requested) + return missing + + +@cache +def fetch_one( + acr: ACRClient, + acr_project_id: str, + acr_stream_id: str, + requested: str, +): + return acr.get_bm_cs_projects_results( + project_id=int(acr_project_id), + stream_id=acr_stream_id, + params=GetBmCsProjectsResultsParams( + type="day", + date=requested, + min_duration=0, + max_duration=3600, + isrc_country="", + ), + ) + + +def oc_fetch( missing: list[datetime], acr: ACRClient, oc: OwnCloudClient, @@ -79,26 +117,52 @@ def fetch( acr_stream_id: str, oc_path: str, ): - """Fetches missing data from ACRCloud and stores it.""" - for requested in tqdm(missing, desc="Loading from ACRCloud"): + """Fetches missing data from ACRCloud and stores it in ownCloud.""" + for requested in tqdm(missing, desc="Loading into ownCloud from ACRCloud"): target = os.path.join( oc_path, str(requested.year), str(requested.month), requested.strftime("%Y-%m-%d.json"), ) - data = acr.get_bm_cs_projects_results( - project_id=int(acr_project_id), - stream_id=acr_stream_id, - params=GetBmCsProjectsResultsParams( - type="day", - date=requested.strftime("%Y%m%d"), - min_duration=0, - max_duration=3600, - isrc_country="", + oc.put_file_contents( + target, + json.dumps( + fetch_one( + acr=acr, + acr_project_id=acr_project_id, + acr_stream_id=acr_stream_id, + requested=requested.strftime("%Y%m%d"), + ) ), ) - oc.put_file_contents(target, json.dumps(data)) + + +def mc_fetch( + missing: list[datetime], + acr: ACRClient, + mc: Minio, + acr_project_id: str, + acr_stream_id: str, + bucket: str, +): + """Fetches missing data from ACRCloud and stores it in MinIO.""" + for requested in tqdm(missing, desc="Loading into MinIO from ACRCloud"): + _as_bytes = json.dumps( + fetch_one( + acr=acr, + acr_project_id=acr_project_id, + acr_stream_id=acr_stream_id, + requested=requested.strftime("%Y%m%d"), + ) + ).encode("utf-8") + mc.put_object( + bucket, + requested.strftime("%Y-%m-%d.json"), + BytesIO(_as_bytes), + length=len(_as_bytes), + content_type="application/json", + ) def main(): # pragma: no cover @@ -134,6 +198,13 @@ def main(): # pragma: no cover env_var="ACR_STREAM_ID", help="ACRCloud stream id", ) + p.add( + "--oc", + default=False, + action="store_true", + env_var="OC_ENABLE", + help="Enable ownCloud", + ) p.add( "--oc-url", default="https://share.rabe.ch", @@ -158,29 +229,106 @@ def main(): # pragma: no cover env_var="OC_PATH", help="ownCloud path", ) + p.add( + "--minio", + default=False, + action="store_true", + env_var="MINIO_URL", + help="MinIO URL", + ) + p.add( + "--minio-url", + default="minio.service.int.rabe.ch:9000", + env_var="MINIO_HOST", + help="MinIO Hostname", + ) + p.add( + "--minio-secure", + default=True, + env_var="MINIO_SECURE", + help="MinIO Secure param", + ) + p.add( + "--minio-cert-reqs", + default="CERT_REQUIRED", + env_var="MINIO_CERT_REQS", + help="cert_reqs for urlib3.PoolManager used by MinIO", + ) + p.add( + "--minio-ca-certs", + default="/etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt", + env_var="MINIO_CA_CERTS", + help="ca_certs for urlib3.PoolManager used by MinIO", + ) + p.add( + "--minio-bucket", + default="acrcloud.raw", + env_var="MINIO_BUCKET", + help="MinIO Bucket Name", + ) + p.add( + "--minio-access-key", + default=None, + env_var="MINIO_ACCESS_KEY", + help="MinIO Access Key", + ) + p.add( + "--minio-secret-key", + default=None, + env_var="MINIO_SECRET_KEY", + help="MinIO Secret Key", + ) options = p.parse_args() - - # figure out what we are missing on ownCloud - oc = OwnCloudClient(options.oc_url) - oc.login(options.oc_user, options.oc_pass) - missing = check(oc=oc, oc_path=options.oc_path) - - # return early if no files are missing - if not missing: - return - - # fetch and store missing data acr_client = ACRClient( bearer_token=options.acr_bearer_token, ) - fetch( - missing, - acr=acr_client, - oc=oc, - acr_project_id=options.acr_project_id, - acr_stream_id=options.acr_stream_id, - oc_path=options.oc_path, - ) + + if options.oc: + # figure out what we are missing on ownCloud + oc = OwnCloudClient(options.oc_url) + oc.login(options.oc_user, options.oc_pass) + missing = oc_check(oc=oc, oc_path=options.oc_path) + + # return early if no files are missing + if not missing: + return + + # fetch and store missing data + oc_fetch( + missing, + acr=acr_client, + oc=oc, + acr_project_id=options.acr_project_id, + acr_stream_id=options.acr_stream_id, + oc_path=options.oc_path, + ) + if options.minio: + mc = Minio( + options.minio_url, + options.minio_access_key, + options.minio_secret_key, + secure=options.minio_secure, + http_client=urllib3.PoolManager( + cert_reqs=options.minio_cert_reqs, ca_certs=options.minio_ca_certs + ), + ) + if not mc.bucket_exists(options.minio_bucket): + mc.make_bucket(options.minio_bucket) + missing = mc_check(mc=mc, bucket=options.minio_bucket) + + # return early if no files are missing + if not missing: + return + + # fetch and store missing data + mc_fetch( + missing, + acr=acr_client, + mc=mc, + acr_project_id=options.acr_project_id, + acr_stream_id=options.acr_stream_id, + bucket=options.bucket, + ) if __name__ == "__main__": # pragma: no cover diff --git a/poetry.lock b/poetry.lock index 0694d24..25a7f7d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.0 and should not be changed by hand. [[package]] name = "acrclient" version = "0.4.0" description = "API wrapper for the v2 ACRCloud API" -category = "main" optional = false python-versions = ">=3.9,<4.0" files = [ @@ -19,7 +18,6 @@ requests = ">=2.28.2" name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -38,7 +36,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "black" version = "23.3.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -86,7 +83,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -98,7 +94,6 @@ files = [ name = "cfgv" version = "3.3.1" description = "Validate configuration and produce human readable error messages." -category = "dev" optional = false python-versions = ">=3.6.1" files = [ @@ -110,7 +105,6 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -195,7 +189,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -210,7 +203,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -222,7 +214,6 @@ files = [ name = "configargparse" version = "1.5.3" description = "A drop-in replacement for argparse that allows options to also be set via config files and/or environment variables." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -236,63 +227,62 @@ yaml = ["PyYAML"] [[package]] name = "coverage" -version = "7.2.5" +version = "7.2.6" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "coverage-7.2.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:883123d0bbe1c136f76b56276074b0c79b5817dd4238097ffa64ac67257f4b6c"}, - {file = "coverage-7.2.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2fbc2a127e857d2f8898aaabcc34c37771bf78a4d5e17d3e1f5c30cd0cbc62a"}, - {file = "coverage-7.2.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f3671662dc4b422b15776cdca89c041a6349b4864a43aa2350b6b0b03bbcc7f"}, - {file = "coverage-7.2.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780551e47d62095e088f251f5db428473c26db7829884323e56d9c0c3118791a"}, - {file = "coverage-7.2.5-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:066b44897c493e0dcbc9e6a6d9f8bbb6607ef82367cf6810d387c09f0cd4fe9a"}, - {file = "coverage-7.2.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b9a4ee55174b04f6af539218f9f8083140f61a46eabcaa4234f3c2a452c4ed11"}, - {file = "coverage-7.2.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:706ec567267c96717ab9363904d846ec009a48d5f832140b6ad08aad3791b1f5"}, - {file = "coverage-7.2.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ae453f655640157d76209f42c62c64c4d4f2c7f97256d3567e3b439bd5c9b06c"}, - {file = "coverage-7.2.5-cp310-cp310-win32.whl", hash = "sha256:f81c9b4bd8aa747d417407a7f6f0b1469a43b36a85748145e144ac4e8d303cb5"}, - {file = "coverage-7.2.5-cp310-cp310-win_amd64.whl", hash = "sha256:dc945064a8783b86fcce9a0a705abd7db2117d95e340df8a4333f00be5efb64c"}, - {file = "coverage-7.2.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40cc0f91c6cde033da493227797be2826cbf8f388eaa36a0271a97a332bfd7ce"}, - {file = "coverage-7.2.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a66e055254a26c82aead7ff420d9fa8dc2da10c82679ea850d8feebf11074d88"}, - {file = "coverage-7.2.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c10fbc8a64aa0f3ed136b0b086b6b577bc64d67d5581acd7cc129af52654384e"}, - {file = "coverage-7.2.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a22cbb5ede6fade0482111fa7f01115ff04039795d7092ed0db43522431b4f2"}, - {file = "coverage-7.2.5-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:292300f76440651529b8ceec283a9370532f4ecba9ad67d120617021bb5ef139"}, - {file = "coverage-7.2.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7ff8f3fb38233035028dbc93715551d81eadc110199e14bbbfa01c5c4a43f8d8"}, - {file = "coverage-7.2.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a08c7401d0b24e8c2982f4e307124b671c6736d40d1c39e09d7a8687bddf83ed"}, - {file = "coverage-7.2.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ef9659d1cda9ce9ac9585c045aaa1e59223b143f2407db0eaee0b61a4f266fb6"}, - {file = "coverage-7.2.5-cp311-cp311-win32.whl", hash = "sha256:30dcaf05adfa69c2a7b9f7dfd9f60bc8e36b282d7ed25c308ef9e114de7fc23b"}, - {file = "coverage-7.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:97072cc90f1009386c8a5b7de9d4fc1a9f91ba5ef2146c55c1f005e7b5c5e068"}, - {file = "coverage-7.2.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bebea5f5ed41f618797ce3ffb4606c64a5de92e9c3f26d26c2e0aae292f015c1"}, - {file = "coverage-7.2.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:828189fcdda99aae0d6bf718ea766b2e715eabc1868670a0a07bf8404bf58c33"}, - {file = "coverage-7.2.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e8a95f243d01ba572341c52f89f3acb98a3b6d1d5d830efba86033dd3687ade"}, - {file = "coverage-7.2.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8834e5f17d89e05697c3c043d3e58a8b19682bf365048837383abfe39adaed5"}, - {file = "coverage-7.2.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d1f25ee9de21a39b3a8516f2c5feb8de248f17da7eead089c2e04aa097936b47"}, - {file = "coverage-7.2.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1637253b11a18f453e34013c665d8bf15904c9e3c44fbda34c643fbdc9d452cd"}, - {file = "coverage-7.2.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8e575a59315a91ccd00c7757127f6b2488c2f914096077c745c2f1ba5b8c0969"}, - {file = "coverage-7.2.5-cp37-cp37m-win32.whl", hash = "sha256:509ecd8334c380000d259dc66feb191dd0a93b21f2453faa75f7f9cdcefc0718"}, - {file = "coverage-7.2.5-cp37-cp37m-win_amd64.whl", hash = "sha256:12580845917b1e59f8a1c2ffa6af6d0908cb39220f3019e36c110c943dc875b0"}, - {file = "coverage-7.2.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b5016e331b75310610c2cf955d9f58a9749943ed5f7b8cfc0bb89c6134ab0a84"}, - {file = "coverage-7.2.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:373ea34dca98f2fdb3e5cb33d83b6d801007a8074f992b80311fc589d3e6b790"}, - {file = "coverage-7.2.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a063aad9f7b4c9f9da7b2550eae0a582ffc7623dca1c925e50c3fbde7a579771"}, - {file = "coverage-7.2.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38c0a497a000d50491055805313ed83ddba069353d102ece8aef5d11b5faf045"}, - {file = "coverage-7.2.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2b3b05e22a77bb0ae1a3125126a4e08535961c946b62f30985535ed40e26614"}, - {file = "coverage-7.2.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0342a28617e63ad15d96dca0f7ae9479a37b7d8a295f749c14f3436ea59fdcb3"}, - {file = "coverage-7.2.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf97ed82ca986e5c637ea286ba2793c85325b30f869bf64d3009ccc1a31ae3fd"}, - {file = "coverage-7.2.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c2c41c1b1866b670573657d584de413df701f482574bad7e28214a2362cb1fd1"}, - {file = "coverage-7.2.5-cp38-cp38-win32.whl", hash = "sha256:10b15394c13544fce02382360cab54e51a9e0fd1bd61ae9ce012c0d1e103c813"}, - {file = "coverage-7.2.5-cp38-cp38-win_amd64.whl", hash = "sha256:a0b273fe6dc655b110e8dc89b8ec7f1a778d78c9fd9b4bda7c384c8906072212"}, - {file = "coverage-7.2.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c587f52c81211d4530fa6857884d37f514bcf9453bdeee0ff93eaaf906a5c1b"}, - {file = "coverage-7.2.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4436cc9ba5414c2c998eaedee5343f49c02ca93b21769c5fdfa4f9d799e84200"}, - {file = "coverage-7.2.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6599bf92f33ab041e36e06d25890afbdf12078aacfe1f1d08c713906e49a3fe5"}, - {file = "coverage-7.2.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:857abe2fa6a4973f8663e039ead8d22215d31db613ace76e4a98f52ec919068e"}, - {file = "coverage-7.2.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f5cab2d7f0c12f8187a376cc6582c477d2df91d63f75341307fcdcb5d60303"}, - {file = "coverage-7.2.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:aa387bd7489f3e1787ff82068b295bcaafbf6f79c3dad3cbc82ef88ce3f48ad3"}, - {file = "coverage-7.2.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:156192e5fd3dbbcb11cd777cc469cf010a294f4c736a2b2c891c77618cb1379a"}, - {file = "coverage-7.2.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bd3b4b8175c1db502adf209d06136c000df4d245105c8839e9d0be71c94aefe1"}, - {file = "coverage-7.2.5-cp39-cp39-win32.whl", hash = "sha256:ddc5a54edb653e9e215f75de377354e2455376f416c4378e1d43b08ec50acc31"}, - {file = "coverage-7.2.5-cp39-cp39-win_amd64.whl", hash = "sha256:338aa9d9883aaaad53695cb14ccdeb36d4060485bb9388446330bef9c361c252"}, - {file = "coverage-7.2.5-pp37.pp38.pp39-none-any.whl", hash = "sha256:8877d9b437b35a85c18e3c6499b23674684bf690f5d96c1006a1ef61f9fdf0f3"}, - {file = "coverage-7.2.5.tar.gz", hash = "sha256:f99ef080288f09ffc687423b8d60978cf3a465d3f404a18d1a05474bd8575a47"}, + {file = "coverage-7.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:496b86f1fc9c81a1cd53d8842ef712e950a4611bba0c42d33366a7b91ba969ec"}, + {file = "coverage-7.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fbe6e8c0a9a7193ba10ee52977d4d5e7652957c1f56ccefed0701db8801a2a3b"}, + {file = "coverage-7.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d06b721c2550c01a60e5d3093f417168658fb454e5dfd9a23570e9bffe39a1"}, + {file = "coverage-7.2.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:77a04b84d01f0e12c66f16e69e92616442dc675bbe51b90bfb074b1e5d1c7fbd"}, + {file = "coverage-7.2.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35db06450272473eab4449e9c2ad9bc6a0a68dab8e81a0eae6b50d9c2838767e"}, + {file = "coverage-7.2.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6727a0d929ff0028b1ed8b3e7f8701670b1d7032f219110b55476bb60c390bfb"}, + {file = "coverage-7.2.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aac1d5fdc5378f6bac2c0c7ebe7635a6809f5b4376f6cf5d43243c1917a67087"}, + {file = "coverage-7.2.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1c9e4a5eb1bbc3675ee57bc31f8eea4cd7fb0cbcbe4912cf1cb2bf3b754f4a80"}, + {file = "coverage-7.2.6-cp310-cp310-win32.whl", hash = "sha256:71f739f97f5f80627f1fee2331e63261355fd1e9a9cce0016394b6707ac3f4ec"}, + {file = "coverage-7.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:fde5c7a9d9864d3e07992f66767a9817f24324f354caa3d8129735a3dc74f126"}, + {file = "coverage-7.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bc7b667f8654376e9353dd93e55e12ce2a59fb6d8e29fce40de682273425e044"}, + {file = "coverage-7.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:697f4742aa3f26c107ddcb2b1784a74fe40180014edbd9adaa574eac0529914c"}, + {file = "coverage-7.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:541280dde49ce74a4262c5e395b48ea1207e78454788887118c421cb4ffbfcac"}, + {file = "coverage-7.2.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7f1a8328eeec34c54f1d5968a708b50fc38d31e62ca8b0560e84a968fbf9a9"}, + {file = "coverage-7.2.6-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4bbd58eb5a2371bf160590f4262109f66b6043b0b991930693134cb617bc0169"}, + {file = "coverage-7.2.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ae82c5f168d2a39a5d69a12a69d4dc23837a43cf2ca99be60dfe59996ea6b113"}, + {file = "coverage-7.2.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f5440cdaf3099e7ab17a5a7065aed59aff8c8b079597b61c1f8be6f32fe60636"}, + {file = "coverage-7.2.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a6f03f87fea579d55e0b690d28f5042ec1368650466520fbc400e7aeaf09e995"}, + {file = "coverage-7.2.6-cp311-cp311-win32.whl", hash = "sha256:dc4d5187ef4d53e0d4c8eaf530233685667844c5fb0b855fea71ae659017854b"}, + {file = "coverage-7.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:c93d52c3dc7b9c65e39473704988602300e3cc1bad08b5ab5b03ca98bbbc68c1"}, + {file = "coverage-7.2.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:42c692b55a647a832025a4c048007034fe77b162b566ad537ce65ad824b12a84"}, + {file = "coverage-7.2.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7786b2fa7809bf835f830779ad285215a04da76293164bb6745796873f0942d"}, + {file = "coverage-7.2.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25bad4196104761bc26b1dae9b57383826542ec689ff0042f7f4f4dd7a815cba"}, + {file = "coverage-7.2.6-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2692306d3d4cb32d2cceed1e47cebd6b1d2565c993d6d2eda8e6e6adf53301e6"}, + {file = "coverage-7.2.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:392154d09bd4473b9d11351ab5d63391f3d5d24d752f27b3be7498b0ee2b5226"}, + {file = "coverage-7.2.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fa079995432037b5e2ef5ddbb270bcd2ded9f52b8e191a5de11fe59a00ea30d8"}, + {file = "coverage-7.2.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d712cefff15c712329113b01088ba71bbcef0f7ea58478ca0bbec63a824844cb"}, + {file = "coverage-7.2.6-cp37-cp37m-win32.whl", hash = "sha256:004948e296149644d208964300cb3d98affc5211e9e490e9979af4030b0d6473"}, + {file = "coverage-7.2.6-cp37-cp37m-win_amd64.whl", hash = "sha256:c1d7a31603c3483ac49c1726723b0934f88f2c011c660e6471e7bd735c2fa110"}, + {file = "coverage-7.2.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3436927d1794fa6763b89b60c896f9e3bd53212001026ebc9080d23f0c2733c1"}, + {file = "coverage-7.2.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44c9b9f1a245f3d0d202b1a8fa666a80b5ecbe4ad5d0859c0fb16a52d9763224"}, + {file = "coverage-7.2.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e3783a286d5a93a2921396d50ce45a909aa8f13eee964465012f110f0cbb611"}, + {file = "coverage-7.2.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cff6980fe7100242170092bb40d2b1cdad79502cd532fd26b12a2b8a5f9aee0"}, + {file = "coverage-7.2.6-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c534431153caffc7c495c3eddf7e6a6033e7f81d78385b4e41611b51e8870446"}, + {file = "coverage-7.2.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3062fd5c62df988cea9f2972c593f77fed1182bfddc5a3b12b1e606cb7aba99e"}, + {file = "coverage-7.2.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6284a2005e4f8061c58c814b1600ad0074ccb0289fe61ea709655c5969877b70"}, + {file = "coverage-7.2.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:97729e6828643f168a2a3f07848e1b1b94a366b13a9f5aba5484c2215724edc8"}, + {file = "coverage-7.2.6-cp38-cp38-win32.whl", hash = "sha256:dc11b42fa61ff1e788dd095726a0aed6aad9c03d5c5984b54cb9e1e67b276aa5"}, + {file = "coverage-7.2.6-cp38-cp38-win_amd64.whl", hash = "sha256:cbcc874f454ee51f158afd604a315f30c0e31dff1d5d5bf499fc529229d964dd"}, + {file = "coverage-7.2.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3cacc6a665221108ecdf90517a8028d07a2783df3417d12dcfef1c517e67478"}, + {file = "coverage-7.2.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:272ab31228a9df857ab5df5d67936d8861464dc89c5d3fab35132626e9369379"}, + {file = "coverage-7.2.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a8723ccec4e564d4b9a79923246f7b9a8de4ec55fa03ec4ec804459dade3c4f"}, + {file = "coverage-7.2.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5906f6a84b47f995cd1bf0aca1c72d591c55ee955f98074e93660d64dfc66eb9"}, + {file = "coverage-7.2.6-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c139b7ab3f0b15f9aad0a3fedef5a1f8c0b2bdc291d88639ca2c97d3682416"}, + {file = "coverage-7.2.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a5ffd45c6b93c23a8507e2f436983015c6457aa832496b6a095505ca2f63e8f1"}, + {file = "coverage-7.2.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4f3c7c19581d471af0e9cb49d928172cd8492cd78a2b7a4e82345d33662929bb"}, + {file = "coverage-7.2.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e8c0e79820cdd67978e1120983786422d279e07a381dbf89d03bbb23ec670a6"}, + {file = "coverage-7.2.6-cp39-cp39-win32.whl", hash = "sha256:13cde6bb0e58fb67d09e2f373de3899d1d1e866c5a9ff05d93615f2f54fbd2bb"}, + {file = "coverage-7.2.6-cp39-cp39-win_amd64.whl", hash = "sha256:6b9f64526286255735847aed0221b189486e0b9ed943446936e41b7e44b08783"}, + {file = "coverage-7.2.6-pp37.pp38.pp39-none-any.whl", hash = "sha256:6babcbf1e66e46052442f10833cfc4a0d3554d8276aa37af8531a83ed3c1a01d"}, + {file = "coverage-7.2.6.tar.gz", hash = "sha256:2025f913f2edb0272ef15d00b1f335ff8908c921c8eb2013536fcaf61f5a683d"}, ] [package.extras] @@ -302,7 +292,6 @@ toml = ["tomli"] name = "distlib" version = "0.3.6" description = "Distribution utilities" -category = "dev" optional = false python-versions = "*" files = [ @@ -314,7 +303,6 @@ files = [ name = "filelock" version = "3.12.0" description = "A platform independent file lock." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -330,7 +318,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "p name = "freezegun" version = "1.2.2" description = "Let your Python tests travel through time" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -345,7 +332,6 @@ python-dateutil = ">=2.7" name = "ghp-import" version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." -category = "dev" optional = false python-versions = "*" files = [ @@ -361,14 +347,13 @@ dev = ["flake8", "markdown", "twine", "wheel"] [[package]] name = "griffe" -version = "0.28.0" +version = "0.29.0" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "griffe-0.28.0-py3-none-any.whl", hash = "sha256:42823aafa666e38bc14d6ab3c955e5eccadd5ada61a8b780f580c8e763a34327"}, - {file = "griffe-0.28.0.tar.gz", hash = "sha256:f2253359bdf84427a46c04c28d59680195863f94cc9453af2a7728cb3d92b7d6"}, + {file = "griffe-0.29.0-py3-none-any.whl", hash = "sha256:e62ff34b04630c2382e2e277301cb2c29221fb09c04028e62ef35afccc64344b"}, + {file = "griffe-0.29.0.tar.gz", hash = "sha256:6fc892aaa251b3761e3a8d2f5893758e1850ec5d81d4605c4557be0666202a0b"}, ] [package.dependencies] @@ -378,7 +363,6 @@ colorama = ">=0.4" name = "identify" version = "2.5.24" description = "File identification library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -393,7 +377,6 @@ license = ["ukkonen"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -405,7 +388,6 @@ files = [ name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -417,7 +399,6 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -435,7 +416,6 @@ i18n = ["Babel (>=2.7)"] name = "markdown" version = "3.3.7" description = "Python implementation of Markdown." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -450,7 +430,6 @@ testing = ["coverage", "pyyaml"] name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -510,7 +489,6 @@ files = [ name = "mergedeep" version = "1.3.4" description = "A deep merge function for 🐍." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -518,11 +496,25 @@ files = [ {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, ] +[[package]] +name = "minio" +version = "7.1.15" +description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage" +optional = false +python-versions = "*" +files = [ + {file = "minio-7.1.15-py3-none-any.whl", hash = "sha256:1afdf01c1bc8b57ddd12d438e3e168d625465b56f4d1c2af7576744c688e84c6"}, + {file = "minio-7.1.15.tar.gz", hash = "sha256:fcf8ac2cef310d5ddff2bef2c42f4e5a8bb546b87bca5bf8832135db054ca4e1"}, +] + +[package.dependencies] +certifi = "*" +urllib3 = "*" + [[package]] name = "mkdocs" version = "1.4.3" description = "Project documentation with Markdown." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -550,7 +542,6 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp name = "mkdocs-autorefs" version = "0.4.1" description = "Automatically link across pages in MkDocs." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -566,7 +557,6 @@ mkdocs = ">=1.1" name = "mkdocs-gen-files" version = "0.5.0" description = "MkDocs plugin to programmatically generate documentation pages during the build" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -581,7 +571,6 @@ mkdocs = ">=1.0.3" name = "mkdocs-literate-nav" version = "0.6.0" description = "MkDocs plugin to specify the navigation in Markdown instead of YAML" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -596,7 +585,6 @@ mkdocs = ">=1.0.3" name = "mkdocs-material" version = "9.1.14" description = "Documentation that simply works" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -619,7 +607,6 @@ requests = ">=2.26" name = "mkdocs-material-extensions" version = "1.1.1" description = "Extension pack for Python Markdown and MkDocs Material." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -631,7 +618,6 @@ files = [ name = "mkdocs-section-index" version = "0.3.5" description = "MkDocs plugin to allow clickable sections that lead to an index page" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -646,7 +632,6 @@ mkdocs = ">=1.0.3" name = "mkdocstrings" version = "0.21.2" description = "Automatic documentation from sources, for MkDocs." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -670,14 +655,13 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] [[package]] name = "mkdocstrings-python" -version = "1.0.0" +version = "1.1.0" description = "A Python handler for mkdocstrings." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "mkdocstrings_python-1.0.0-py3-none-any.whl", hash = "sha256:c59d67009a7a85172f4da990d8523e95606b6a1ff93a22a2351ad3b5f8cafed1"}, - {file = "mkdocstrings_python-1.0.0.tar.gz", hash = "sha256:b89d849df990204f909d5452548b6936a185f912da06208a93909bebe25d6e67"}, + {file = "mkdocstrings_python-1.1.0-py3-none-any.whl", hash = "sha256:4e9a9d728e6ba742ecbd7379f5091f3a32b16d723881963bb457b2149656e167"}, + {file = "mkdocstrings_python-1.1.0.tar.gz", hash = "sha256:00cca5e47bf63eb2aece08b9887421b6828bdb939a13a481e4e8b495569e8101"}, ] [package.dependencies] @@ -688,7 +672,6 @@ mkdocstrings = ">=0.20" name = "mypy" version = "1.3.0" description = "Optional static typing for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -734,7 +717,6 @@ reports = ["lxml"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -746,7 +728,6 @@ files = [ name = "nodeenv" version = "1.8.0" description = "Node.js virtual environment builder" -category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -761,7 +742,6 @@ setuptools = "*" name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -773,7 +753,6 @@ files = [ name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -785,7 +764,6 @@ files = [ name = "platformdirs" version = "3.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -801,7 +779,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest- name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -817,7 +794,6 @@ testing = ["pytest", "pytest-benchmark"] name = "pre-commit" version = "3.3.2" description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -836,7 +812,6 @@ virtualenv = ">=20.10.0" name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -851,7 +826,6 @@ plugins = ["importlib-metadata"] name = "pymdown-extensions" version = "10.0.1" description = "Extension pack for Python Markdown." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -867,7 +841,6 @@ pyyaml = "*" name = "pyocclient" version = "0.6" description = "Python client library for ownCloud" -category = "main" optional = false python-versions = "*" files = [ @@ -882,7 +855,6 @@ six = "*" name = "pytest" version = "7.3.1" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -903,7 +875,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-cov" version = "4.1.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -922,7 +893,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-freezegun" version = "0.4.2" description = "Wrap tests with fixtures in freeze_time" -category = "dev" optional = false python-versions = "*" files = [ @@ -938,7 +908,6 @@ pytest = ">=3.0.0" name = "pytest-mypy" version = "0.10.3" description = "Mypy static type checker plugin for Pytest" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -956,7 +925,6 @@ pytest = {version = ">=6.2", markers = "python_version >= \"3.10\""} name = "pytest-random-order" version = "1.1.0" description = "Randomise the order in which pytest tests are run with some control over the randomness" -category = "dev" optional = false python-versions = ">=3.5.0" files = [ @@ -971,7 +939,6 @@ pytest = ">=3.0.0" name = "pytest-ruff" version = "0.0.6" description = "pytest plugin to check ruff requirements." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -986,7 +953,6 @@ ruff = "*" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1001,7 +967,6 @@ six = ">=1.5" name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1051,7 +1016,6 @@ files = [ name = "pyyaml-env-tag" version = "0.1" description = "A custom YAML tag for referencing environment variables in YAML files. " -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1066,7 +1030,6 @@ pyyaml = "*" name = "regex" version = "2023.5.5" description = "Alternative regular expression module, to replace re." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1164,7 +1127,6 @@ files = [ name = "requests" version = "2.29.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1186,7 +1148,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-mock" version = "1.10.0" description = "Mock out responses from the requests package" -category = "dev" optional = false python-versions = "*" files = [ @@ -1206,7 +1167,6 @@ test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "tes name = "ruff" version = "0.0.270" description = "An extremely fast Python linter, written in Rust." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1231,26 +1191,24 @@ files = [ [[package]] name = "setuptools" -version = "67.7.2" +version = "67.8.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, - {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, + {file = "setuptools-67.8.0-py3-none-any.whl", hash = "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f"}, + {file = "setuptools-67.8.0.tar.gz", hash = "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1262,7 +1220,6 @@ files = [ name = "tqdm" version = "4.65.0" description = "Fast, Extensible Progress Meter" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1283,7 +1240,6 @@ telegram = ["requests"] name = "types-tqdm" version = "4.65.0.1" description = "Typing stubs for tqdm" -category = "dev" optional = false python-versions = "*" files = [ @@ -1291,28 +1247,37 @@ files = [ {file = "types_tqdm-4.65.0.1-py3-none-any.whl", hash = "sha256:4894fe2b1581374ce9bca3f23d53729e4409d69b352e3d5db5829fa19482962c"}, ] +[[package]] +name = "types-urllib3" +version = "1.26.25.13" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.13.tar.gz", hash = "sha256:3300538c9dc11dad32eae4827ac313f5d986b8b21494801f1bf97a1ac6c03ae5"}, + {file = "types_urllib3-1.26.25.13-py3-none-any.whl", hash = "sha256:5dbd1d2bef14efee43f5318b5d36d805a489f6600252bb53626d4bfafd95e27c"}, +] + [[package]] name = "typing-extensions" -version = "4.5.0" +version = "4.6.2" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, + {file = "typing_extensions-4.6.2-py3-none-any.whl", hash = "sha256:3a8b36f13dd5fdc5d1b16fe317f5668545de77fa0b8e02006381fd49d731ab98"}, + {file = "typing_extensions-4.6.2.tar.gz", hash = "sha256:06006244c70ac8ee83fa8282cb188f697b8db25bc8b4df07be1873c43897060c"}, ] [[package]] name = "urllib3" -version = "1.26.15" +version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, - {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, + {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, + {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, ] [package.extras] @@ -1324,7 +1289,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "virtualenv" version = "20.23.0" description = "Virtual Python Environment builder" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1345,7 +1309,6 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess name = "watchdog" version = "3.0.0" description = "Filesystem events monitoring" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1384,4 +1347,4 @@ watchmedo = ["PyYAML (>=3.10)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "a25c56d48145387c30ea3ae64e15a9b311520c9d414d484577705cc2250ac1d5" +content-hash = "8585a99674bde2c5dd000a219b96db0fc64b94588d566ba6944e19a17c61ddad" diff --git a/pyproject.toml b/pyproject.toml index 4a7d01d..a3fc7f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ configargparse = "^1.5.3" pyocclient = "^0.6" requests = "<2.30" # https://github.com/urllib3/urllib3/issues/3010 tqdm = "^4.65.0" +minio = "^7.1.15" [tool.poetry.group.dev.dependencies] black = "^23.3.0" @@ -38,6 +39,7 @@ mkdocs-gen-files = "^0.5.0" mkdocs-literate-nav = "^0.6.0" mkdocs-section-index = "^0.3.5" pre-commit = "^3.3.2" +types-urllib3 = "^1.26.25.13" [tool.pytest.ini_options] minversion = "7.2" diff --git a/test_main.py b/test_main.py index 2e3f233..70cf08d 100644 --- a/test_main.py +++ b/test_main.py @@ -1,19 +1,20 @@ from datetime import datetime +from unittest.mock import ANY, call, patch -from unittest.mock import patch, call -from pytest import mark +from minio.error import S3Error # type: ignore # from owncloud import Client as OwnCloudClient from owncloud.owncloud import HTTPResponseError as OCResponseError # type: ignore +from pytest import mark import main @mark.freeze_time("2023-01-08") @patch("owncloud.Client") -def test_check_files_exist(oc_client): +def test_oc_check_files_exist(oc_client): oc = oc_client() - missing = main.check(oc=oc, oc_path="/tmp/test") + missing = main.oc_check(oc=oc, oc_path="/tmp/test") assert oc_client.called mkdir_calls = [ call("/tmp/test"), @@ -30,12 +31,12 @@ def test_check_files_exist(oc_client): @mark.freeze_time("2023-01-08") @patch("owncloud.Client") -def test_check_files_missing(oc_client): +def test_oc_check_files_missing(oc_client): oc = oc_client() oc.file_info.side_effect = OCResponseError( type("", (object,), {"status_code": 404})() ) - missing = main.check(oc=oc, oc_path="/tmp/test") + missing = main.oc_check(oc=oc, oc_path="/tmp/test") oc.file_info.assert_has_calls( [call(f"/tmp/test/2023/1/2023-01-0{x}.json") for x in range(1, 8)] ) @@ -45,13 +46,13 @@ def test_check_files_missing(oc_client): @mark.freeze_time("2023-01-08") @patch("acrclient.Client") @patch("owncloud.Client") -def test_fetch(oc_client, acr_client): +def test_oc_fetch(oc_client, acr_client): oc = oc_client() acr = acr_client() acr.get_bm_cs_projects_results.return_value = "JSON" - main.fetch( + main.oc_fetch( [datetime(2023, 1, 1)], acr=acr, oc=oc, @@ -62,3 +63,61 @@ def test_fetch(oc_client, acr_client): oc.put_file_contents.assert_called_with( "/tmp/test/2023/1/2023-01-01.json", '"JSON"' ) + + +@mark.freeze_time("2023-01-08") +@patch("minio.Minio") +def test_mc_check_files_exist(mc_client): + mc = mc_client() + missing = main.mc_check(mc=mc, bucket="acrcloud.raw") + assert mc_client.called + mc.stat_object.assert_has_calls( + [call("acrcloud.raw", f"2023-01-0{x}.json") for x in range(1, 8)] + ) + assert missing == [] + + +@mark.freeze_time("2023-01-08") +@patch("minio.Minio") +def test_mc_check_files_missing(mc_client): + mc = mc_client() + mc.stat_object.side_effect = S3Error( + code="NoSuchKey", + message="", + resource=None, + request_id=None, + host_id=None, + response=None, + ) + missing = main.mc_check(mc=mc, bucket="acrcloud.raw") + assert mc_client.called + mc.stat_object.assert_has_calls( + [call("acrcloud.raw", f"2023-01-0{x}.json") for x in range(1, 8)] + ) + assert len(missing) == 7 + + +@mark.freeze_time("2023-01-08") +@patch("acrclient.Client") +@patch("minio.Minio") +def test_mc_fetch(mc_client, acr_client): + mc = mc_client() + acr = acr_client() + + acr.get_bm_cs_projects_results.return_value = "JSON" + + main.mc_fetch( + [datetime(2023, 1, 1)], + acr=acr, + mc=mc, + acr_project_id=1, + acr_stream_id="asdf", + bucket="acrcloud.raw", + ) + mc.put_object.assert_called_with( + "acrcloud.raw", + "2023-01-01.json", + ANY, + length=6, + content_type="application/json", + )