From cc4d7ea10d713fde5f9bdf55e1c92225e2ba4770 Mon Sep 17 00:00:00 2001 From: Ralph Verburg Date: Mon, 13 Jan 2025 15:58:45 +0100 Subject: [PATCH 1/4] Add folder argument in upload config for S3 file uploads This update introduces an optional folder argument, allowing users to specify the target location within the S3 bucket for file uploads. --- acquire/uploaders/minio.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/acquire/uploaders/minio.py b/acquire/uploaders/minio.py index 40f68255..ff0e52a9 100644 --- a/acquire/uploaders/minio.py +++ b/acquire/uploaders/minio.py @@ -20,6 +20,7 @@ def __init__(self, upload: dict[str, str], **kwargs: dict[str, Any]) -> None: self.access_id = upload.get("access_id") self.access_key = upload.get("access_key") self.bucket_name = upload.get("bucket") + self.folder = upload.get("folder", "") if not all((self.endpoint, self.access_id, self.access_key, self.bucket_name)): raise ValueError("Invalid cloud upload configuration") @@ -45,7 +46,13 @@ def prepare_client(self, paths: list[Path], proxies: Optional[dict[str, str]] = return Minio(self.endpoint, self.access_id, self.access_key, http_client=http_client) def upload_file(self, client: Any, path: Path) -> None: - client.fput_object(self.bucket_name, os.path.basename(path), path) + + if self.folder: + object_path = f"{self.folder.rstrip('/')}/{os.path.basename(path)}" + else: + object_path = os.path.basename(path) + + client.fput_object(self.bucket_name, object_path, path) def finish(self, client: Any) -> None: pass From ae64538dbfc5a6ac3b6ad46ad147c957d0c627ef Mon Sep 17 00:00:00 2001 From: raverburg Date: Tue, 14 Jan 2025 13:55:02 +0000 Subject: [PATCH 2/4] commit suggestions and created tests for test_minio_uploader --- acquire/uploaders/minio.py | 7 +++--- tests/test_minio_uploader.py | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/acquire/uploaders/minio.py b/acquire/uploaders/minio.py index ff0e52a9..7291163a 100644 --- a/acquire/uploaders/minio.py +++ b/acquire/uploaders/minio.py @@ -20,7 +20,7 @@ def __init__(self, upload: dict[str, str], **kwargs: dict[str, Any]) -> None: self.access_id = upload.get("access_id") self.access_key = upload.get("access_key") self.bucket_name = upload.get("bucket") - self.folder = upload.get("folder", "") + self.folder = upload.get("folder", "").rstrip("/") if not all((self.endpoint, self.access_id, self.access_key, self.bucket_name)): raise ValueError("Invalid cloud upload configuration") @@ -47,10 +47,9 @@ def prepare_client(self, paths: list[Path], proxies: Optional[dict[str, str]] = def upload_file(self, client: Any, path: Path) -> None: + object_path = path.name if self.folder: - object_path = f"{self.folder.rstrip('/')}/{os.path.basename(path)}" - else: - object_path = os.path.basename(path) + object_path = f"{self.folder}/{object_path}" client.fput_object(self.bucket_name, object_path, path) diff --git a/tests/test_minio_uploader.py b/tests/test_minio_uploader.py index 001a8fed..c27305e1 100644 --- a/tests/test_minio_uploader.py +++ b/tests/test_minio_uploader.py @@ -79,3 +79,46 @@ def test_upload_file_multiple_failures(minio_instance: MinIO): with patch("acquire.uploaders.plugin.log") as mocked_logger: upload_files_using_uploader(minio_instance, [Path("hello")]) mocked_logger.error.assert_called_with("Upload %s FAILED after too many attempts. Stopping.", Path("hello")) + +def test_minio_folder_initialization(minio_plugin): + arguments = { + "endpoint": "test", + "access_id": "test", + "access_key": "test", + "bucket": "test", + "folder": "Uploads/" + } + minio = minio_plugin(upload=arguments) + assert minio.folder == "Uploads" + + minio.folder = "Uploads/test_folder" + assert minio.folder == "Uploads/test_folder" + + arguments.pop("folder") + minio_no_folder = minio_plugin(upload=arguments) + assert minio_no_folder.folder == "" + + +def test_upload_file_with_folder(minio_instance: MinIO): + mock_client = Mock() + test_path = Path("example.txt") + minio_instance.folder = 'test_folder' + minio_instance.upload_file(mock_client, test_path) + + mock_client.fput_object.assert_called_once_with( + "test", "test_folder/example.txt", test_path + ) + + +def test_upload_file_without_folder(minio_instance: MinIO): + mock_client = Mock() + minio_instance.folder = "" + mock_client = Mock() + test_path = Path("example.txt") + minio_instance.upload_file(mock_client, test_path) + + mock_client.fput_object.assert_called_once_with( + "test", "example.txt", test_path + ) + + From d36f4b74727289cecae748e177227c7736c1424a Mon Sep 17 00:00:00 2001 From: raverburg Date: Tue, 14 Jan 2025 14:56:54 +0000 Subject: [PATCH 3/4] Applied suggestions in minio uploader tests and reformatted with tox --- acquire/uploaders/minio.py | 1 - tests/test_minio_uploader.py | 32 +++++++++++--------------------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/acquire/uploaders/minio.py b/acquire/uploaders/minio.py index 7291163a..b22bef66 100644 --- a/acquire/uploaders/minio.py +++ b/acquire/uploaders/minio.py @@ -46,7 +46,6 @@ def prepare_client(self, paths: list[Path], proxies: Optional[dict[str, str]] = return Minio(self.endpoint, self.access_id, self.access_key, http_client=http_client) def upload_file(self, client: Any, path: Path) -> None: - object_path = path.name if self.folder: object_path = f"{self.folder}/{object_path}" diff --git a/tests/test_minio_uploader.py b/tests/test_minio_uploader.py index c27305e1..b804b505 100644 --- a/tests/test_minio_uploader.py +++ b/tests/test_minio_uploader.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Callable from unittest.mock import Mock, patch import pytest @@ -80,19 +81,15 @@ def test_upload_file_multiple_failures(minio_instance: MinIO): upload_files_using_uploader(minio_instance, [Path("hello")]) mocked_logger.error.assert_called_with("Upload %s FAILED after too many attempts. Stopping.", Path("hello")) -def test_minio_folder_initialization(minio_plugin): - arguments = { - "endpoint": "test", - "access_id": "test", - "access_key": "test", - "bucket": "test", - "folder": "Uploads/" - } + +def test_minio_folder_initialization(minio_plugin: Callable) -> None: + arguments = {"endpoint": "test", "access_id": "test", "access_key": "test", "bucket": "test", "folder": "Uploads/"} minio = minio_plugin(upload=arguments) assert minio.folder == "Uploads" - minio.folder = "Uploads/test_folder" - assert minio.folder == "Uploads/test_folder" + arguments["folder"] = "Uploads/test_folder" + minio_no_backslash = minio_plugin(upload=arguments) + assert minio_no_backslash.folder == "Uploads/test_folder" arguments.pop("folder") minio_no_folder = minio_plugin(upload=arguments) @@ -102,23 +99,16 @@ def test_minio_folder_initialization(minio_plugin): def test_upload_file_with_folder(minio_instance: MinIO): mock_client = Mock() test_path = Path("example.txt") - minio_instance.folder = 'test_folder' + minio_instance.folder = "test_folder" minio_instance.upload_file(mock_client, test_path) - mock_client.fput_object.assert_called_once_with( - "test", "test_folder/example.txt", test_path - ) + mock_client.fput_object.assert_called_once_with("test", "test_folder/example.txt", test_path) def test_upload_file_without_folder(minio_instance: MinIO): - mock_client = Mock() - minio_instance.folder = "" mock_client = Mock() test_path = Path("example.txt") + minio_instance.folder = "" minio_instance.upload_file(mock_client, test_path) - mock_client.fput_object.assert_called_once_with( - "test", "example.txt", test_path - ) - - + mock_client.fput_object.assert_called_once_with("test", "example.txt", test_path) From 06eb6c9ea21666f61f5f10f441cb916dd42bcef3 Mon Sep 17 00:00:00 2001 From: Ralph Verburg Date: Tue, 14 Jan 2025 17:19:11 +0100 Subject: [PATCH 4/4] Removed unused import in minio uploader --- acquire/uploaders/minio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/acquire/uploaders/minio.py b/acquire/uploaders/minio.py index b22bef66..cbdf81ad 100644 --- a/acquire/uploaders/minio.py +++ b/acquire/uploaders/minio.py @@ -1,4 +1,3 @@ -import os from pathlib import Path from typing import Any, Optional