Add optional folder argument for file uploads to S3 Buckets (#208)

This update introduces an optional folder argument, allowing users to specify the target location within the S3 bucket for file uploads

acquire/uploaders/minio.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 from typing import Any, Optional
 
@@ -20,6 +19,7 @@ def __init__(self, upload: dict[str, str], **kwargs: dict[str, Any]) -> None:
         self.access_id = upload.get("access_id")
         self.access_key = upload.get("access_key")
         self.bucket_name = upload.get("bucket")
+        self.folder = upload.get("folder", "").rstrip("/")
 
         if not all((self.endpoint, self.access_id, self.access_key, self.bucket_name)):
             raise ValueError("Invalid cloud upload configuration")
@@ -45,7 +45,11 @@ def prepare_client(self, paths: list[Path], proxies: Optional[dict[str, str]] =
         return Minio(self.endpoint, self.access_id, self.access_key, http_client=http_client)
 
     def upload_file(self, client: Any, path: Path) -> None:
-        client.fput_object(self.bucket_name, os.path.basename(path), path)
+        object_path = path.name
+        if self.folder:
+            object_path = f"{self.folder}/{object_path}"
+
+        client.fput_object(self.bucket_name, object_path, path)
 
     def finish(self, client: Any) -> None:
         pass

tests/test_minio_uploader.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Callable
 from unittest.mock import Mock, patch
 
 import pytest
@@ -79,3 +80,35 @@ def test_upload_file_multiple_failures(minio_instance: MinIO):
     with patch("acquire.uploaders.plugin.log") as mocked_logger:
         upload_files_using_uploader(minio_instance, [Path("hello")])
         mocked_logger.error.assert_called_with("Upload %s FAILED after too many attempts. Stopping.", Path("hello"))
+
+
+def test_minio_folder_initialization(minio_plugin: Callable) -> None:
+    arguments = {"endpoint": "test", "access_id": "test", "access_key": "test", "bucket": "test", "folder": "Uploads/"}
+    minio = minio_plugin(upload=arguments)
+    assert minio.folder == "Uploads"
+
+    arguments["folder"] = "Uploads/test_folder"
+    minio_no_backslash = minio_plugin(upload=arguments)
+    assert minio_no_backslash.folder == "Uploads/test_folder"
+
+    arguments.pop("folder")
+    minio_no_folder = minio_plugin(upload=arguments)
+    assert minio_no_folder.folder == ""
+
+
+def test_upload_file_with_folder(minio_instance: MinIO):
+    mock_client = Mock()
+    test_path = Path("example.txt")
+    minio_instance.folder = "test_folder"
+    minio_instance.upload_file(mock_client, test_path)
+
+    mock_client.fput_object.assert_called_once_with("test", "test_folder/example.txt", test_path)
+
+
+def test_upload_file_without_folder(minio_instance: MinIO):
+    mock_client = Mock()
+    test_path = Path("example.txt")
+    minio_instance.folder = ""
+    minio_instance.upload_file(mock_client, test_path)
+
+    mock_client.fput_object.assert_called_once_with("test", "example.txt", test_path)