create multiple buckets at init

cads_broker/entry_points.py

@@ -6,7 +6,7 @@
 import random
 import uuid
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, List, Optional
 
 import sqlalchemy as sa
 import typer
@@ -245,10 +245,12 @@ def init_db(connection_string: Optional[str] = None, force: bool = False) -> Non
         "aws_access_key_id": os.environ["STORAGE_ADMIN"],
         "aws_secret_access_key": os.environ["STORAGE_PASSWORD"],
     }
-    object_storage.create_download_bucket(
-        os.environ.get("CACHE_BUCKET", "cache"), object_storage_url, **storage_kws
-    )
-    print("successfully created the cache area in the object storage.")
+    download_buckets: List[str] = object_storage.parse_data_volumes_config()
+    for download_bucket in download_buckets:
+        object_storage.create_download_bucket(
+            download_bucket, object_storage_url, **storage_kws
+        )
+    print("successfully created the cache areas in the object storage.")
 
 
 @app.command()

cads_broker/object_storage.py

@@ -1,5 +1,7 @@
 """utility module to interface to the object storage."""
 
+import os.path
+import urllib.parse
 from typing import Any
 
 import boto3  # type: ignore
@@ -9,6 +11,18 @@
 logger: structlog.stdlib.BoundLogger = structlog.get_logger(__name__)
 
 
+def parse_data_volumes_config(path: str | None = None) -> list[str]:
+    if path is None:
+        path = os.environ["DATA_VOLUMES_CONFIG"]
+
+    data_volumes = []
+    with open(path) as fp:
+        for line in fp:
+            if data_volume := os.path.expandvars(line.rstrip("\n")):
+                data_volumes.append(data_volume)
+    return data_volumes
+
+
 def is_bucket_existing(client: Any, bucket_name: str) -> bool | None:
     """Return True if the bucket exists."""
     try:
@@ -43,13 +57,18 @@ def create_download_bucket(
 
     Parameters
     ----------
-    bucket_name: name of the bucket
+    bucket_name: name of the bucket (something as 's3://mybucketname' or just 'mybucketname')
     object_storage_url: endpoint URL of the object storage
     client: client to use, default is boto3 (used for testing)
     storage_kws: dictionary of parameters used to pass to the storage client.
     """
+    bucket_url_obj = urllib.parse.urlparse(bucket_name)
+    scheme = "s3"
+    if bucket_url_obj.scheme:
+        scheme = bucket_url_obj.scheme
+        bucket_name = bucket_url_obj.netloc
     if not client:
-        client = boto3.client("s3", endpoint_url=object_storage_url, **storage_kws)
+        client = boto3.client(scheme, endpoint_url=object_storage_url, **storage_kws)
     if not is_bucket_existing(client, bucket_name):
         logger.info(f"creation of bucket {bucket_name}")
         client.create_bucket(Bucket=bucket_name)

tests/test_90_entry_points.py

@@ -1,4 +1,6 @@
 import datetime
+import os
+import unittest.mock
 import uuid
 from typing import Any
 
@@ -57,8 +59,11 @@ def mock_config(
     return adaptor_properties
 
 
-def test_init_db(postgresql: Connection[str], mocker) -> None:
+def test_init_db(postgresql: Connection[str], tmpdir, mocker) -> None:
     patch_storage = mocker.patch.object(object_storage, "create_download_bucket")
+    data_volumes_config_path = os.path.join(str(tmpdir), "data_volumes.config")
+    with open(data_volumes_config_path, "w") as fp:
+        fp.writelines(["s3://mybucket1\n", "s3://mybucket2\n"])
     connection_string = (
         f"postgresql://{postgresql.info.user}:"
         f"@{postgresql.info.host}:{postgresql.info.port}/{postgresql.info.dbname}"
@@ -80,12 +85,14 @@ def test_init_db(postgresql: Connection[str], mocker) -> None:
             "OBJECT_STORAGE_URL": object_storage_url,
             "STORAGE_ADMIN": object_storage_kws["aws_access_key_id"],
             "STORAGE_PASSWORD": object_storage_kws["aws_secret_access_key"],
+            "DATA_VOLUMES_CONFIG": data_volumes_config_path,
         },
     )
     assert result.exit_code == 0
-    patch_storage.assert_called_once_with(
-        "cache", object_storage_url, **object_storage_kws
-    )
+    assert patch_storage.mock_calls == [
+        unittest.mock.call("s3://mybucket1", object_storage_url, **object_storage_kws),
+        unittest.mock.call("s3://mybucket2", object_storage_url, **object_storage_kws),
+    ]
     assert set(conn.execute(query).scalars()) == set(
         database.BaseModel.metadata.tables
     ).union(