🏗️ maintenance: upgrade minio

packages/pytest-simcore/src/pytest_simcore/minio_service.py

@@ -9,6 +9,8 @@
 import pytest
 import tenacity
 from minio import Minio
+from minio.datatypes import Object
+from minio.deleteobjects import DeleteError, DeleteObject
 from tenacity import Retrying
 
 from .helpers.utils_docker import get_ip, get_service_published_port
@@ -19,11 +21,18 @@
 def _ensure_remove_bucket(client: Minio, bucket_name: str):
     if client.bucket_exists(bucket_name):
         # remove content
-        objs = client.list_objects(bucket_name, prefix=None, recursive=True)
-        errors = client.remove_objects(bucket_name, [o.object_name for o in objs])
-        assert not list(errors)
+        objs: Iterator[Object] = client.list_objects(
+            bucket_name, prefix=None, recursive=True
+        )
+        errors: Iterator[DeleteError] = client.remove_objects(
+            bucket_name, [DeleteObject(o.object_name) for o in objs]
+        )
+
+        assert not any(errors), list(errors)
+
         # remove bucket
         client.remove_bucket(bucket_name)
+
     assert not client.bucket_exists(bucket_name)
 
 

packages/simcore-sdk/requirements/_test.txt

@@ -35,8 +35,6 @@ chardet==4.0.0
     #   -c requirements/_base.txt
     #   aiohttp
     #   requests
-configparser==5.0.2
-    # via minio
 coverage==5.5
     # via
     #   -r requirements/_test.in
@@ -83,10 +81,8 @@ markupsafe==1.1.1
     # via mako
 mccabe==0.6.1
     # via pylint
-minio==6.0.2
-    # via
-    #   -c requirements/../../../requirements/constraints.txt
-    #   -r requirements/_test.in
+minio==7.0.3
+    # via -r requirements/_test.in
 multidict==5.1.0
     # via
     #   -c requirements/_base.txt
@@ -148,13 +144,10 @@ python-dateutil==2.8.1
     # via
     #   alembic
     #   faker
-    #   minio
 python-dotenv==0.17.0
     # via -r requirements/_test.in
 python-editor==1.0.4
     # via alembic
-pytz==2021.1
-    # via minio
 requests==2.25.1
     # via
     #   -r requirements/_test.in

requirements/constraints.txt

@@ -25,11 +25,6 @@ httpx<0.15.0
 respx<0.13.0
 
 
-# limited by aiobotocore in storage. In addition, pytest_simcore.minio_service uses minio
-# Version 7 changes API! Has Breaking changes TODO: review ASAP
- minio<7.0.0
-
-
 # Keeps all docker compose to the same version. TODO: remove when all synced
 docker-compose==1.29.1
 

services/sidecar/requirements/_test.txt

@@ -40,8 +40,6 @@ chardet==3.0.4
     #   -c requirements/_packages.txt
     #   aiohttp
     #   requests
-configparser==5.0.2
-    # via minio
 coverage==5.5
     # via
     #   -r requirements/_test.in
@@ -89,10 +87,8 @@ markupsafe==1.1.1
     # via mako
 mccabe==0.6.1
     # via pylint
-minio==6.0.2
-    # via
-    #   -c requirements/../../../requirements/constraints.txt
-    #   -r requirements/_test.in
+minio==7.0.3
+    # via -r requirements/_test.in
 multidict==4.7.6
     # via
     #   -c requirements/_base.txt
@@ -152,15 +148,10 @@ python-dateutil==2.8.1
     # via
     #   alembic
     #   faker
-    #   minio
 python-dotenv==0.17.0
     # via -r requirements/_test.in
 python-editor==1.0.4
     # via alembic
-pytz==2020.1
-    # via
-    #   -c requirements/_base.txt
-    #   minio
 requests==2.25.1
     # via
     #   coveralls

services/storage/requirements/_base.in

@@ -1,15 +1,16 @@
+# WARNING: manually disabled urllib3 contraint to upgrade minio
 -c ../../../requirements/constraints.txt
 
 -r ../../../packages/postgres-database/requirements/_base.in
 -r ../../../packages/service-library/requirements/_base.in
 -r ../../../packages/models-library/requirements/_base.in
 
+-c ./constraints.txt
+
 # These packages introduce incompatible dependencies:
 blackfynn==4.0.0
 aiobotocore==1.0.7
 
--c ./constraints.txt
-
 
 aiohttp
 aiohttp-swagger[performance]

services/storage/requirements/_base.txt

@@ -64,9 +64,7 @@ chardet==4.0.0
 click==7.1.2
     # via -r requirements/_base.in
 configparser==5.0.2
-    # via
-    #   blackfynn
-    #   minio
+    # via blackfynn
 dataclasses==0.8
     # via pydantic
 deprecated==1.2.12
@@ -99,7 +97,12 @@ isodate==0.6.0
     #   openapi-core
     #   openapi-schema-validator
 jinja2==2.11.3
-    # via aiohttp-swagger
+    # via
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   aiohttp-swagger
 jmespath==0.10.0
     # via
     #   boto3
@@ -117,13 +120,8 @@ markupsafe==1.1.1
     # via
     #   aiohttp-swagger
     #   jinja2
-minio==6.0.2
-    # via
-    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../requirements/constraints.txt
-    #   -r requirements/_base.in
+minio==7.0.3
+    # via -r requirements/_base.in
 multidict==5.1.0
     # via
     #   aiohttp
@@ -155,11 +153,8 @@ python-dateutil==2.8.1
     # via
     #   blackfynn
     #   botocore
-    #   minio
 pytz==2021.1
-    # via
-    #   blackfynn
-    #   minio
+    # via blackfynn
 pyyaml==5.4.1
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -218,10 +213,6 @@ ujson==4.0.2
     #   aiohttp-swagger
 urllib3==1.25.11
     # via
-    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/_base.in
     #   botocore
     #   minio

services/storage/requirements/_test.txt

@@ -208,7 +208,6 @@ typing-extensions==3.7.4.3
     #   yarl
 urllib3==1.25.11
     # via
-    #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   requests
 websocket-client==0.58.0

services/storage/src/simcore_service_storage/dsm.py

@@ -588,10 +588,9 @@ async def copy_file_s3_s3(self, user_id: str, dest_uuid: str, source_uuid: str):
         to_object_name = dest_uuid
         from_bucket = self.simcore_bucket_name
         from_object_name = source_uuid
-        from_bucket_object_name = os.path.join(from_bucket, from_object_name)
         # FIXME: This is not async!
         self.s3_client.copy_object(
-            to_bucket_name, to_object_name, from_bucket_object_name
+            to_bucket_name, to_object_name, from_bucket, from_object_name
         )
 
         # update db