implement storage list and add rudimentary test

renku/command/format/storage.py

@@ -25,12 +25,8 @@
 def tabular(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
     """Format cloud_storages with a tabular output."""
     if not columns:
-        columns = "id,start_time,status,provider,url"
-
-        if any(s.ssh_enabled for s in cloud_storages):
-            columns += ",ssh"
-
-    return tabulate(collection=cloud_storages, columns=columns, columns_mapping=cloud_storage_COLUMNS)
+        columns = "id,name,private,type"
+    return tabulate(collection=cloud_storages, columns=columns, columns_mapping=CLOUD_STORAGE_COLUMNS)
 
 
 def log(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
@@ -41,7 +37,7 @@ def log(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
 
     for cloud_storage in cloud_storages:
         output.append(style_header(f"CloudStorage {cloud_storage.name}"))
-        output.append(style_key("Id: ") + cloud_storage.storage_id)
+        output.append(style_key("Id: ") + cloud_storage.storage_id)  # type: ignore
         output.append(style_key("Source Path: ") + cloud_storage.source_path)
         output.append(style_key("Target path: ") + cloud_storage.target_path)
         output.append(style_key("Private: ") + "Yes" if cloud_storage.private else "No")
@@ -54,12 +50,10 @@ def log(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
 """Valid formatting options."""
 
 CLOUD_STORAGE_COLUMNS = {
-    "id": ("id", "id"),
-    "status": ("status", "status"),
-    "url": ("url", "url"),
-    "ssh": ("ssh_enabled", "SSH enabled"),
-    "start_time": ("start_time", "start_time"),
-    "commit": ("commit", "commit"),
-    "branch": ("branch", "branch"),
-    "provider": ("provider", "provider"),
+    "id": ("storage_id", "id"),
+    "name": ("name", "name"),
+    "source_path": ("source_path", "source path"),
+    "target_path": ("target_path", "target path"),
+    "private": ("private", "private"),
+    "type": ("storage_type", "type"),
 }

renku/command/storage.py

@@ -13,11 +13,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""Cloud storage commands."""
 from renku.command.command_builder.command import Command
 
 
 def list_storage_command():
     """Command to list configured cloud storage."""
     from renku.core.storage import list_storage
 
-    return Command().command(list_storage).require_login().with_database()
+    return Command().command(list_storage).with_database().require_login().with_gitlab_api().with_storage_api()

renku/core/interface/git_api_provider.py

@@ -23,6 +23,10 @@
 class IGitAPIProvider(ABC):
     """Interface a Git API Provider."""
 
+    def __init__(self, token: str):
+        """Initialize class."""
+        raise NotImplementedError()
+
     def download_files_from_api(
         self,
         files: List[Union[Path, str]],
@@ -34,6 +38,6 @@ def download_files_from_api(
         """Download files through a remote Git API."""
         raise NotImplementedError()
 
-    def get_project_id(self, gitlab_url: str, namespace: str, name: str) -> str:
+    def get_project_id(self, gitlab_url: str, namespace: str, name: str) -> Optional[str]:
         """Get a gitlab project id from namespace/name."""
         raise NotImplementedError()

renku/core/interface/storage_service_gateway.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 """Interface for a remote storage service."""
 
-from typing import List, Protocol
+from typing import List, Protocol, Optional
 
 from renku.domain_model.cloud_storage import CloudStorage, CloudStorageWithSensitiveFields
 
@@ -24,7 +24,7 @@ class IStorageService(Protocol):
     """Interface for a storage service."""
 
     @property
-    def project_id(self) -> str:
+    def project_id(self) -> Optional[str]:
         """Get the current gitlab project id.
 
         Note: This is mostly a workaround since storage service is already done to only accept

renku/core/session/renkulab.py

@@ -21,7 +21,7 @@
 from datetime import datetime
 from pathlib import Path
 from time import monotonic, sleep
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
 
 from renku.command.command_builder.command import inject
 from renku.core import errors
@@ -265,8 +265,12 @@ def find_image(self, image_name: str, config: Optional[Dict[str, Any]]) -> bool:
 
     def get_cloudstorage(self):
         """Get cloudstorage configured for the project."""
-        storage_service: IStorageService = inject.instance(IStorageService)
-        storages = storage_service.list(storage_service.project_id)
+        storage_service = cast(IStorageService, inject.instance(IStorageService))
+        project_id = storage_service.project_id
+        if not project_id:
+            communication.warn("Couldn't get project ID from Gitlab, skipping mounting cloudstorage")
+
+        storages = storage_service.list(project_id)
 
         if not storages:
             return []
@@ -283,7 +287,7 @@ def get_cloudstorage(self):
                         continue
                     field = next(f for f in private_fields if f["name"] == name)
 
-                    secret = communication.prompt(f"{field['help']}\nPlease provide a value for secret '{name}':")
+                    secret = communication.prompt(f"{field['help']}\nPlease provide a value for secret '{name}'")
                     storage.configuration[name] = secret
 
             storages_to_mount.append({"storage_id": storage.storage_id, "configuration": storage.configuration})

renku/core/storage.py

@@ -26,5 +26,5 @@
 def list_storage(storage_service: IStorageService):
     """List configured cloud storage for project."""
     project_id = storage_service.project_id
-    storages = storage_service.ls(project_id)
+    storages = storage_service.list(project_id)
     return storages

renku/infrastructure/gitlab_api_provider.py

@@ -70,6 +70,10 @@ def download_files_from_api(
         target_folder = Path(target_folder)
 
         git_data = GitURL.parse(remote)
+
+        if git_data.name is None:
+            raise errors.InvalidGitURL("Couldn't parse repo name from git url")
+
         project = self._get_project(git_data.instance_url, git_data.owner, git_data.name)
 
         for file in files:

renku/infrastructure/storage/storage_service.py

@@ -47,7 +47,7 @@ def __init__(self):
         self.base_url = f"{renku_url}api/data"
 
     @cached_property
-    def project_id(self) -> str:
+    def project_id(self) -> Optional[str]:
         """Get the current gitlab project id.
 
         Note: This is mostly a workaround since storage service is already done to only accept

renku/ui/cli/storage.py

@@ -56,9 +56,11 @@ def ls(columns, format):
     """List configured cloud storage for a project."""
     from renku.command.storage import list_storage_command
 
-    storages = list_storage_command().build().execute()
+    result = list_storage_command().build().execute()
 
-    click.echo(STORAGE_FORMATS[format](storages.output, columns=columns))
+    storages = [s.storage for s in result.output]
+
+    click.echo(CLOUD_STORAGE_FORMATS[format](storages, columns=columns))
 
 
 # =============================================

renku/ui/service/controllers/cache_migrations_check.py

@@ -23,10 +23,10 @@
 
 from renku.command.migrate import MigrationCheckResult, migrations_check
 from renku.core.errors import AuthenticationError, MinimumVersionError, ProjectNotFound, RenkuException
+from renku.core.interface.git_api_provider import IGitAPIProvider
 from renku.core.util.contexts import renku_project_context
 from renku.ui.service.controllers.api.abstract import ServiceCtrl
 from renku.ui.service.controllers.api.mixins import RenkuOperationMixin
-from renku.ui.service.interfaces.git_api_provider import IGitAPIProvider
 from renku.ui.service.logger import service_log
 from renku.ui.service.serializers.cache import ProjectMigrationCheckRequest, ProjectMigrationCheckResponseRPC
 from renku.ui.service.views import result_response
@@ -70,7 +70,6 @@ def _fast_op_without_cache(self):
                 target_folder=tempdir_path,
                 remote=self.ctx["git_url"],
                 branch=self.request_data.get("branch", None),
-                token=self.user.token,
             )
             with renku_project_context(tempdir_path):
                 self.project_path = tempdir_path

renku/ui/service/views/v1/cache.py

@@ -93,7 +93,7 @@ def migration_check_project_view_1_5(user_data, cache):
     from renku.ui.service.serializers.rpc import JsonRPCResponse
     from renku.ui.service.views.error_handlers import pretty_print_error
 
-    ctrl = MigrationsCheckCtrl(cache, user_data, dict(request.args), GitlabAPIProvider())
+    ctrl = MigrationsCheckCtrl(cache, user_data, dict(request.args), GitlabAPIProvider)
 
     if "project_id" in ctrl.context:  # type: ignore
         result = ctrl.execute_op()

tests/core/commands/test_storage.py → tests/core/commands/test_lfs.py

@@ -1,5 +1,4 @@
-#
-# Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+# Copyright Swiss Data Science Center (SDSC)
 # A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
 # Eidgenössische Technische Hochschule Zürich (ETHZ).
 #
@@ -14,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Storage command tests."""
+"""LFS command tests."""
 
 import os
 import subprocess

tests/core/test_storage.py

@@ -0,0 +1,56 @@
+# Copyright Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for storage service."""
+
+from unittest.mock import MagicMock
+
+import renku.infrastructure.storage.storage_service as storage_service
+from renku.command.command_builder.command import inject, remove_injector
+from renku.core.interface.git_api_provider import IGitAPIProvider
+
+
+def test_storage_service_list(monkeypatch):
+    """Test listing storage."""
+    inject.configure(lambda binder: binder.bind(IGitAPIProvider, MagicMock()), bind_in_runtime=False)
+
+    try:
+        with monkeypatch.context() as monkey:
+
+            def _send_request(*_, **__):
+                return [
+                    {
+                        "storage": {
+                            "storage_id": "ABCDEFG",
+                            "name": "mystorage",
+                            "source_path": "source/path",
+                            "target_path": "target/path",
+                            "private": True,
+                            "configuration": {"type": "s3", "endpoint": "example.com"},
+                        },
+                        "sensitive_fields": {},
+                    }
+                ]
+
+            monkey.setattr(storage_service.StorageService, "_send_request", _send_request)
+            monkey.setattr(storage_service, "get_renku_url", lambda: "http://example.com")
+            svc = storage_service.StorageService()
+            storages = svc.list("123456")
+            assert len(storages) == 1
+            assert storages[0].storage.name == "mystorage"
+            assert storages[0].storage.storage_type == "s3"
+
+    finally:
+        remove_injector()