feat: add cloud storage commands

.gitignore

@@ -94,6 +94,7 @@ helm-chart/renku-core/charts
 renku/templates/
 temp/
 tmp/
+.ropeproject/
 
 # pytest-recording cache
 cassettes

docs/reference/commands/storage.rst

@@ -3,4 +3,4 @@
 renku storage
 *************
 
-.. automodule:: renku.ui.cli.storage
+.. automodule:: renku.ui.cli.lfs

renku/command/checks/storage.py

@@ -16,7 +16,7 @@
 """Check for large files in Git history."""
 
 from renku.command.util import WARNING
-from renku.core.storage import check_external_storage, check_lfs_migrate_info
+from renku.core.lfs import check_external_storage, check_lfs_migrate_info
 
 
 def check_lfs_info(**_):

renku/command/command_builder/command.py

@@ -17,6 +17,7 @@
 
 import contextlib
 import functools
+import shutil
 import threading
 from collections import defaultdict
 from pathlib import Path
@@ -455,6 +456,13 @@ def require_clean(self) -> "Command":
 
         return RequireClean(self)
 
+    @check_finalized
+    def require_login(self) -> "Command":
+        """Check that the user is logged in."""
+        from renku.command.command_builder.repo import RequireLogin
+
+        return RequireLogin(self)
+
     @check_finalized
     def with_communicator(self, communicator: CommunicationCallback) -> "Command":
         """Create a communicator.
@@ -479,6 +487,20 @@ def with_database(self, write: bool = False, path: Optional[str] = None, create:
 
         return DatabaseCommand(self, write, path, create)
 
+    @check_finalized
+    def with_gitlab_api(self) -> "Command":
+        """Inject gitlab api client."""
+        from renku.command.command_builder.gitlab import GitlabApiCommand
+
+        return GitlabApiCommand(self)
+
+    @check_finalized
+    def with_storage_api(self) -> "Command":
+        """Inject storage api client."""
+        from renku.command.command_builder.storage import StorageApiCommand
+
+        return StorageApiCommand(self)
+
 
 class CommandResult:
     """The result of a command.
@@ -496,3 +518,26 @@ def __init__(self, output, error, status) -> None:
         self.output = output
         self.error = error
         self.status = status
+
+
+class RequireExecutable(Command):
+    """Builder to check if an executable is installed."""
+
+    HOOK_ORDER = 4
+
+    def __init__(self, builder: Command, executable: str) -> None:
+        """__init__ of RequireExecutable."""
+        self._builder = builder
+        self._executable = executable
+
+    def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Check if an executable exists on the system.
+
+        Args:
+            builder(Command): Current ``CommandBuilder``.
+            context(dict): Current context.
+        """
+        if not shutil.which(self._executable):
+            raise errors.ExecutableNotFound(
+                f"Couldn't find the executable '{self._executable}' on this system. Please make sure it's installed"
+            )

renku/command/command_builder/gitlab.py

@@ -0,0 +1,54 @@
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command builder for gitlab api."""
+
+
+from renku.command.command_builder.command import Command, check_finalized
+from renku.core.interface.git_api_provider import IGitAPIProvider
+from renku.domain_model.project_context import project_context
+from renku.infrastructure.gitlab_api_provider import GitlabAPIProvider
+
+
+class GitlabApiCommand(Command):
+    """Builder to get a gitlab api client."""
+
+    PRE_ORDER = 4
+
+    def __init__(self, builder: Command) -> None:
+        self._builder = builder
+
+    def _injection_pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Create a gitlab api provider."""
+
+        if not project_context.has_context():
+            raise ValueError("Gitlab API builder needs a ProjectContext to be set.")
+
+        def _get_provider():
+            from renku.core.login import read_renku_token
+
+            token = read_renku_token(None, True)
+            if not token:
+                return None
+            return GitlabAPIProvider(token=token)
+
+        context["constructor_bindings"][IGitAPIProvider] = _get_provider
+
+    @check_finalized
+    def build(self) -> Command:
+        """Build the command."""
+        self._builder.add_injection_pre_hook(self.PRE_ORDER, self._injection_pre_hook)
+
+        return self._builder.build()

renku/command/command_builder/repo.py

@@ -21,6 +21,7 @@
 from renku.command.command_builder.command import Command, CommandResult, check_finalized
 from renku.core import errors
 from renku.core.git import ensure_clean
+from renku.core.login import ensure_login
 from renku.domain_model.project_context import project_context
 
 
@@ -42,6 +43,7 @@ def __init__(
         """__init__ of Commit.
 
         Args:
+            builder(Command): The current ``CommandBuilder``.
             message (str): The commit message. Auto-generated if left empty (Default value = None).
             commit_if_empty (bool): Whether to commit if there are no modified files (Default value = None).
             raise_if_empty (bool): Whether to raise an exception if there are no modified files (Default value = None).
@@ -164,6 +166,39 @@ def build(self) -> Command:
         return self._builder.build()
 
 
+class RequireLogin(Command):
+    """Builder to check if a user is logged in."""
+
+    HOOK_ORDER = 4
+
+    def __init__(self, builder: Command) -> None:
+        """__init__ of RequireLogin."""
+        self._builder = builder
+
+    def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Check if the user is logged in.
+
+        Args:
+            builder(Command): Current ``CommandBuilder``.
+            context(dict): Current context.
+        """
+        if not project_context.has_context():
+            raise ValueError("RequireLogin builder needs a ProjectContext to be set.")
+
+        ensure_login()
+
+    @check_finalized
+    def build(self) -> Command:
+        """Build the command.
+
+        Returns:
+            Command: Finalized version of this command.
+        """
+        self._builder.add_pre_hook(self.HOOK_ORDER, self._pre_hook)
+
+        return self._builder.build()
+
+
 class Isolation(Command):
     """Builder to run a command in git isolation."""
 

renku/command/command_builder/storage.py

@@ -0,0 +1,46 @@
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command builder for storage api."""
+
+
+from renku.command.command_builder.command import Command, check_finalized
+from renku.core.interface.storage_service_gateway import IStorageService
+from renku.domain_model.project_context import project_context
+from renku.infrastructure.storage.storage_service import StorageService
+
+
+class StorageApiCommand(Command):
+    """Builder to get a storage api client."""
+
+    PRE_ORDER = 4
+
+    def __init__(self, builder: Command) -> None:
+        self._builder = builder
+
+    def _injection_pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Create a storage api provider."""
+
+        if not project_context.has_context():
+            raise ValueError("storage api builder needs a ProjectContext to be set.")
+
+        context["constructor_bindings"][IStorageService] = lambda: StorageService()
+
+    @check_finalized
+    def build(self) -> Command:
+        """Build the command."""
+        self._builder.add_injection_pre_hook(self.PRE_ORDER, self._injection_pre_hook)
+
+        return self._builder.build()

renku/command/format/storage.py

@@ -0,0 +1,65 @@
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Serializers for storage."""
+
+import json
+from typing import List, Optional
+
+from renku.command.format.tabulate import tabulate
+from renku.domain_model.cloud_storage import CloudStorage
+
+
+def tabular(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
+    """Format cloud_storages with a tabular output."""
+    if not columns:
+        columns = "id,start_time,status,provider,url"
+
+        if any(s.ssh_enabled for s in cloud_storages):
+            columns += ",ssh"
+
+    return tabulate(collection=cloud_storages, columns=columns, columns_mapping=cloud_storage_COLUMNS)
+
+
+def log(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
+    """Format cloud_storages in a log like output."""
+    from renku.ui.cli.utils.terminal import style_header, style_key
+
+    output = []
+
+    for cloud_storage in cloud_storages:
+        output.append(style_header(f"CloudStorage {cloud_storage.name}"))
+        output.append(style_key("Id: ") + cloud_storage.storage_id)
+        output.append(style_key("Source Path: ") + cloud_storage.source_path)
+        output.append(style_key("Target path: ") + cloud_storage.target_path)
+        output.append(style_key("Private: ") + "Yes" if cloud_storage.private else "No")
+        output.append(style_key("Configuration: \n") + json.dumps(cloud_storage.configuration, indent=4))
+        output.append("")
+    return "\n".join(output)
+
+
+CLOUD_STORAGE_FORMATS = {"tabular": tabular, "log": log}
+"""Valid formatting options."""
+
+CLOUD_STORAGE_COLUMNS = {
+    "id": ("id", "id"),
+    "status": ("status", "status"),
+    "url": ("url", "url"),
+    "ssh": ("ssh_enabled", "SSH enabled"),
+    "start_time": ("start_time", "start_time"),
+    "commit": ("commit", "commit"),
+    "branch": ("branch", "branch"),
+    "provider": ("provider", "provider"),
+}