feat: add cloud storage support for session start

.pre-commit-config.yaml

@@ -43,6 +43,7 @@ repos:
           - --ignore=D105,D107,D202,D203,D212,D213,D401,D406,D407,D410,D411,D413
         additional_dependencies:
           - toml
+        exclude: ^renku/ui/service/views/
   - repo: https://github.com/koalaman/shellcheck-precommit
     rev: v0.8.0
     hooks:

docs/_static/cheatsheet/cheatsheet.json

@@ -441,7 +441,7 @@
           ]
         },
         {
-          "command": "$ renku storage pull <path>...",
+          "command": "$ renku lfs pull <path>...",
           "description": "Pull <path>'s from external storage (LFS).",
           "target": [
             "rp"

docs/cheatsheet_hash

@@ -1,2 +1,2 @@
-5316163d742bdb6792ed8bcb35031f6c  cheatsheet.tex
+b8a4fc75c7ba023773b0ccc2e98ebc02  cheatsheet.tex
 c70c179e07f04186ec05497564165f11  sdsc_cheatsheet.cls

docs/cheatsheet_json_hash

@@ -1 +1 @@
-1856fb451165d013777c7c4cdd56e575  cheatsheet.json
+171f230e9ec6372e52129df1bfcf485a  cheatsheet.json

docs/reference/commands/storage.rst

@@ -3,4 +3,4 @@
 renku storage
 *************
 
-.. automodule:: renku.ui.cli.storage
+.. automodule:: renku.ui.cli.lfs

pyproject.toml

@@ -241,7 +241,7 @@ pattern = """(?x)                                          (?# ignore whitespace
 """
 
 [tool.pytest.ini_options]
-addopts = "--doctest-glob=\"*.rst\" --doctest-modules --cov --cov-report=term-missing --ignore=docs/cheatsheet/ --tb=line"
+addopts = "--doctest-glob=\"*.rst\" --doctest-modules --cov --cov-report=term-missing --ignore=docs/cheatsheet/ -ra"
 doctest_optionflags = "ALLOW_UNICODE"
 flake8-ignore = ["*.py", "E121", "E126", "E203", "E226", "E231", "W503", "W504", "docs/conf.py", "docs/cheatsheet/conf.py", "ALL"]
 flake8-max-line-length = 120

renku/command/checks/storage.py

@@ -16,7 +16,7 @@
 """Check for large files in Git history."""
 
 from renku.command.util import WARNING
-from renku.core.storage import check_external_storage, check_lfs_migrate_info
+from renku.core.lfs import check_external_storage, check_lfs_migrate_info
 
 
 def check_lfs_info(**_):

renku/command/command_builder/command.py

@@ -17,6 +17,7 @@
 
 import contextlib
 import functools
+import shutil
 import threading
 from collections import defaultdict
 from pathlib import Path
@@ -455,6 +456,13 @@ def require_clean(self) -> "Command":
 
         return RequireClean(self)
 
+    @check_finalized
+    def require_login(self) -> "Command":
+        """Check that the user is logged in."""
+        from renku.command.command_builder.repo import RequireLogin
+
+        return RequireLogin(self)
+
     @check_finalized
     def with_communicator(self, communicator: CommunicationCallback) -> "Command":
         """Create a communicator.
@@ -479,6 +487,20 @@ def with_database(self, write: bool = False, path: Optional[str] = None, create:
 
         return DatabaseCommand(self, write, path, create)
 
+    @check_finalized
+    def with_gitlab_api(self) -> "Command":
+        """Inject gitlab api client."""
+        from renku.command.command_builder.gitlab import GitlabApiCommand
+
+        return GitlabApiCommand(self)
+
+    @check_finalized
+    def with_storage_api(self) -> "Command":
+        """Inject storage api client."""
+        from renku.command.command_builder.storage import StorageApiCommand
+
+        return StorageApiCommand(self)
+
 
 class CommandResult:
     """The result of a command.
@@ -496,3 +518,26 @@ def __init__(self, output, error, status) -> None:
         self.output = output
         self.error = error
         self.status = status
+
+
+class RequireExecutable(Command):
+    """Builder to check if an executable is installed."""
+
+    HOOK_ORDER = 4
+
+    def __init__(self, builder: Command, executable: str) -> None:
+        """__init__ of RequireExecutable."""
+        self._builder = builder
+        self._executable = executable
+
+    def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Check if an executable exists on the system.
+
+        Args:
+            builder(Command): Current ``CommandBuilder``.
+            context(dict): Current context.
+        """
+        if not shutil.which(self._executable):
+            raise errors.ExecutableNotFound(
+                f"Couldn't find the executable '{self._executable}' on this system. Please make sure it's installed"
+            )

renku/command/command_builder/gitlab.py

@@ -0,0 +1,54 @@
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command builder for gitlab api."""
+
+
+from renku.command.command_builder.command import Command, check_finalized
+from renku.core.interface.git_api_provider import IGitAPIProvider
+from renku.domain_model.project_context import project_context
+from renku.infrastructure.gitlab_api_provider import GitlabAPIProvider
+
+
+class GitlabApiCommand(Command):
+    """Builder to get a gitlab api client."""
+
+    PRE_ORDER = 4
+
+    def __init__(self, builder: Command) -> None:
+        self._builder = builder
+
+    def _injection_pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Create a gitlab api provider."""
+
+        if not project_context.has_context():
+            raise ValueError("Gitlab API builder needs a ProjectContext to be set.")
+
+        def _get_provider():
+            from renku.core.login import read_renku_token
+
+            token = read_renku_token(None, True)
+            if not token:
+                return None
+            return GitlabAPIProvider(token=token)
+
+        context["constructor_bindings"][IGitAPIProvider] = _get_provider
+
+    @check_finalized
+    def build(self) -> Command:
+        """Build the command."""
+        self._builder.add_injection_pre_hook(self.PRE_ORDER, self._injection_pre_hook)
+
+        return self._builder.build()

renku/command/command_builder/repo.py

@@ -21,6 +21,7 @@
 from renku.command.command_builder.command import Command, CommandResult, check_finalized
 from renku.core import errors
 from renku.core.git import ensure_clean
+from renku.core.login import ensure_login
 from renku.domain_model.project_context import project_context
 
 
@@ -42,6 +43,7 @@ def __init__(
         """__init__ of Commit.
 
         Args:
+            builder(Command): The current ``CommandBuilder``.
             message (str): The commit message. Auto-generated if left empty (Default value = None).
             commit_if_empty (bool): Whether to commit if there are no modified files (Default value = None).
             raise_if_empty (bool): Whether to raise an exception if there are no modified files (Default value = None).
@@ -164,6 +166,39 @@ def build(self) -> Command:
         return self._builder.build()
 
 
+class RequireLogin(Command):
+    """Builder to check if a user is logged in."""
+
+    HOOK_ORDER = 4
+
+    def __init__(self, builder: Command) -> None:
+        """__init__ of RequireLogin."""
+        self._builder = builder
+
+    def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Check if the user is logged in.
+
+        Args:
+            builder(Command): Current ``CommandBuilder``.
+            context(dict): Current context.
+        """
+        if not project_context.has_context():
+            raise ValueError("RequireLogin builder needs a ProjectContext to be set.")
+
+        ensure_login()
+
+    @check_finalized
+    def build(self) -> Command:
+        """Build the command.
+
+        Returns:
+            Command: Finalized version of this command.
+        """
+        self._builder.add_pre_hook(self.HOOK_ORDER, self._pre_hook)
+
+        return self._builder.build()
+
+
 class Isolation(Command):
     """Builder to run a command in git isolation."""
 

renku/command/command_builder/storage.py

@@ -0,0 +1,46 @@
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command builder for storage api."""
+
+
+from renku.command.command_builder.command import Command, check_finalized
+from renku.core.interface.storage_service_gateway import IStorageService
+from renku.domain_model.project_context import project_context
+from renku.infrastructure.storage.storage_service import StorageService
+
+
+class StorageApiCommand(Command):
+    """Builder to get a storage api client."""
+
+    PRE_ORDER = 4
+
+    def __init__(self, builder: Command) -> None:
+        self._builder = builder
+
+    def _injection_pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Create a storage api provider."""
+
+        if not project_context.has_context():
+            raise ValueError("storage api builder needs a ProjectContext to be set.")
+
+        context["constructor_bindings"][IStorageService] = lambda: StorageService()
+
+    @check_finalized
+    def build(self) -> Command:
+        """Build the command."""
+        self._builder.add_injection_pre_hook(self.PRE_ORDER, self._injection_pre_hook)
+
+        return self._builder.build()

renku/command/format/storage.py

@@ -0,0 +1,59 @@
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Serializers for storage."""
+
+import json
+from typing import List, Optional
+
+from renku.command.format.tabulate import tabulate
+from renku.domain_model.cloud_storage import CloudStorage
+
+
+def tabular(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
+    """Format cloud_storages with a tabular output."""
+    if not columns:
+        columns = "id,name,private,type"
+    return tabulate(collection=cloud_storages, columns=columns, columns_mapping=CLOUD_STORAGE_COLUMNS)
+
+
+def log(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
+    """Format cloud_storages in a log like output."""
+    from renku.ui.cli.utils.terminal import style_header, style_key
+
+    output = []
+
+    for cloud_storage in cloud_storages:
+        output.append(style_header(f"CloudStorage {cloud_storage.name}"))
+        output.append(style_key("Id: ") + cloud_storage.storage_id)  # type: ignore
+        output.append(style_key("Source Path: ") + cloud_storage.source_path)
+        output.append(style_key("Target path: ") + cloud_storage.target_path)
+        output.append(style_key("Private: ") + ("Yes" if cloud_storage.private else "No"))
+        output.append(style_key("Configuration: \n") + json.dumps(cloud_storage.configuration, indent=4))
+        output.append("")
+    return "\n".join(output)
+
+
+CLOUD_STORAGE_FORMATS = {"tabular": tabular, "log": log}
+"""Valid formatting options."""
+
+CLOUD_STORAGE_COLUMNS = {
+    "id": ("storage_id", "id"),
+    "name": ("name", "name"),
+    "source_path": ("source_path", "source path"),
+    "target_path": ("target_path", "target path"),
+    "private": ("private", "private"),
+    "type": ("storage_type", "type"),
+}