Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add cloud storage support for session start #3629

Merged
merged 28 commits into from
Jan 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
63ed138
feat: add cloud storage commands
Sep 13, 2023
90e0b5f
chore: Update cheatsheet pdf
RenkuBot Sep 27, 2023
64ca6b2
chore: Update cheatsheet json
RenkuBot Sep 27, 2023
bcb57ba
implement storage list and add rudimentary test
Panaetius Sep 28, 2023
2e36541
Merge branch 'develop' into feat/rclone-storage-commands
Panaetius Sep 28, 2023
b4f137b
fix tests
Panaetius Sep 28, 2023
d5afc61
Merge branch 'develop' into feat/rclone-storage-commands
Panaetius Nov 8, 2023
770dc17
chore: Update cheatsheet pdf
RenkuBot Nov 8, 2023
b4f2f0a
chore: Update cheatsheet json
RenkuBot Nov 8, 2023
99c1895
Merge branch 'develop' into feat/rclone-storage-commands
Panaetius Nov 8, 2023
23553b6
fix errors
Panaetius Nov 8, 2023
9287614
change toil test to log to file
Nov 9, 2023
fa9f513
comment out failing test line
Nov 9, 2023
3675827
Merge branch 'develop' into feat/rclone-storage-commands
Panaetius Nov 10, 2023
322277e
Merge branch 'develop' into feat/rclone-storage-commands
Panaetius Nov 13, 2023
05a90c5
Merge branch 'develop' into feat/rclone-storage-commands
Panaetius Nov 15, 2023
535685e
fix storage cmmand
Nov 24, 2023
f3dd0d9
Merge branch 'develop' into feat/rclone-storage-commands
Panaetius Dec 6, 2023
362b15f
fix pytest output to show summary at end and traceback
Panaetius Dec 6, 2023
0b01efc
fix test
Panaetius Dec 6, 2023
f407d5a
fix test
Panaetius Dec 6, 2023
e9d09a7
adapt to storage changes
Dec 7, 2023
7abb677
Merge branch 'feat/rclone-storage-commands' of github.com:SwissDataSc…
Dec 7, 2023
1016961
Merge branch 'develop' into feat/rclone-storage-commands
Panaetius Dec 20, 2023
e821623
Merge branch 'develop' into feat/rclone-storage-commands
Jan 12, 2024
6abaded
Merge branch 'develop' into feat/rclone-storage-commands
Panaetius Jan 15, 2024
d9ef01f
Merge branch 'develop' into feat/rclone-storage-commands
Panaetius Jan 16, 2024
85df232
Merge branch 'develop' into feat/rclone-storage-commands
Jan 17, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ repos:
- --ignore=D105,D107,D202,D203,D212,D213,D401,D406,D407,D410,D411,D413
additional_dependencies:
- toml
exclude: ^renku/ui/service/views/
- repo: https://github.com/koalaman/shellcheck-precommit
rev: v0.8.0
hooks:
Expand Down
2 changes: 1 addition & 1 deletion docs/_static/cheatsheet/cheatsheet.json
Original file line number Diff line number Diff line change
Expand Up @@ -441,7 +441,7 @@
]
},
{
"command": "$ renku storage pull <path>...",
"command": "$ renku lfs pull <path>...",
"description": "Pull <path>'s from external storage (LFS).",
"target": [
"rp"
Expand Down
Binary file modified docs/_static/cheatsheet/cheatsheet.pdf
Binary file not shown.
2 changes: 1 addition & 1 deletion docs/cheatsheet_hash
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
5316163d742bdb6792ed8bcb35031f6c cheatsheet.tex
b8a4fc75c7ba023773b0ccc2e98ebc02 cheatsheet.tex
c70c179e07f04186ec05497564165f11 sdsc_cheatsheet.cls
2 changes: 1 addition & 1 deletion docs/cheatsheet_json_hash
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1856fb451165d013777c7c4cdd56e575 cheatsheet.json
171f230e9ec6372e52129df1bfcf485a cheatsheet.json
2 changes: 1 addition & 1 deletion docs/reference/commands/storage.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
renku storage
*************

.. automodule:: renku.ui.cli.storage
.. automodule:: renku.ui.cli.lfs
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ pattern = """(?x) (?# ignore whitespace
"""

[tool.pytest.ini_options]
addopts = "--doctest-glob=\"*.rst\" --doctest-modules --cov --cov-report=term-missing --ignore=docs/cheatsheet/ --tb=line"
addopts = "--doctest-glob=\"*.rst\" --doctest-modules --cov --cov-report=term-missing --ignore=docs/cheatsheet/ -ra"
doctest_optionflags = "ALLOW_UNICODE"
flake8-ignore = ["*.py", "E121", "E126", "E203", "E226", "E231", "W503", "W504", "docs/conf.py", "docs/cheatsheet/conf.py", "ALL"]
flake8-max-line-length = 120
Expand Down
2 changes: 1 addition & 1 deletion renku/command/checks/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"""Check for large files in Git history."""

from renku.command.util import WARNING
from renku.core.storage import check_external_storage, check_lfs_migrate_info
from renku.core.lfs import check_external_storage, check_lfs_migrate_info


def check_lfs_info(**_):
Expand Down
45 changes: 45 additions & 0 deletions renku/command/command_builder/command.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import contextlib
import functools
import shutil
import threading
from collections import defaultdict
from pathlib import Path
Expand Down Expand Up @@ -455,6 +456,13 @@ def require_clean(self) -> "Command":

return RequireClean(self)

@check_finalized
def require_login(self) -> "Command":
"""Check that the user is logged in."""
from renku.command.command_builder.repo import RequireLogin

return RequireLogin(self)

@check_finalized
def with_communicator(self, communicator: CommunicationCallback) -> "Command":
"""Create a communicator.
Expand All @@ -479,6 +487,20 @@ def with_database(self, write: bool = False, path: Optional[str] = None, create:

return DatabaseCommand(self, write, path, create)

@check_finalized
def with_gitlab_api(self) -> "Command":
"""Inject gitlab api client."""
from renku.command.command_builder.gitlab import GitlabApiCommand

return GitlabApiCommand(self)

@check_finalized
def with_storage_api(self) -> "Command":
"""Inject storage api client."""
from renku.command.command_builder.storage import StorageApiCommand

return StorageApiCommand(self)


class CommandResult:
"""The result of a command.
Expand All @@ -496,3 +518,26 @@ def __init__(self, output, error, status) -> None:
self.output = output
self.error = error
self.status = status


class RequireExecutable(Command):
"""Builder to check if an executable is installed."""

HOOK_ORDER = 4

def __init__(self, builder: Command, executable: str) -> None:
"""__init__ of RequireExecutable."""
self._builder = builder
self._executable = executable

def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
"""Check if an executable exists on the system.
Args:
builder(Command): Current ``CommandBuilder``.
context(dict): Current context.
"""
if not shutil.which(self._executable):
raise errors.ExecutableNotFound(
f"Couldn't find the executable '{self._executable}' on this system. Please make sure it's installed"
)
54 changes: 54 additions & 0 deletions renku/command/command_builder/gitlab.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Copyright Swiss Data Science Center (SDSC). A partnership between
# École Polytechnique Fédérale de Lausanne (EPFL) and
# Eidgenössische Technische Hochschule Zürich (ETHZ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command builder for gitlab api."""


from renku.command.command_builder.command import Command, check_finalized
from renku.core.interface.git_api_provider import IGitAPIProvider
from renku.domain_model.project_context import project_context
from renku.infrastructure.gitlab_api_provider import GitlabAPIProvider


class GitlabApiCommand(Command):
"""Builder to get a gitlab api client."""

PRE_ORDER = 4

def __init__(self, builder: Command) -> None:
self._builder = builder

def _injection_pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
"""Create a gitlab api provider."""

if not project_context.has_context():
raise ValueError("Gitlab API builder needs a ProjectContext to be set.")

def _get_provider():
from renku.core.login import read_renku_token

token = read_renku_token(None, True)
if not token:
return None
return GitlabAPIProvider(token=token)

context["constructor_bindings"][IGitAPIProvider] = _get_provider

@check_finalized
def build(self) -> Command:
"""Build the command."""
self._builder.add_injection_pre_hook(self.PRE_ORDER, self._injection_pre_hook)

return self._builder.build()
35 changes: 35 additions & 0 deletions renku/command/command_builder/repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from renku.command.command_builder.command import Command, CommandResult, check_finalized
from renku.core import errors
from renku.core.git import ensure_clean
from renku.core.login import ensure_login
from renku.domain_model.project_context import project_context


Expand All @@ -42,6 +43,7 @@ def __init__(
"""__init__ of Commit.
Args:
builder(Command): The current ``CommandBuilder``.
message (str): The commit message. Auto-generated if left empty (Default value = None).
commit_if_empty (bool): Whether to commit if there are no modified files (Default value = None).
raise_if_empty (bool): Whether to raise an exception if there are no modified files (Default value = None).
Expand Down Expand Up @@ -164,6 +166,39 @@ def build(self) -> Command:
return self._builder.build()


class RequireLogin(Command):
"""Builder to check if a user is logged in."""

HOOK_ORDER = 4

def __init__(self, builder: Command) -> None:
"""__init__ of RequireLogin."""
self._builder = builder

def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
"""Check if the user is logged in.
Args:
builder(Command): Current ``CommandBuilder``.
context(dict): Current context.
"""
if not project_context.has_context():
raise ValueError("RequireLogin builder needs a ProjectContext to be set.")

ensure_login()

@check_finalized
def build(self) -> Command:
"""Build the command.
Returns:
Command: Finalized version of this command.
"""
self._builder.add_pre_hook(self.HOOK_ORDER, self._pre_hook)

return self._builder.build()


class Isolation(Command):
"""Builder to run a command in git isolation."""

Expand Down
46 changes: 46 additions & 0 deletions renku/command/command_builder/storage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Copyright Swiss Data Science Center (SDSC). A partnership between
# École Polytechnique Fédérale de Lausanne (EPFL) and
# Eidgenössische Technische Hochschule Zürich (ETHZ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command builder for storage api."""


from renku.command.command_builder.command import Command, check_finalized
from renku.core.interface.storage_service_gateway import IStorageService
from renku.domain_model.project_context import project_context
from renku.infrastructure.storage.storage_service import StorageService


class StorageApiCommand(Command):
"""Builder to get a storage api client."""

PRE_ORDER = 4

def __init__(self, builder: Command) -> None:
self._builder = builder

def _injection_pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
"""Create a storage api provider."""

if not project_context.has_context():
raise ValueError("storage api builder needs a ProjectContext to be set.")

context["constructor_bindings"][IStorageService] = lambda: StorageService()

@check_finalized
def build(self) -> Command:
"""Build the command."""
self._builder.add_injection_pre_hook(self.PRE_ORDER, self._injection_pre_hook)

return self._builder.build()
59 changes: 59 additions & 0 deletions renku/command/format/storage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# Copyright Swiss Data Science Center (SDSC). A partnership between
# École Polytechnique Fédérale de Lausanne (EPFL) and
# Eidgenössische Technische Hochschule Zürich (ETHZ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Serializers for storage."""

import json
from typing import List, Optional

from renku.command.format.tabulate import tabulate
from renku.domain_model.cloud_storage import CloudStorage


def tabular(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
"""Format cloud_storages with a tabular output."""
if not columns:
columns = "id,name,private,type"
return tabulate(collection=cloud_storages, columns=columns, columns_mapping=CLOUD_STORAGE_COLUMNS)


def log(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
"""Format cloud_storages in a log like output."""
from renku.ui.cli.utils.terminal import style_header, style_key

output = []

for cloud_storage in cloud_storages:
output.append(style_header(f"CloudStorage {cloud_storage.name}"))
output.append(style_key("Id: ") + cloud_storage.storage_id) # type: ignore
output.append(style_key("Source Path: ") + cloud_storage.source_path)
output.append(style_key("Target path: ") + cloud_storage.target_path)
output.append(style_key("Private: ") + ("Yes" if cloud_storage.private else "No"))
output.append(style_key("Configuration: \n") + json.dumps(cloud_storage.configuration, indent=4))
Panaetius marked this conversation as resolved.
Show resolved Hide resolved
output.append("")
return "\n".join(output)


CLOUD_STORAGE_FORMATS = {"tabular": tabular, "log": log}
"""Valid formatting options."""

CLOUD_STORAGE_COLUMNS = {
"id": ("storage_id", "id"),
"name": ("name", "name"),
"source_path": ("source_path", "source path"),
"target_path": ("target_path", "target path"),
"private": ("private", "private"),
"type": ("storage_type", "type"),
}
Loading