feat: rclone csi support (#1707)

* feat: rclone csi support

* return cloud storage in responses

* add support for virtual switch provider

* support readonly mode

* switch to new deploy action

---------

Co-authored-by: Ralf Grubenmann <[email protected]>

.github/workflows/acceptance-tests.yml

@@ -23,6 +23,7 @@ jobs:
       renku-gateway: ${{ steps.deploy-comment.outputs.renku-gateway}}
       renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}}
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
+      amalthea: ${{ steps.deploy-comment.outputs.amalthea}}
       renku-data-services: ${{ steps.deploy-comment.outputs.renku-data-services}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
@@ -65,6 +66,7 @@ jobs:
           renku_gateway: "${{ needs.check-deploy.outputs.renku-gateway }}"
           renku_data_services: "${{ needs.check-deploy.outputs.renku-data-services }}"
           renku_ui: "${{ needs.check-deploy.outputs.renku-ui }}"
+          amalthea: "${{ needs.check-deploy.outputs.amalthea }}"
           extra_values: "${{ needs.check-deploy.outputs.extra-values }}"
       - name: Check existing renkubot comment
         uses: peter-evans/find-comment@v2

git_services/git_services/init/clone.py

@@ -21,5 +21,5 @@
     git_cloner.run(
         session_branch=config.branch,
         root_commit_sha=config.commit_sha,
-        s3_mounts=config.s3_mounts,
+        storage_mounts=config.storage_mounts,
     )

git_services/git_services/init/cloner.py

@@ -172,7 +172,7 @@ def _repo_exists(self):
             return False
         return res.lower().strip() == "true"
 
-    def run(self, *, session_branch: str, root_commit_sha: str, s3_mounts: List[str]):
+    def run(self, *, session_branch: str, root_commit_sha: str, storage_mounts: List[str]):
         logging.info("Checking if the repo already exists.")
         if self._repo_exists():
             # NOTE: This will run when a session is resumed, removing the repo here
@@ -186,12 +186,12 @@ def run(self, *, session_branch: str, root_commit_sha: str, s3_mounts: List[str]
         else:
             with self._temp_plaintext_credentials():
                 self._clone(session_branch)
-        # NOTE: If the S3 mount location already exists it means that the repo folder/file
+        # NOTE: If the storage mount location already exists it means that the repo folder/file
         # or another existing file will be overwritten, so raise an error here and crash.
-        for a_mount in s3_mounts:
+        for a_mount in storage_mounts:
             if Path(a_mount).exists():
                 raise errors.CloudStorageOverwritesExistingFilesError
         self._setup_proxy()
-        logging.info(f"Excluding cloud storage from git: {s3_mounts}")
-        if s3_mounts:
-            self._exclude_storages_from_git(s3_mounts)
+        logging.info(f"Excluding cloud storage from git: {storage_mounts}")
+        if storage_mounts:
+            self._exclude_storages_from_git(storage_mounts)

git_services/git_services/init/config.py

@@ -5,8 +5,8 @@
 
 import dataconf
 
-from git_services.init import errors
 from git_services.cli.sentry import SentryConfig
+from git_services.init import errors
 
 
 @dataclass
@@ -41,15 +41,15 @@ class Config:
     sentry: SentryConfig
     lfs_auto_fetch: Union[str, bool] = "0"
     mount_path: str = "/work"
-    s3_mounts: List[str] = field(default_factory=list)
+    storage_mounts: List[str] = field(default_factory=list)
 
     def __post_init__(self):
         allowed_string_flags = ["0", "1"]
         if self.lfs_auto_fetch not in allowed_string_flags:
             raise ValueError("lfs_auto_fetch can only be a string with values '0' or '1'")
         if isinstance(self.lfs_auto_fetch, str):
             self.lfs_auto_fetch = self.lfs_auto_fetch == "1"
-        for mount in self.s3_mounts:
+        for mount in self.storage_mounts:
             if not Path(mount).is_absolute():
                 raise errors.CloudStorageMountPathNotAbsolute
 

git_services/tests/test_init_clone.py

@@ -33,7 +33,7 @@ def test_simple_git_clone(test_user, clone_dir, mocker):
     mocker.patch("git_services.init.cloner.GitCloner._temp_plaintext_credentials", autospec=True)
     cloner = GitCloner(git_url=git_url, repo_url=repo_url, repo_directory=clone_dir, user=test_user)
     assert len(os.listdir(clone_dir)) == 0
-    cloner.run(session_branch="main", root_commit_sha="test", s3_mounts=[])
+    cloner.run(session_branch="main", root_commit_sha="test", storage_mounts=[])
     assert len(os.listdir(clone_dir)) != 0
 
 
@@ -56,7 +56,7 @@ def test_lfs_size_check(test_user, clone_dir, mocker):
     )
     assert len(os.listdir(clone_dir)) == 0
     with pytest.raises(errors.NoDiskSpaceError):
-        cloner.run(session_branch="main", root_commit_sha="test", s3_mounts=[])
+        cloner.run(session_branch="main", root_commit_sha="test", storage_mounts=[])
 
 
 @pytest.mark.parametrize(

pyproject.toml

@@ -25,8 +25,6 @@ marshmallow = "*"
 apispec = {extras = ["marshmallow"], version = "*"}
 importlib-metadata = "*"
 dataconf = "^2.1.3"
-azure-storage-blob = "^12.19.0"
-boto3 = "^1.28.26"
 
 [tool.poetry.dev-dependencies]
 pylint = "*"

renku_notebooks/__init__.py

@@ -37,11 +37,7 @@
 from .api.schemas.config_server_options import ServerOptionsEndpointResponse
 from .api.schemas.errors import ErrorResponse
 from .api.schemas.logs import ServerLogs
-from .api.schemas.servers_get import (
-    NotebookResponse,
-    ServersGetRequest,
-    ServersGetResponse,
-)
+from .api.schemas.servers_get import NotebookResponse, ServersGetRequest, ServersGetResponse
 from .api.schemas.servers_patch import PatchServerRequest
 from .api.schemas.servers_post import LaunchNotebookRequest
 from .api.schemas.version import VersionResponse

renku_notebooks/api/amalthea_patches/cloudstorage.py

@@ -24,7 +24,7 @@ def main(server: "UserServer") -> List[Dict[str, Any]]:
                         "op": "add",
                         "path": "/statefulset/spec/template/spec/initContainers/2/env/-",
                         "value": {
-                            "name": f"GIT_CLONE_S3_MOUNTS_{i}",
+                            "name": f"GIT_CLONE_STORAGE_MOUNTS_{i}",
                             "value": cloud_storage_request.mount_folder,
                         },
                     },

renku_notebooks/api/amalthea_patches/general.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, List
 
 from ...config import config
 from ..classes.user import RegisteredUser

renku_notebooks/api/amalthea_patches/jupyter_server.py

@@ -3,8 +3,8 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, List
 
-from renku_notebooks.errors.user import OverriddenEnvironmentVariableError
 from renku_notebooks.config import config
+from renku_notebooks.errors.user import OverriddenEnvironmentVariableError
 
 if TYPE_CHECKING:
     from renku_notebooks.api.classes.server import UserServer