Merge branch 'develop' into feat/rclone-storage-commands

.github/actions/install-linux/action.yml

@@ -4,11 +4,11 @@ inputs:
   python-version:
     description: "Python version to install"
     required: false
-    default: "3.9"
+    default: "3.10"
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v3.6.0
       with:
         fetch-depth: 0
     - name: Checkout repository

.github/actions/install-macos/action.yml

@@ -4,11 +4,11 @@ inputs:
   python-version:
     description: "Python version to install"
     required: false
-    default: "3.9"
+    default: "3.10"
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v3.6.0
       with:
         fetch-depth: 0
     - name: Checkout repository

pyproject.toml

@@ -64,7 +64,7 @@ deal = "~4.24.3"
 deepdiff = "~6.7.1"
 deepmerge = "~1.1.0"
 docker = "^5.0.3"
-gitpython = "~3.1.40"
+gitpython = "~3.1.41"
 grandalf = "^0.8"
 humanize = "~4.9.0"
 importlib-resources = "~5.12.0"
@@ -112,7 +112,7 @@ pillow = { version = "^10.1.0", optional = true }
 python-dotenv = { version = "^0.20", optional = true }
 redis = { version = "~5.0.1", optional = true }
 rq = { version = "~1.15.1", optional = true }
-sentry-sdk = { version = "~1.38.0", extras = ["flask"],  optional = true }
+sentry-sdk = { version = "~1.39.0", extras = ["flask"],  optional = true }
 walrus = { version = "^0.9", optional = true }
 prometheus-flask-exporter = "^0.23"
 filetype = "^1.2.0"
@@ -163,7 +163,7 @@ plantweb = ">=1.2.1,<1.3.0"
 renku-sphinx-theme = "^0.4"
 sphinx-click = "^4.3.0"
 sphinx-rtd-theme = "~1.3"
-sphinx-tabs = "==3.4.1"
+sphinx-tabs = "==3.4.4"
 sphinxcontrib-spelling = ">=7,<9"
 
 [tool.poetry.extras]
@@ -314,6 +314,7 @@ module = [
     "ruamel",
     "rq",
     "shellingham",
+    "setuptools",
     "toil.*",
     "tqdm",
     "urllib3.*",
@@ -346,5 +347,5 @@ exclude = ["docs"]
 
 
 [build-system]
-requires = ["poetry-core>=1.3.0,<1.7.0", "poetry-dynamic-versioning==0.21.5", "gitpython==3.1.24"]
+requires = ["poetry-core>=1.3.0,<1.7.0", "poetry-dynamic-versioning==0.21.5", "gitpython==3.1.41"]
 build-backend = "poetry_dynamic_versioning.backend"

renku/__init__.py

@@ -23,6 +23,14 @@
 
 from renku.version import __template_version__, __version__
 
+# distutils is deprecated and fully replaced by setuptools. we don't depend on either, but some of our
+# dependencies do and if distutils gets imported before setuptools, we get an annoying warning.
+# By forcing the import here first, we prevent that warning and ensure the setuptools version is used.
+try:
+    import setuptools  # noqa: F401 # type: ignore
+except ImportError:
+    pass
+
 
 class LoaderWrapper(importlib.abc.Loader):
     """Wrap an importlib loader and add the loaded module to sys.modules with an additional name."""

renku/core/session/docker.py

@@ -20,7 +20,7 @@
 import webbrowser
 from datetime import datetime
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union, cast
 from uuid import uuid4
 
 import docker
@@ -29,8 +29,11 @@
 from renku.core import errors
 from renku.core.config import get_value
 from renku.core.constant import ProviderPriority
+from renku.core.login import read_renku_token
 from renku.core.plugin import hookimpl
+from renku.core.session.utils import get_renku_url
 from renku.core.util import communication
+from renku.core.util.jwt import is_token_expired
 from renku.domain_model.project_context import project_context
 from renku.domain_model.session import ISessionProvider, Session, SessionStopStatus
 
@@ -67,11 +70,14 @@ def docker_client(self) -> docker.client.DockerClient:
         return self._docker_client
 
     @staticmethod
-    def _get_jupyter_urls(ports: Dict[str, Any], auth_token: str, jupyter_port: int = 8888) -> Iterable[str]:
+    def _get_jupyter_urls(ports: Dict[str, Any], auth_token: str, jupyter_port: int = 8888) -> Iterator[str]:
         port_key = f"{jupyter_port}/tcp"
         if port_key not in ports:
-            return list()
-        return map(lambda x: f"http://{x['HostIp']}:{x['HostPort']}/?token={auth_token}", ports[port_key])
+            return list()  # type: ignore
+        default_url = get_value("interactive", "default_url")
+        if not default_url:
+            default_url = "/lab"
+        return map(lambda x: f"http://{x['HostIp']}:{x['HostPort']}{default_url}?token={auth_token}", ports[port_key])
 
     def _get_docker_containers(self, project_name: str) -> List[docker.models.containers.Container]:
         return self.docker_client().containers.list(filters={"label": f"renku_project={project_name}"})
@@ -92,9 +98,22 @@ def build_image(self, image_descriptor: Path, image_name: str, config: Optional[
     def find_image(self, image_name: str, config: Optional[Dict[str, Any]]) -> bool:
         """Find the given container image."""
         with communication.busy(msg=f"Checking for image {image_name}"):
+            renku_url = get_renku_url()
+
+            # only search remote image if a user is logged in
+            find_remote = True
+            if renku_url is None:
+                find_remote = False
+            else:
+                token = read_renku_token(endpoint=renku_url)
+                if not token or is_token_expired(token):
+                    find_remote = False
+
             try:
                 self.docker_client().images.get(image_name)
             except docker.errors.ImageNotFound:
+                if not find_remote:
+                    return False
                 try:
                     self.docker_client().images.get_registry_data(image_name)
                 except docker.errors.APIError:
@@ -457,13 +476,23 @@ def session_url(self, session_name: Optional[str]) -> Optional[str]:
             sessions = self.docker_client().containers.list()
         except errors.DockerError:
             return None
+        default_url = get_value("interactive", "default_url")
+        if not default_url:
+            default_url = "/lab"
 
         for c in sessions:
             if (
                 c.short_id == session_name or (not session_name and len(sessions) == 1)
             ) and f"{DockerSessionProvider.JUPYTER_PORT}/tcp" in c.ports:
-                host = c.ports[f"{DockerSessionProvider.JUPYTER_PORT}/tcp"][0]
-                return f'http://{host["HostIp"]}:{host["HostPort"]}/?token={c.labels["jupyter_token"]}'
+                url = next(
+                    DockerSessionProvider._get_jupyter_urls(
+                        c.ports, c.labels["jupyter_token"], DockerSessionProvider.JUPYTER_PORT
+                    ),
+                    None,
+                )
+                if not url:
+                    continue
+                return url
         return None
 
     def force_build_image(self, force_build: bool = False, **kwargs) -> bool:

renku/core/session/session.py

@@ -179,9 +179,11 @@ def session_start(
     if image_name is None:
         tag = project_context.repository.head.commit.hexsha[:7]
         repo_host = get_image_repository_host()
-        image_name = f"{project_name}:{tag}"
+        image_name = f"{project_name.lower()}:{tag}"
         if repo_host:
             image_name = f"{repo_host}/{image_name}"
+    if image_name.lower() != image_name:
+        raise errors.SessionStartError(f"Image name '{image_name}' cannot contain upper-case letters.")
 
     force_build_image = provider_api.force_build_image(**kwargs)
 

renku/core/session/utils.py

@@ -18,7 +18,9 @@
 import urllib
 from typing import Optional
 
+from renku.core.login import read_renku_token
 from renku.core.util.git import get_remote
+from renku.core.util.jwt import is_token_expired
 from renku.core.util.urls import parse_authentication_endpoint
 from renku.domain_model.project_context import project_context
 
@@ -54,4 +56,8 @@ def get_image_repository_host() -> Optional[str]:
     renku_url = get_renku_url()
     if not renku_url:
         return None
+    token = read_renku_token(endpoint=renku_url)
+    if not token or is_token_expired(token):
+        # only guess host if user is logged in
+        return None
     return "registry." + urllib.parse.urlparse(renku_url).netloc

renku/ui/service/controllers/datasets_edit.py

@@ -73,7 +73,7 @@ def renku_op(self):
 
         creators: Union[NoValueType, List[Person]]
         if "creators" in self.ctx:
-            creators, warnings = construct_creators(self.ctx.get("creators"))  # type: ignore
+            creators, warnings = construct_creators(self.ctx.get("creators"), ignore_email=True)  # type: ignore
         else:
             creators = NO_VALUE
 

renku/ui/service/controllers/templates_create_project.py

@@ -182,6 +182,7 @@ def new_project(self):
                 data_dir=self.ctx.get("data_directory"),
                 ssh_supported=self.template.ssh_supported,
                 image_request=image,
+                keywords=self.ctx.get("project_keywords", []),
             )
 
         self.new_project_push(new_project_path)

renku/ui/service/controllers/version.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Renku service version controller."""
+import os
+
 from renku import __version__
 from renku.core.migration.migrate import SUPPORTED_PROJECT_VERSION
 from renku.ui.service.controllers.api.abstract import ServiceCtrl
@@ -33,6 +35,7 @@ def to_response(self, minimum_version, maximum_version):
             {
                 "latest_version": __version__,
                 "supported_project_version": SUPPORTED_PROJECT_VERSION,
+                "cli_version": os.environ.get("RENKU_PROJECT_DEFAULT_CLI_VERSION") or __version__,
                 "minimum_api_version": minimum_version.name,
                 "maximum_api_version": maximum_version.name,
             },

renku/ui/service/errors.py

@@ -121,7 +121,7 @@ def _handle_sentry(self):
                         sentry_target = sentry_url.netloc.split("@")[-1]
                         # NOTE: sentry doesn't support a global search. A proper link would require the specific org
                         sentry = f"{sentry_url.scheme }://{sentry_target}/organizations/sentry?query={sentry}"
-                    except Exception:
+                    except Exception:  # nosec
                         pass
             except KeyError as e:
                 sentry = f"Unexpected error while reporting to Sentry: {str(e)}"
@@ -348,6 +348,19 @@ def __init__(self, exception=None):
         super().__init__(exception=exception)
 
 
+class UserDatasetsNotFoundError(ServiceError):
+    """Dataset couldn't be found in project."""
+
+    code = SVC_ERROR_USER + 133
+    userMessage = (
+        "The dataset doesn't exist in the project. Please check your inputs or create the target dataset first."
+    )
+    devMessage = "The dataset couldn't be found in the project."
+
+    def __init__(self, exception=None):
+        super().__init__(exception=exception)
+
+
 class UserOutdatedProjectError(ServiceError):
     """The operation can be done only after updating the target project."""
 

renku/ui/service/views/error_handlers.py

@@ -28,6 +28,7 @@
     AuthenticationError,
     DatasetExistsError,
     DatasetImageError,
+    DatasetNotFound,
     DockerfileUpdateError,
     GitCommandError,
     GitError,
@@ -65,6 +66,7 @@
     ProgramUpdateProjectError,
     ServiceError,
     UserDatasetsMultipleImagesError,
+    UserDatasetsNotFoundError,
     UserDatasetsUnlinkError,
     UserDatasetsUnreachableImageError,
     UserInvalidGenericFieldsError,
@@ -368,6 +370,8 @@ def decorated_function(*args, **kwargs):
                 if "".join(value) == "Field may not be null.":
                     raise UserMissingFieldError(e, key)
             raise
+        except DatasetNotFound as e:
+            raise UserDatasetsNotFoundError(e)
         except DatasetExistsError as e:
             raise IntermittentDatasetExistsError(e)
         except RenkuException as e:

tests/service/views/test_templates_views.py

@@ -134,6 +134,7 @@ def test_create_project_from_template(svc_client_templates_creation, with_inject
         "content_url": "https://en.wikipedia.org/static/images/icons/wikipedia.png",
         "mirror_locally": True,
     }
+    payload["project_keywords"] = ["test", "ci"]
 
     response = svc_client.post("/templates.create_project", data=json.dumps(payload), headers=headers)
 
@@ -157,6 +158,7 @@ def test_create_project_from_template(svc_client_templates_creation, with_inject
         with with_injection():
             project = project_context.project
         assert project_context.datadir == "my-folder/"
+        assert project.keywords == ["test", "ci"]
 
     expected_id = f"/projects/{payload['project_namespace']}/{stripped_name}"
     assert expected_id == project.id