diff --git a/components/renku_data_services/migrations/versions/d71f0f795d30_allow_environments_to_be_archived.py b/components/renku_data_services/migrations/versions/d71f0f795d30_allow_environments_to_be_archived.py new file mode 100644 index 000000000..b1cee71b0 --- /dev/null +++ b/components/renku_data_services/migrations/versions/d71f0f795d30_allow_environments_to_be_archived.py @@ -0,0 +1,32 @@ +"""allow environments to be archived + +Revision ID: d71f0f795d30 +Revises: d1cdcbb2adc3 +Create Date: 2025-01-10 07:50:44.144549 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d71f0f795d30" +down_revision = "d1cdcbb2adc3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "environments", + sa.Column("is_archived", sa.Boolean(), server_default=sa.text("false"), nullable=False), + schema="sessions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("environments", "is_archived", schema="sessions") + # ### end Alembic commands ### diff --git a/components/renku_data_services/session/api.spec.yaml b/components/renku_data_services/session/api.spec.yaml index 995828a5d..8be1a97b1 100644 --- a/components/renku_data_services/session/api.spec.yaml +++ b/components/renku_data_services/session/api.spec.yaml @@ -11,6 +11,19 @@ paths: /environments: get: summary: Get all global environments + parameters: + - in: query + style: form + explode: true + name: get_environment_params + schema: + type: object + additionalProperties: false + properties: + with_archived: + type: boolean + default: false + description: Whether to return archived environments or not responses: "200": description: List of global environments @@ -277,6 +290,8 @@ components: $ref: "#/components/schemas/EnvironmentCommand" args: $ref: "#/components/schemas/EnvironmentArgs" + is_archived: + $ref: "#/components/schemas/IsArchived" required: - id - name @@ -298,6 +313,7 @@ components: mount_directory: /home/jovyan/work uid: 1000 gid: 1000 + is_archive: false EnvironmentGetInLauncher: allOf: - $ref: "#/components/schemas/Environment" @@ -358,6 +374,9 @@ components: $ref: "#/components/schemas/EnvironmentCommand" args: $ref: "#/components/schemas/EnvironmentArgs" + is_archived: + $ref: "#/components/schemas/IsArchived" + default: false required: - name - container_image @@ -395,6 +414,8 @@ components: $ref: "#/components/schemas/EnvironmentCommand" args: $ref: "#/components/schemas/EnvironmentArgs" + is_archived: + $ref: "#/components/schemas/IsArchived" SessionLaunchersList: description: A list of Renku session launchers type: array @@ -597,6 +618,9 @@ components: type: string description: The arguments that will follow the command, i.e. will overwrite the image Dockerfile CMD, equivalent to args in Kubernetes minLength: 1 + IsArchived: + type: boolean + description: Whether this environment is archived and not for use in new projects or not ErrorResponse: type: object properties: diff --git a/components/renku_data_services/session/apispec.py b/components/renku_data_services/session/apispec.py index 574fa0621..7aa8f5d1d 100644 --- a/components/renku_data_services/session/apispec.py +++ b/components/renku_data_services/session/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-12-19T08:38:19+00:00 +# timestamp: 2025-01-10T08:17:36+00:00 from __future__ import annotations @@ -29,6 +29,19 @@ class ErrorResponse(BaseAPISpec): error: Error +class GetEnvironmentParams(BaseAPISpec): + model_config = ConfigDict( + extra="forbid", + ) + with_archived: bool = Field( + False, description="Whether to return archived environments or not" + ) + + +class EnvironmentsGetParametersQuery(BaseAPISpec): + get_environment_params: Optional[GetEnvironmentParams] = None + + class Environment(BaseAPISpec): id: str = Field( ..., @@ -99,6 +112,10 @@ class Environment(BaseAPISpec): description="The arguments that will follow the command, i.e. will overwrite the image Dockerfile CMD, equivalent to args in Kubernetes", min_length=1, ) + is_archived: Optional[bool] = Field( + None, + description="Whether this environment is archived and not for use in new projects or not", + ) class EnvironmentGetInLauncher(Environment): @@ -163,6 +180,10 @@ class EnvironmentPost(BaseAPISpec): description="The arguments that will follow the command, i.e. will overwrite the image Dockerfile CMD, equivalent to args in Kubernetes", min_length=1, ) + is_archived: bool = Field( + False, + description="Whether this environment is archived and not for use in new projects or not", + ) class EnvironmentPatch(BaseAPISpec): @@ -216,6 +237,10 @@ class EnvironmentPatch(BaseAPISpec): description="The arguments that will follow the command, i.e. will overwrite the image Dockerfile CMD, equivalent to args in Kubernetes", min_length=1, ) + is_archived: Optional[bool] = Field( + None, + description="Whether this environment is archived and not for use in new projects or not", + ) class SessionLauncher(BaseAPISpec): diff --git a/components/renku_data_services/session/blueprints.py b/components/renku_data_services/session/blueprints.py index 93359fd7f..67e58aa38 100644 --- a/components/renku_data_services/session/blueprints.py +++ b/components/renku_data_services/session/blueprints.py @@ -10,6 +10,7 @@ from renku_data_services import base_models from renku_data_services.base_api.auth import authenticate, only_authenticated from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint +from renku_data_services.base_api.misc import validate_query from renku_data_services.base_models.validation import validated_json from renku_data_services.session import apispec, models from renku_data_services.session.core import ( @@ -31,8 +32,9 @@ class EnvironmentsBP(CustomBlueprint): def get_all(self) -> BlueprintFactoryResponse: """List all session environments.""" - async def _get_all(_: Request) -> JSONResponse: - environments = await self.session_repo.get_environments() + @validate_query(query=apispec.GetEnvironmentParams) + async def _get_all(_: Request, query: apispec.GetEnvironmentParams) -> JSONResponse: + environments = await self.session_repo.get_environments(with_archived=query.with_archived) return validated_json(apispec.EnvironmentList, environments) return "/environments", ["GET"], _get_all diff --git a/components/renku_data_services/session/core.py b/components/renku_data_services/session/core.py index de4ab4637..d3ee48d8e 100644 --- a/components/renku_data_services/session/core.py +++ b/components/renku_data_services/session/core.py @@ -25,6 +25,7 @@ def validate_unsaved_environment( environment_kind=environment_kind, args=environment.args, command=environment.command, + is_archived=environment.is_archived, ) @@ -59,6 +60,7 @@ def validate_environment_patch(patch: apispec.EnvironmentPatch) -> models.Enviro gid=patch.gid, args=RESET if "args" in data_dict and data_dict["args"] is None else patch.args, command=RESET if "command" in data_dict and data_dict["command"] is None else patch.command, + is_archived=patch.is_archived, ) diff --git a/components/renku_data_services/session/db.py b/components/renku_data_services/session/db.py index e13888646..822414158 100644 --- a/components/renku_data_services/session/db.py +++ b/components/renku_data_services/session/db.py @@ -31,14 +31,15 @@ def __init__( self.project_authz: Authz = project_authz self.resource_pools: ResourcePoolRepository = resource_pools - async def get_environments(self) -> list[models.Environment]: + async def get_environments(self, with_archived: bool = False) -> list[models.Environment]: """Get all global session environments from the database.""" async with self.session_maker() as session: - res = await session.scalars( - select(schemas.EnvironmentORM).where( - schemas.EnvironmentORM.environment_kind == models.EnvironmentKind.GLOBAL.value - ) + statement = select(schemas.EnvironmentORM).where( + schemas.EnvironmentORM.environment_kind == models.EnvironmentKind.GLOBAL.value ) + if not with_archived: + statement = statement.where(schemas.EnvironmentORM.is_archived.is_(False)) + res = await session.scalars(statement) environments = res.all() return [e.dump() for e in environments] @@ -82,6 +83,7 @@ def __insert_environment( command=new_environment.command, args=new_environment.args, creation_date=datetime.now(UTC).replace(microsecond=0), + is_archived=new_environment.is_archived, ) session.add(environment) @@ -141,6 +143,9 @@ def __update_environment( elif isinstance(update.command, list): environment.command = update.command + if update.is_archived is not None: + environment.is_archived = update.is_archived + async def update_environment( self, user: base_models.APIUser, environment_id: ULID, patch: models.EnvironmentPatch ) -> models.Environment: @@ -288,6 +293,10 @@ async def insert_launcher( raise errors.MissingResourceError( message=f"Session environment with id '{environment_id}' does not exist or you do not have access to it." # noqa: E501 ) + if environment_orm.is_archived: + raise errors.ValidationError( + message="Cannot create a new session launcher with an archived environment." + ) environment = environment_orm.dump() environment_id = environment.id diff --git a/components/renku_data_services/session/models.py b/components/renku_data_services/session/models.py index 98bfe70fe..1a28d85e5 100644 --- a/components/renku_data_services/session/models.py +++ b/components/renku_data_services/session/models.py @@ -41,6 +41,7 @@ class UnsavedEnvironment: environment_kind: EnvironmentKind args: list[str] | None = None command: list[str] | None = None + is_archived: bool = False def __post_init__(self) -> None: if self.working_directory and not self.working_directory.is_absolute(): @@ -88,6 +89,7 @@ class EnvironmentPatch: gid: int | None = None args: list[str] | None | ResetType = None command: list[str] | None | ResetType = None + is_archived: bool | None = None @dataclass(frozen=True, eq=True, kw_only=True) diff --git a/components/renku_data_services/session/orm.py b/components/renku_data_services/session/orm.py index 3b9442180..1354de50c 100644 --- a/components/renku_data_services/session/orm.py +++ b/components/renku_data_services/session/orm.py @@ -3,7 +3,7 @@ from datetime import datetime from pathlib import PurePosixPath -from sqlalchemy import JSON, DateTime, MetaData, String, func +from sqlalchemy import JSON, Boolean, DateTime, MetaData, String, false, func from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column, relationship from sqlalchemy.schema import ForeignKey @@ -63,6 +63,10 @@ class EnvironmentORM(BaseORM): ) """Creation date and time.""" + is_archived: Mapped[bool] = mapped_column( + "is_archived", Boolean(), default=False, server_default=false(), nullable=False + ) + def dump(self) -> models.Environment: """Create a session environment model from the EnvironmentORM.""" return models.Environment( @@ -81,6 +85,7 @@ def dump(self) -> models.Environment: port=self.port, args=self.args, command=self.command, + is_archived=self.is_archived, ) diff --git a/test/bases/renku_data_services/data_api/test_sessions.py b/test/bases/renku_data_services/data_api/test_sessions.py index 8a14c538d..d70d45e60 100644 --- a/test/bases/renku_data_services/data_api/test_sessions.py +++ b/test/bases/renku_data_services/data_api/test_sessions.py @@ -46,6 +46,7 @@ async def test_get_all_session_environments( await create_session_environment("Environment 1") await create_session_environment("Environment 2") await create_session_environment("Environment 3") + await create_session_environment("Environment 4", {"is_archived": True}) _, res = await sanic_client.get("/api/data/environments", headers=unauthorized_headers) @@ -57,6 +58,17 @@ async def test_get_all_session_environments( "Environment 2", "Environment 3", } + _, res = await sanic_client.get("/api/data/environments?with_archived=true", headers=unauthorized_headers) + + assert res.status_code == 200, res.text + assert res.json is not None + environments = res.json + assert {env["name"] for env in environments} == { + "Environment 1", + "Environment 2", + "Environment 3", + "Environment 4", + } @pytest.mark.asyncio @@ -104,6 +116,7 @@ async def test_post_session_environment(sanic_client: SanicASGITestClient, admin assert res.json.get("name") == "Environment 1" assert res.json.get("description") == "A session environment." assert res.json.get("container_image") == image_name + assert not res.json.get("is_archived") @pytest.mark.asyncio @@ -192,6 +205,57 @@ async def test_patch_session_environment( assert res.json.get("mount_directory") is None +@pytest.mark.asyncio +async def test_patch_session_environment_archived( + sanic_client: SanicASGITestClient, + admin_headers, + create_session_environment, + create_project, + valid_resource_pool_payload, + create_resource_pool, +) -> None: + env = await create_session_environment("Environment 1") + environment_id = env["id"] + + payload = {"is_archived": True} + + _, res = await sanic_client.patch(f"/api/data/environments/{environment_id}", headers=admin_headers, json=payload) + + assert res.status_code == 200, res.text + assert res.json is not None + assert res.json.get("is_archived") + + # Test that you can't create a launcher with an archived environment + project = await create_project("Some project") + resource_pool_data = valid_resource_pool_payload + resource_pool_data["public"] = False + + resource_pool = await create_resource_pool(admin=True, **resource_pool_data) + + payload = { + "name": "Launcher 1", + "project_id": project["id"], + "description": "A session launcher.", + "resource_class_id": resource_pool["classes"][0]["id"], + "environment": {"id": environment_id}, + } + + _, res = await sanic_client.post("/api/data/session_launchers", headers=admin_headers, json=payload) + + assert res.status_code == 422, res.text + + # test unarchiving allows launcher creation again + payload = {"is_archived": False} + + _, res = await sanic_client.patch(f"/api/data/environments/{environment_id}", headers=admin_headers, json=payload) + assert res.status_code == 200, res.text + assert not res.json.get("is_archived") + + _, res = await sanic_client.post("/api/data/session_launchers", headers=admin_headers, json=payload) + + assert res.status_code == 201, res.text + + @pytest.mark.asyncio async def test_patch_session_environment_unauthorized( sanic_client: SanicASGITestClient, user_headers, create_session_environment