diff --git a/bases/renku_data_services/data_api/app.py b/bases/renku_data_services/data_api/app.py index 8cf78e880..f50a124df 100644 --- a/bases/renku_data_services/data_api/app.py +++ b/bases/renku_data_services/data_api/app.py @@ -153,6 +153,7 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic: url_prefix=url_prefix, data_connector_repo=config.data_connector_repo, data_connector_to_project_link_repo=config.data_connector_to_project_link_repo, + data_connector_secret_repo=config.data_connector_secret_repo, authenticator=config.authenticator, ) app.blueprint( diff --git a/components/renku_data_services/app_config/config.py b/components/renku_data_services/app_config/config.py index cfb0393b2..efe7c5038 100644 --- a/components/renku_data_services/app_config/config.py +++ b/components/renku_data_services/app_config/config.py @@ -44,7 +44,11 @@ ServerOptionsDefaults, generate_default_resource_pool, ) -from renku_data_services.data_connectors.db import DataConnectorProjectLinkRepository, DataConnectorRepository +from renku_data_services.data_connectors.db import ( + DataConnectorProjectLinkRepository, + DataConnectorRepository, + DataConnectorSecretRepository, +) from renku_data_services.db_config import DBConfig from renku_data_services.git.gitlab import DummyGitlabAPI, GitlabAPI from renku_data_services.k8s.clients import DummyCoreClient, DummySchedulingClient, K8sCoreClient, K8sSchedulingClient @@ -181,6 +185,7 @@ class Config: _data_connector_to_project_link_repo: DataConnectorProjectLinkRepository | None = field( default=None, repr=False, init=False ) + _data_connector_secret_repo: DataConnectorSecretRepository | None = field(default=None, repr=False, init=False) def __post_init__(self) -> None: # NOTE: Read spec files required for Swagger @@ -441,6 +446,18 @@ def data_connector_to_project_link_repo(self) -> DataConnectorProjectLinkReposit ) return self._data_connector_to_project_link_repo + @property + def data_connector_secret_repo(self) -> DataConnectorSecretRepository: + """The DB adapter for data connector secrets.""" + if not self._data_connector_secret_repo: + self._data_connector_secret_repo = DataConnectorSecretRepository( + session_maker=self.db.async_session_maker, + data_connector_repo=self.data_connector_repo, + user_repo=self.kc_user_repo, + secret_service_public_key=self.secrets_service_public_key, + ) + return self._data_connector_secret_repo + @classmethod def from_env(cls, prefix: str = "") -> "Config": """Create a config from environment variables.""" diff --git a/components/renku_data_services/data_connectors/api.spec.yaml b/components/renku_data_services/data_connectors/api.spec.yaml index 2275a4217..85c763b89 100644 --- a/components/renku_data_services/data_connectors/api.spec.yaml +++ b/components/renku_data_services/data_connectors/api.spec.yaml @@ -244,7 +244,7 @@ paths: content: "application/json": schema: - $ref: "#/components/schemas/CloudStorageSecretGetList" + $ref: "#/components/schemas/DataConnectorSecretsList" "404": description: Storage was not found content: @@ -255,21 +255,22 @@ paths: $ref: "#/components/responses/Error" tags: - data_connectors - post: + patch: summary: Save secrets for a data connector + description: New secrets will be added and existing secrets will have their value updated. Using `null` as a value will remove the corresponding secret. requestBody: required: true content: application/json: schema: - $ref: "#/components/schemas/CloudStorageSecretPostList" + $ref: "#/components/schemas/DataConnectorSecretPatchList" responses: "201": description: The secrets for cloud storage were saved content: "application/json": schema: - $ref: "#/components/schemas/CloudStorageSecretGetList" + $ref: "#/components/schemas/DataConnectorSecretsList" default: $ref: "#/components/responses/Error" tags: @@ -306,10 +307,6 @@ components: $ref: "#/components/schemas/Slug" storage: $ref: "#/components/schemas/CloudStorageCore" - secrets: - type: array - items: - $ref: "#/components/schemas/CloudStorageSecretGet" creation_date: $ref: "#/components/schemas/CreationDate" created_by: @@ -498,67 +495,49 @@ components: $ref: "#/components/schemas/Ulid" required: - project_id - CloudStorageSecretPost: + DataConnectorSecretsList: + description: A list of data connectors + type: array + items: + $ref: "#/components/schemas/DataConnectorSecret" + DataConnectorSecret: + description: Information about a credential saved for a data connector type: object - description: Data for storing secret for a storage field properties: name: - type: string - description: Name of the field to store credential for - minLength: 1 - maxLength: 99 - value: - $ref: "#/components/schemas/SecretValue" + $ref: "#/components/schemas/DataConnectorSecretFieldName" + secret_id: + $ref: "#/components/schemas/Ulid" required: - name - - value - CloudStorageSecretPostList: - description: List of storage secrets that are saved - type: array - items: - $ref: "#/components/schemas/CloudStorageSecretPost" - CloudStorageSecretGetList: - description: List of storage secrets that are saved + - secret_id + DataConnectorSecretPatchList: + description: List of secrets to be saved for a data connector type: array items: - $ref: "#/components/schemas/CloudStorageSecretGet" - CloudStorageSecretGet: - type: object - description: Data for saved storage secrets + $ref: "#/components/schemas/DataConnectorSecretPatch" + DataConnectorSecretPatch: + description: Information about a credential to save for a data connector properties: name: - type: string - description: Name of the field to store credential for - minLength: 1 - maxLength: 99 - secret_id: - $ref: "#/components/schemas/Ulid" + $ref: "#/components/schemas/DataConnectorSecretFieldName" + value: + $ref: "#/components/schemas/SecretValueNullable" required: - name - - secret_id - SecretValue: + - value + DataConnectorSecretFieldName: + description: Name of the credential field + type: string + minLength: 1 + maxLength: 99 + example: "secret_key" + SecretValueNullable: description: Secret value that can be any text type: string minLength: 1 maxLength: 5000 - RCloneEntry: - type: object - description: Schema for a storage type in rclone, like S3 or Azure Blob Storage. Contains fields for that storage type. - properties: - name: - type: string - description: Human readable name of the provider - description: - type: string - description: description of the provider - prefix: - type: string - description: Machine readable name of the provider - options: - description: Fields/properties used for this storage. - type: array - items: - $ref: "#/components/schemas/RCloneOption" + nullable: true RCloneOption: type: object description: Single field on an RClone storage, like "remote" or "access_key_id" diff --git a/components/renku_data_services/data_connectors/apispec.py b/components/renku_data_services/data_connectors/apispec.py index ad539d2a3..41931cc73 100644 --- a/components/renku_data_services/data_connectors/apispec.py +++ b/components/renku_data_services/data_connectors/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-09-17T13:48:49+00:00 +# timestamp: 2024-10-03T13:29:06+00:00 from __future__ import annotations @@ -246,31 +246,11 @@ class DataConnectorToProjectLinkPost(BaseAPISpec): ) -class CloudStorageSecretPost(BaseAPISpec): +class DataConnectorSecret(BaseAPISpec): name: str = Field( ..., - description="Name of the field to store credential for", - max_length=99, - min_length=1, - ) - value: str = Field( - ..., - description="Secret value that can be any text", - max_length=5000, - min_length=1, - ) - - -class CloudStorageSecretPostList(RootModel[List[CloudStorageSecretPost]]): - root: List[CloudStorageSecretPost] = Field( - ..., description="List of storage secrets that are saved" - ) - - -class CloudStorageSecretGet(BaseAPISpec): - name: str = Field( - ..., - description="Name of the field to store credential for", + description="Name of the credential field", + example="secret_key", max_length=99, min_length=1, ) @@ -283,14 +263,19 @@ class CloudStorageSecretGet(BaseAPISpec): ) -class RCloneEntry(BaseAPISpec): - name: Optional[str] = Field(None, description="Human readable name of the provider") - description: Optional[str] = Field(None, description="description of the provider") - prefix: Optional[str] = Field( - None, description="Machine readable name of the provider" +class DataConnectorSecretPatch(BaseAPISpec): + name: str = Field( + ..., + description="Name of the credential field", + example="secret_key", + max_length=99, + min_length=1, ) - options: Optional[List[RCloneOption]] = Field( - None, description="Fields/properties used for this storage." + value: Optional[str] = Field( + ..., + description="Secret value that can be any text", + max_length=5000, + min_length=1, ) @@ -337,7 +322,6 @@ class DataConnector(BaseAPISpec): pattern="^(?!.*\\.git$|.*\\.atom$|.*[\\-._][\\-._].*)[a-zA-Z0-9][a-zA-Z0-9\\-_.]*$", ) storage: CloudStorageCore - secrets: Optional[List[CloudStorageSecretGet]] = None creation_date: datetime = Field( ..., description="The date and time the resource was created (in UTC and ISO-8601 format)", @@ -450,9 +434,15 @@ class DataConnectorToProjectLinksList(RootModel[List[DataConnectorToProjectLink] ) -class CloudStorageSecretGetList(RootModel[List[CloudStorageSecretGet]]): - root: List[CloudStorageSecretGet] = Field( - ..., description="List of storage secrets that are saved" +class DataConnectorSecretsList(RootModel[List[DataConnectorSecret]]): + root: List[DataConnectorSecret] = Field( + ..., description="A list of data connectors" + ) + + +class DataConnectorSecretPatchList(RootModel[List[DataConnectorSecretPatch]]): + root: List[DataConnectorSecretPatch] = Field( + ..., description="List of secrets to be saved for a data connector" ) diff --git a/components/renku_data_services/data_connectors/blueprints.py b/components/renku_data_services/data_connectors/blueprints.py index eb0cd7ce8..745474f58 100644 --- a/components/renku_data_services/data_connectors/blueprints.py +++ b/components/renku_data_services/data_connectors/blueprints.py @@ -15,16 +15,21 @@ ) from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint from renku_data_services.base_api.etag import extract_if_none_match, if_match_required -from renku_data_services.base_api.misc import validate_query +from renku_data_services.base_api.misc import validate_body_root_model, validate_query from renku_data_services.base_api.pagination import PaginationRequest, paginate from renku_data_services.base_models.validation import validate_and_dump, validated_json from renku_data_services.data_connectors import apispec, models from renku_data_services.data_connectors.core import ( dump_storage_with_sensitive_fields, validate_data_connector_patch, + validate_data_connector_secrets_patch, validate_unsaved_data_connector, ) -from renku_data_services.data_connectors.db import DataConnectorProjectLinkRepository, DataConnectorRepository +from renku_data_services.data_connectors.db import ( + DataConnectorProjectLinkRepository, + DataConnectorRepository, + DataConnectorSecretRepository, +) from renku_data_services.storage.rclone import RCloneValidator @@ -34,6 +39,7 @@ class DataConnectorsBP(CustomBlueprint): data_connector_repo: DataConnectorRepository data_connector_to_project_link_repo: DataConnectorProjectLinkRepository + data_connector_secret_repo: DataConnectorSecretRepository authenticator: base_models.Authenticator def get_all(self) -> BlueprintFactoryResponse: @@ -262,6 +268,64 @@ async def _get_all_data_connectors_links_to_project( return "/projects//data_connector_links", ["GET"], _get_all_data_connectors_links_to_project + def get_secrets(self) -> BlueprintFactoryResponse: + """List all saved secrets for a data connector.""" + + @authenticate(self.authenticator) + @only_authenticated + async def _get_secrets( + _: Request, + user: base_models.APIUser, + data_connector_id: ULID, + ) -> JSONResponse: + secrets = await self.data_connector_secret_repo.get_data_connector_secrets( + user=user, data_connector_id=data_connector_id + ) + return validated_json( + apispec.DataConnectorSecretsList, [self._dump_data_connector_secret(secret) for secret in secrets] + ) + + return "/data_connectors//secrets", ["GET"], _get_secrets + + def patch_secrets(self) -> BlueprintFactoryResponse: + """Create, update or delete saved secrets for a data connector.""" + + @authenticate(self.authenticator) + @only_authenticated + @validate_body_root_model(json=apispec.DataConnectorSecretPatchList) + async def _patch_secrets( + _: Request, + user: base_models.APIUser, + data_connector_id: ULID, + body: apispec.DataConnectorSecretPatchList, + ) -> JSONResponse: + unsaved_secrets = validate_data_connector_secrets_patch(put=body) + secrets = await self.data_connector_secret_repo.patch_data_connector_secrets( + user=user, data_connector_id=data_connector_id, secrets=unsaved_secrets + ) + return validated_json( + apispec.DataConnectorSecretsList, [self._dump_data_connector_secret(secret) for secret in secrets] + ) + + return "/data_connectors//secrets", ["PATCH"], _patch_secrets + + def delete_secrets(self) -> BlueprintFactoryResponse: + """Delete all saved secrets for a data connector.""" + + @authenticate(self.authenticator) + @only_authenticated + async def _delete_secrets( + _: Request, + user: base_models.APIUser, + data_connector_id: ULID, + ) -> HTTPResponse: + await self.data_connector_secret_repo.delete_data_connector_secrets( + user=user, data_connector_id=data_connector_id + ) + return HTTPResponse(status=204) + + return "/data_connectors//secrets", ["DELETE"], _delete_secrets + @staticmethod def _dump_data_connector(data_connector: models.DataConnector, validator: RCloneValidator) -> dict[str, Any]: """Dumps a data connector for API responses.""" @@ -291,3 +355,11 @@ def _dump_data_connector_to_project_link(link: models.DataConnectorToProjectLink creation_date=link.creation_date, created_by=link.created_by, ) + + @staticmethod + def _dump_data_connector_secret(secret: models.DataConnectorSecret) -> dict[str, Any]: + """Dumps a data connector secret for API responses.""" + return dict( + name=secret.name, + secret_id=str(secret.secret_id), + ) diff --git a/components/renku_data_services/data_connectors/core.py b/components/renku_data_services/data_connectors/core.py index 5e7a3efdd..64a65ccdb 100644 --- a/components/renku_data_services/data_connectors/core.py +++ b/components/renku_data_services/data_connectors/core.py @@ -3,7 +3,7 @@ from dataclasses import asdict from typing import Any -from renku_data_services import base_models +from renku_data_services import base_models, errors from renku_data_services.authz.models import Visibility from renku_data_services.data_connectors import apispec, models from renku_data_services.storage import models as storage_models @@ -119,3 +119,22 @@ def validate_data_connector_patch( keywords=keywords, storage=storage, ) + + +def validate_data_connector_secrets_patch( + put: apispec.DataConnectorSecretPatchList, +) -> list[models.DataConnectorSecretUpdate]: + """Validate the update to a data connector's secrets.""" + seen_names: set[str] = set() + for secret in put.root: + if secret.name in seen_names: + raise errors.ValidationError(message=f"Found duplicate name '{secret.name}' in the list of secrets.") + seen_names.add(secret.name) + + return [ + models.DataConnectorSecretUpdate( + name=secret.name, + value=secret.value, + ) + for secret in put.root + ] diff --git a/components/renku_data_services/data_connectors/db.py b/components/renku_data_services/data_connectors/db.py index 58d347e82..6adef7993 100644 --- a/components/renku_data_services/data_connectors/db.py +++ b/components/renku_data_services/data_connectors/db.py @@ -3,7 +3,8 @@ from collections.abc import Callable from typing import TypeVar -from sqlalchemy import Select, func, select +from cryptography.hazmat.primitives.asymmetric import rsa +from sqlalchemy import Select, delete, func, select from sqlalchemy.ext.asyncio import AsyncSession from ulid import ULID @@ -14,6 +15,10 @@ from renku_data_services.data_connectors import apispec, models from renku_data_services.data_connectors import orm as schemas from renku_data_services.namespace import orm as ns_schemas +from renku_data_services.secrets import orm as secrets_schemas +from renku_data_services.secrets.core import encrypt_user_secret +from renku_data_services.secrets.models import SecretKind +from renku_data_services.users.db import UserRepo from renku_data_services.utils.core import with_db_transaction @@ -438,6 +443,126 @@ async def delete_link( return link +class DataConnectorSecretRepository: + """Repository for data connector secrets.""" + + def __init__( + self, + session_maker: Callable[..., AsyncSession], + data_connector_repo: DataConnectorRepository, + user_repo: UserRepo, + secret_service_public_key: rsa.RSAPublicKey, + ) -> None: + self.session_maker = session_maker + self.data_connector_repo = data_connector_repo + self.user_repo = user_repo + self.secret_service_public_key = secret_service_public_key + + async def get_data_connector_secrets( + self, + user: base_models.APIUser, + data_connector_id: ULID, + ) -> list[models.DataConnectorSecret]: + """Get data connectors secrets from the database.""" + if user.id is None: + raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") + + async with self.session_maker() as session: + stmt = ( + select(schemas.DataConnectorSecretORM) + .where(schemas.DataConnectorSecretORM.user_id == user.id) + .where(schemas.DataConnectorSecretORM.data_connector_id == data_connector_id) + .where(schemas.DataConnectorSecretORM.secret_id == secrets_schemas.SecretORM.id) + .where(secrets_schemas.SecretORM.user_id == user.id) + ) + results = await session.scalars(stmt) + secrets = results.all() + + return [secret.dump() for secret in secrets] + + async def patch_data_connector_secrets( + self, user: base_models.APIUser, data_connector_id: ULID, secrets: list[models.DataConnectorSecretUpdate] + ) -> list[models.DataConnectorSecret]: + """Create, update or remove data connector secrets.""" + if user.id is None: + raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") + + # NOTE: check that the user can access the data connector + await self.data_connector_repo.get_data_connector(user=user, data_connector_id=data_connector_id) + + secrets_as_dict = {s.name: s.value for s in secrets} + + async with self.session_maker() as session, session.begin(): + stmt = ( + select(schemas.DataConnectorSecretORM) + .where(schemas.DataConnectorSecretORM.user_id == user.id) + .where(schemas.DataConnectorSecretORM.data_connector_id == data_connector_id) + .where(schemas.DataConnectorSecretORM.secret_id == secrets_schemas.SecretORM.id) + .where(secrets_schemas.SecretORM.user_id == user.id) + ) + result = await session.scalars(stmt) + existing_secrets = result.all() + existing_secrets_as_dict = {s.name: s for s in existing_secrets} + + all_secrets = [] + + for name, value in secrets_as_dict.items(): + if value is None: + # Remove the secret + data_connector_secret_orm = existing_secrets_as_dict.get(name) + if data_connector_secret_orm is None: + continue + await session.delete(data_connector_secret_orm.secret) + del existing_secrets_as_dict[name] + continue + + encrypted_value, encrypted_key = await encrypt_user_secret( + user_repo=self.user_repo, + requested_by=user, + secret_service_public_key=self.secret_service_public_key, + secret_value=value, + ) + + if data_connector_secret_orm := existing_secrets_as_dict.get(name): + data_connector_secret_orm.secret.update( + encrypted_value=encrypted_value, encrypted_key=encrypted_key + ) + else: + secret_orm = secrets_schemas.SecretORM( + name=f"{data_connector_id}-{name}", + user_id=user.id, + encrypted_value=encrypted_value, + encrypted_key=encrypted_key, + kind=SecretKind.storage, + ) + data_connector_secret_orm = schemas.DataConnectorSecretORM( + name=name, + user_id=user.id, + data_connector_id=data_connector_id, + secret_id=secret_orm.id, + ) + session.add(secret_orm) + session.add(data_connector_secret_orm) + + all_secrets.append(data_connector_secret_orm.dump()) + + return all_secrets + + async def delete_data_connector_secrets(self, user: base_models.APIUser, data_connector_id: ULID) -> None: + """Delete data connector secrets.""" + if user.id is None: + raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") + + async with self.session_maker() as session, session.begin(): + stmt = ( + delete(secrets_schemas.SecretORM) + .where(secrets_schemas.SecretORM.user_id == user.id) + .where(secrets_schemas.SecretORM.id == schemas.DataConnectorSecretORM.secret_id) + .where(schemas.DataConnectorSecretORM.data_connector_id == data_connector_id) + ) + await session.execute(stmt) + + _T = TypeVar("_T") diff --git a/components/renku_data_services/data_connectors/models.py b/components/renku_data_services/data_connectors/models.py index b63103797..7245bf8e9 100644 --- a/components/renku_data_services/data_connectors/models.py +++ b/components/renku_data_services/data_connectors/models.py @@ -116,3 +116,21 @@ class DataConnectorToProjectLink(UnsavedDataConnectorToProjectLink): created_by: str creation_date: datetime updated_at: datetime + + +@dataclass(frozen=True, eq=True, kw_only=True) +class DataConnectorSecret: + """Data connector secret model.""" + + name: str + user_id: str + data_connector_id: ULID + secret_id: ULID + + +@dataclass(frozen=True, eq=True, kw_only=True) +class DataConnectorSecretUpdate: + """Secret to be saved for a data connector.""" + + name: str + value: str | None diff --git a/components/renku_data_services/data_connectors/orm.py b/components/renku_data_services/data_connectors/orm.py index afd1767cb..447168d34 100644 --- a/components/renku_data_services/data_connectors/orm.py +++ b/components/renku_data_services/data_connectors/orm.py @@ -121,6 +121,13 @@ class DataConnectorToProjectLinkORM(BaseORM): """A link from a data connector to a project in Renku 2.0.""" __tablename__ = "data_connector_to_project_links" + __table_args__ = ( + UniqueConstraint( + "data_connector_id", + "project_id", + name="_unique_data_connector_id_project_id_uc", + ), + ) id: Mapped[ULID] = mapped_column("id", ULIDType, primary_key=True, default_factory=lambda: str(ULID()), init=False) """ID of this data connector to project link.""" @@ -150,14 +157,6 @@ class DataConnectorToProjectLinkORM(BaseORM): nullable=False, ) - __table_args__ = ( - UniqueConstraint( - "data_connector_id", - "project_id", - name="_unique_data_connector_id_project_id_uc", - ), - ) - def dump(self) -> models.DataConnectorToProjectLink: """Create a link model from the DataConnectorProjectLinkORM.""" return models.DataConnectorToProjectLink( @@ -188,3 +187,12 @@ class DataConnectorSecretORM(BaseORM): secret_id: Mapped[ULID] = mapped_column("secret_id", ForeignKey(SecretORM.id, ondelete="CASCADE")) secret: Mapped[SecretORM] = relationship(init=False, repr=False, lazy="selectin") + + def dump(self) -> models.DataConnectorSecret: + """Create a data connector secret model from the DataConnectorSecretORM.""" + return models.DataConnectorSecret( + name=self.name, + user_id=self.user_id, + data_connector_id=self.data_connector_id, + secret_id=self.secret_id, + ) diff --git a/test/bases/renku_data_services/data_api/test_data_connectors.py b/test/bases/renku_data_services/data_api/test_data_connectors.py index 96f7d1d76..cd38306ec 100644 --- a/test/bases/renku_data_services/data_api/test_data_connectors.py +++ b/test/bases/renku_data_services/data_api/test_data_connectors.py @@ -1017,3 +1017,169 @@ async def test_delete_project_after_linking( assert response.status_code == 200, response.text assert response.json is not None assert len(response.json) == 0 + + +@pytest.mark.asyncio +async def test_patch_data_connector_secrets( + sanic_client: SanicASGITestClient, create_data_connector, user_headers +) -> None: + data_connector = await create_data_connector("My data connector") + data_connector_id = data_connector["id"] + + payload = [ + {"name": "access_key_id", "value": "access key id value"}, + {"name": "secret_access_key", "value": "secret access key value"}, + ] + _, response = await sanic_client.patch( + f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers, json=payload + ) + + assert response.status_code == 200, response.json + assert response.json is not None + secrets = response.json + assert len(secrets) == 2 + assert {s["name"] for s in secrets} == {"access_key_id", "secret_access_key"} + + # Check that the secrets are returned from a GET request + _, response = await sanic_client.get(f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers) + assert response.status_code == 200, response.json + assert response.json is not None + secrets = response.json + assert len(secrets) == 2 + assert {s["name"] for s in secrets} == {"access_key_id", "secret_access_key"} + + +@pytest.mark.asyncio +async def test_patch_data_connector_secrets_update_secrets( + sanic_client: SanicASGITestClient, create_data_connector, user_headers +) -> None: + data_connector = await create_data_connector("My data connector") + data_connector_id = data_connector["id"] + payload = [ + {"name": "access_key_id", "value": "access key id value"}, + {"name": "secret_access_key", "value": "secret access key value"}, + ] + _, response = await sanic_client.patch( + f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers, json=payload + ) + assert response.status_code == 200, response.json + assert response.json is not None + secrets = response.json + assert len(secrets) == 2 + assert {s["name"] for s in secrets} == {"access_key_id", "secret_access_key"} + secret_ids = {s["secret_id"] for s in secrets} + + payload = [ + {"name": "access_key_id", "value": "new access key id value"}, + {"name": "secret_access_key", "value": "new secret access key value"}, + ] + _, response = await sanic_client.patch( + f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers, json=payload + ) + + assert response.status_code == 200, response.json + assert response.json is not None + secrets = response.json + assert len(secrets) == 2 + assert {s["name"] for s in secrets} == {"access_key_id", "secret_access_key"} + assert {s["secret_id"] for s in secrets} == secret_ids + + # Check that the secrets are returned from a GET request + _, response = await sanic_client.get(f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers) + assert response.status_code == 200, response.json + assert response.json is not None + secrets = response.json + assert len(secrets) == 2 + assert {s["name"] for s in secrets} == {"access_key_id", "secret_access_key"} + assert {s["secret_id"] for s in secrets} == secret_ids + + +@pytest.mark.asyncio +async def test_patch_data_connector_secrets_add_and_remove_secrets( + sanic_client: SanicASGITestClient, create_data_connector, user_headers +) -> None: + data_connector = await create_data_connector("My data connector") + data_connector_id = data_connector["id"] + payload = [ + {"name": "access_key_id", "value": "access key id value"}, + {"name": "secret_access_key", "value": "secret access key value"}, + ] + _, response = await sanic_client.patch( + f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers, json=payload + ) + assert response.status_code == 200, response.json + assert response.json is not None + secrets = response.json + assert len(secrets) == 2 + assert {s["name"] for s in secrets} == {"access_key_id", "secret_access_key"} + access_key_id_secret_id = next(filter(lambda s: s["name"] == "access_key_id", secrets), None) + + payload = [ + {"name": "access_key_id", "value": "new access key id value"}, + {"name": "secret_access_key", "value": None}, + {"name": "password", "value": "password"}, + ] + _, response = await sanic_client.patch( + f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers, json=payload + ) + + assert response.status_code == 200, response.json + assert response.json is not None + secrets = response.json + assert len(secrets) == 2 + assert {s["name"] for s in secrets} == {"access_key_id", "password"} + new_access_key_id_secret_id = next(filter(lambda s: s["name"] == "access_key_id", secrets), None) + assert new_access_key_id_secret_id == access_key_id_secret_id + + # Check that the secrets are returned from a GET request + _, response = await sanic_client.get(f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers) + assert response.status_code == 200, response.json + assert response.json is not None + secrets = response.json + assert len(secrets) == 2 + assert {s["name"] for s in secrets} == {"access_key_id", "password"} + + # Check the associated secrets + _, response = await sanic_client.get("/api/data/user/secrets", params={"kind": "storage"}, headers=user_headers) + + assert response.status_code == 200 + assert response.json is not None + assert len(response.json) == 2 + assert {s["name"] for s in secrets} == {"access_key_id", "password"} + + +@pytest.mark.asyncio +async def test_delete_data_connector_secrets( + sanic_client: SanicASGITestClient, create_data_connector, user_headers +) -> None: + data_connector = await create_data_connector("My data connector") + data_connector_id = data_connector["id"] + payload = [ + {"name": "access_key_id", "value": "access key id value"}, + {"name": "secret_access_key", "value": "secret access key value"}, + ] + _, response = await sanic_client.patch( + f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers, json=payload + ) + assert response.status_code == 200, response.json + assert response.json is not None + secrets = response.json + assert len(secrets) == 2 + assert {s["name"] for s in secrets} == {"access_key_id", "secret_access_key"} + + _, response = await sanic_client.delete( + f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers + ) + + assert response.status_code == 204, response.json + + # Check that the secrets list is empty from the GET request + _, response = await sanic_client.get(f"/api/data/data_connectors/{data_connector_id}/secrets", headers=user_headers) + assert response.status_code == 200, response.json + assert response.json == [], response.json + + # Check that the associated secrets are deleted + _, response = await sanic_client.get("/api/data/user/secrets", params={"kind": "storage"}, headers=user_headers) + + assert response.status_code == 200 + assert response.json == [], response.json