diff --git a/src/middlewared/middlewared/api/base/decorator.py b/src/middlewared/middlewared/api/base/decorator.py index e5e71eae73841..4b17cb8cc5221 100644 --- a/src/middlewared/middlewared/api/base/decorator.py +++ b/src/middlewared/middlewared/api/base/decorator.py @@ -1,13 +1,22 @@ import asyncio import functools +from typing import Callable from .handler.accept import accept_params +from ..base.model import BaseModel from middlewared.schema.processor import calculate_args_index __all__ = ["api_method"] -def api_method(accepts, returns, audit=None, audit_callback=False, audit_extended=None, roles=None): +def api_method( + accepts: type[BaseModel], + returns: type[BaseModel], + audit: str | None = None, + audit_callback: bool = False, + audit_extended: Callable[..., str] | None = None, + roles: list[str] | None = None, +): """ Mark a `Service` class method as a public API method. @@ -25,6 +34,9 @@ def api_method(accepts, returns, audit=None, audit_callback=False, audit_extende `roles` is a list of user roles that will gain access to this method. """ + if list(returns.model_fields.keys()) != ["result"]: + raise TypeError("`returns` model must only have one field called `result`") + def wrapper(func): args_index = calculate_args_index(func, audit_callback) diff --git a/src/middlewared/middlewared/api/v25_04_0/cloud_sync.py b/src/middlewared/middlewared/api/v25_04_0/cloud_sync.py index e72ea44b83d1e..bafd9ff1f0e40 100644 --- a/src/middlewared/middlewared/api/v25_04_0/cloud_sync.py +++ b/src/middlewared/middlewared/api/v25_04_0/cloud_sync.py @@ -1,4 +1,5 @@ -from middlewared.api.base import BaseModel, Excluded, excluded_field, ForUpdateMetaclass, NonEmptyString, Private +from middlewared.api.base import (BaseModel, Excluded, excluded_field, ForUpdateMetaclass, NonEmptyString, Private, + single_argument_args, single_argument_result) __all__ = ["CloudCredentialEntry", "CloudCredentialCreateArgs", "CloudCredentialCreateResult", @@ -47,11 +48,13 @@ class CloudCredentialDeleteResult(BaseModel): result: bool +@single_argument_args("cloud_sync_credentials_create") class CloudCredentialVerifyArgs(BaseModel): provider: str attributes: Private[dict] +@single_argument_result class CloudCredentialVerifyResult(BaseModel): valid: bool error: str | None = None diff --git a/src/middlewared/middlewared/test/integration/assets/cloud_sync.py b/src/middlewared/middlewared/test/integration/assets/cloud_sync.py index 4102397ea19ce..6642afd68165a 100644 --- a/src/middlewared/middlewared/test/integration/assets/cloud_sync.py +++ b/src/middlewared/middlewared/test/integration/assets/cloud_sync.py @@ -43,9 +43,9 @@ def task(data): @contextlib.contextmanager -def local_ftp_credential(): +def local_ftp_credential_data(): with anonymous_ftp_server(dataset_name="cloudsync_remote") as ftp: - with credential({ + yield { "provider": "FTP", "attributes": { "host": "localhost", @@ -53,7 +53,13 @@ def local_ftp_credential(): "user": ftp.username, "pass": ftp.password, }, - }) as c: + } + + +@contextlib.contextmanager +def local_ftp_credential(): + with local_ftp_credential_data() as data: + with credential(data) as c: yield c diff --git a/src/middlewared/middlewared/test/integration/utils/client.py b/src/middlewared/middlewared/test/integration/utils/client.py index 0ae1c6e2ba336..4a9b5a20cb8ba 100644 --- a/src/middlewared/middlewared/test/integration/utils/client.py +++ b/src/middlewared/middlewared/test/integration/utils/client.py @@ -103,6 +103,8 @@ def client(self) -> Client: self._client.close() self._client = None + host() # Has to be called in order for `truenas_server` global variable to be correctly initialized when + # running `runtest.py` with a single test name if (addr := self.ip) is None: raise RuntimeError('IP is not set') diff --git a/tests/api2/test_130_cloudsync.py b/tests/api2/test_130_cloudsync.py deleted file mode 100644 index 8135445ba7b86..0000000000000 --- a/tests/api2/test_130_cloudsync.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 - -import pytest -import sys -import os -import time -import urllib.parse -from pytest_dependency import depends -apifolder = os.getcwd() -sys.path.append(apifolder) -from functions import PUT, POST, GET, DELETE, SSH_TEST -from auto_config import pool_name, password, user - -dataset = f"{pool_name}/cloudsync" -dataset_path = os.path.join("/mnt", dataset) - -try: - from config import ( - AWS_ACCESS_KEY_ID, - AWS_SECRET_ACCESS_KEY, - AWS_BUCKET - ) -except ImportError: - Reason = 'AWS credential are missing in config.py' - pytestmark = pytest.mark.skip(reason=Reason) - - -@pytest.fixture(scope='module') -def credentials(): - return {} - - -@pytest.fixture(scope='module') -def task(): - return {} - - -def test_01_create_dataset(request): - result = POST("/pool/dataset/", {"name": dataset}) - assert result.status_code == 200, result.text - - -def test_02_create_cloud_credentials(request, credentials): - result = POST("/cloudsync/credentials/", { - "name": "Test", - "provider": "S3", - "attributes": { - "access_key_id": AWS_ACCESS_KEY_ID, - "secret_access_key": "garbage", - }, - }) - assert result.status_code == 200, result.text - credentials.update(result.json()) - - -def test_03_update_cloud_credentials(request, credentials): - result = PUT(f"/cloudsync/credentials/id/{credentials['id']}/", { - "name": "Test", - "provider": "S3", - "attributes": { - "access_key_id": AWS_ACCESS_KEY_ID, - "secret_access_key": AWS_SECRET_ACCESS_KEY, - }, - }) - assert result.status_code == 200, result.text - - -def test_04_create_cloud_sync(request, credentials, task): - result = POST("/cloudsync/", { - "description": "Test", - "direction": "PULL", - "transfer_mode": "COPY", - "path": dataset_path, - "credentials": credentials["id"], - "schedule": { - "minute": "00", - "hour": "00", - "dom": "1", - "month": "1", - "dow": "1", - }, - "attributes": { - "bucket": AWS_BUCKET, - "folder": "", - }, - "args": "", - }) - assert result.status_code == 200, result.text - task.update(result.json()) - - -def test_05_update_cloud_sync(request, credentials, task): - result = PUT(f"/cloudsync/id/{task['id']}/", { - "description": "Test", - "direction": "PULL", - "transfer_mode": "COPY", - "path": dataset_path, - "credentials": credentials["id"], - "schedule": { - "minute": "00", - "hour": "00", - "dom": "1", - "month": "1", - "dow": "1", - }, - "attributes": { - "bucket": AWS_BUCKET, - "folder": "", - }, - "args": "", - }) - assert result.status_code == 200, result.text - - -def test_06_run_cloud_sync(request, task): - result = POST(f"/cloudsync/id/{task['id']}/sync/") - assert result.status_code == 200, result.text - for i in range(120): - result = GET(f"/cloudsync/id/{task['id']}/") - assert result.status_code == 200, result.text - state = result.json() - if state["job"] is None: - time.sleep(1) - continue - if state["job"]["state"] in ["WAITING", "RUNNING"]: - time.sleep(1) - continue - assert state["job"]["state"] == "SUCCESS", state - cmd = f'cat {dataset_path}/freenas-test.txt' - ssh_result = SSH_TEST(cmd, user, password) - assert ssh_result['result'] is True, ssh_result['output'] - assert ssh_result['stdout'] == 'freenas-test\n', ssh_result['output'] - return - assert False, state - - -def test_07_restore_cloud_sync(request, task): - result = POST(f"/cloudsync/id/{task['id']}/restore/", { - "transfer_mode": "COPY", - "path": dataset_path, - }) - assert result.status_code == 200, result.text - - result = DELETE(f"/cloudsync/id/{result.json()['id']}/") - assert result.status_code == 200, result.text - - -def test_96_delete_cloud_credentials_error(request, credentials): - result = DELETE(f"/cloudsync/credentials/id/{credentials['id']}/") - assert result.status_code == 422 - assert "This credential is used by cloud sync task" in result.json()["message"] - - -def test_97_delete_cloud_sync(request, task): - result = DELETE(f"/cloudsync/id/{task['id']}/") - assert result.status_code == 200, result.text - - -def test_98_delete_cloud_credentials(request, credentials): - result = DELETE(f"/cloudsync/credentials/id/{credentials['id']}/") - assert result.status_code == 200, result.text - - -def test_99_destroy_dataset(request): - result = DELETE(f"/pool/dataset/id/{urllib.parse.quote(dataset, '')}/") - assert result.status_code == 200, result.text diff --git a/tests/api2/test_cloud_sync.py b/tests/api2/test_cloud_sync.py index 98fd7220addc0..10b1a9d14fee1 100644 --- a/tests/api2/test_cloud_sync.py +++ b/tests/api2/test_cloud_sync.py @@ -2,8 +2,9 @@ import time import pytest -from middlewared.test.integration.assets.cloud_sync import credential, task, local_ftp_credential -from middlewared.test.integration.assets.cloud_sync import local_ftp_task, run_task +from middlewared.test.integration.assets.cloud_sync import ( + credential, task, local_ftp_credential, local_ftp_task, run_task, +) from middlewared.test.integration.assets.ftp import anonymous_ftp_server, ftp_server_with_user_account from middlewared.test.integration.assets.pool import dataset from middlewared.test.integration.utils import call, pool, ssh diff --git a/tests/api2/test_cloud_sync_credentials.py b/tests/api2/test_cloud_sync_credentials.py new file mode 100644 index 0000000000000..3123e56c81678 --- /dev/null +++ b/tests/api2/test_cloud_sync_credentials.py @@ -0,0 +1,13 @@ +from middlewared.test.integration.assets.cloud_sync import local_ftp_credential_data +from middlewared.test.integration.utils import call + + +def test_verify_cloud_credential(): + with local_ftp_credential_data() as data: + assert call("cloudsync.credentials.verify", data)["valid"] + + +def test_verify_cloud_credential_fail(): + with local_ftp_credential_data() as data: + data["attributes"]["user"] = "root" + assert not call("cloudsync.credentials.verify", data)["valid"] diff --git a/tests/api2/test_cloud_sync_crud.py b/tests/api2/test_cloud_sync_crud.py new file mode 100644 index 0000000000000..d21184fc5b12b --- /dev/null +++ b/tests/api2/test_cloud_sync_crud.py @@ -0,0 +1,88 @@ +import pytest + +from middlewared.service_exception import CallError +from middlewared.test.integration.assets.cloud_sync import credential as _credential, task as _task +from middlewared.test.integration.assets.pool import dataset +from middlewared.test.integration.utils import call, ssh + +try: + from config import ( + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + AWS_BUCKET + ) +except ImportError: + Reason = 'AWS credential are missing in config.py' + pytestmark = pytest.mark.skip(reason=Reason) + + +@pytest.fixture(scope='module') +def credentials(): + with _credential({ + "provider": "S3", + "attributes": { + "access_key_id": AWS_ACCESS_KEY_ID, + "secret_access_key": AWS_SECRET_ACCESS_KEY, + } + }) as c: + yield c + + +@pytest.fixture(scope='module') +def task(credentials): + with dataset("cloudsync_local") as local_dataset: + with _task({ + "direction": "PUSH", + "transfer_mode": "COPY", + "path": f"/mnt/{local_dataset}", + "credentials": credentials["id"], + "attributes": { + "bucket": AWS_BUCKET, + "folder": "", + }, + }) as t: + yield t + + +def test_update_cloud_credentials(credentials): + call("cloudsync.credentials.update", credentials["id"], { + "attributes": { + "access_key_id": "garbage", + "secret_access_key": AWS_SECRET_ACCESS_KEY, + } + }) + + assert call("cloudsync.credentials.get_instance", credentials["id"])["attributes"]["access_key_id"] == "garbage" + + call("cloudsync.credentials.update", credentials["id"], { + "attributes": { + "access_key_id": AWS_ACCESS_KEY_ID, + "secret_access_key": AWS_SECRET_ACCESS_KEY, + }, + }) + + +def test_update_cloud_sync(task): + assert call("cloudsync.update", task["id"], {"direction": "PULL"}) + + +def test_run_cloud_sync(task): + call("cloudsync.sync", task["id"], job=True) + print(ssh(f"ls {task['path']}")) + assert ssh(f"cat {task['path']}/freenas-test.txt") == "freenas-test\n" + + +def test_restore_cloud_sync(task): + restore_task = call("cloudsync.restore", task["id"], { + "transfer_mode": "COPY", + "path": task["path"], + }) + + call("cloudsync.delete", restore_task["id"]) + + +def test_delete_cloud_credentials_error(credentials, task): + with pytest.raises(CallError) as ve: + call("cloudsync.credentials.delete", credentials["id"]) + + assert "This credential is used by cloud sync task" in ve.value.errmsg