NAS-129790 / 24.10 / Fix cloudsync.credentials.verify (#13956)

* Fix `cloudsync.credentials.verify`

* Fix cloud sync tests

* Type hints

src/middlewared/middlewared/api/base/decorator.py

@@ -1,13 +1,22 @@
 import asyncio
 import functools
+from typing import Callable
 
 from .handler.accept import accept_params
+from ..base.model import BaseModel
 from middlewared.schema.processor import calculate_args_index
 
 __all__ = ["api_method"]
 
 
-def api_method(accepts, returns, audit=None, audit_callback=False, audit_extended=None, roles=None):
+def api_method(
+    accepts: type[BaseModel],
+    returns: type[BaseModel],
+    audit: str | None = None,
+    audit_callback: bool = False,
+    audit_extended: Callable[..., str] | None = None,
+    roles: list[str] | None = None,
+):
     """
     Mark a `Service` class method as a public API method.
 
@@ -25,6 +34,9 @@ def api_method(accepts, returns, audit=None, audit_callback=False, audit_extende
 
     `roles` is a list of user roles that will gain access to this method.
     """
+    if list(returns.model_fields.keys()) != ["result"]:
+        raise TypeError("`returns` model must only have one field called `result`")
+
     def wrapper(func):
         args_index = calculate_args_index(func, audit_callback)
 

src/middlewared/middlewared/api/v25_04_0/cloud_sync.py

@@ -1,4 +1,5 @@
-from middlewared.api.base import BaseModel, Excluded, excluded_field, ForUpdateMetaclass, NonEmptyString, Private
+from middlewared.api.base import (BaseModel, Excluded, excluded_field, ForUpdateMetaclass, NonEmptyString, Private,
+                                  single_argument_args, single_argument_result)
 
 __all__ = ["CloudCredentialEntry",
            "CloudCredentialCreateArgs", "CloudCredentialCreateResult",
@@ -47,11 +48,13 @@ class CloudCredentialDeleteResult(BaseModel):
     result: bool
 
 
+@single_argument_args("cloud_sync_credentials_create")
 class CloudCredentialVerifyArgs(BaseModel):
     provider: str
     attributes: Private[dict]
 
 
+@single_argument_result
 class CloudCredentialVerifyResult(BaseModel):
     valid: bool
     error: str | None = None

src/middlewared/middlewared/test/integration/assets/cloud_sync.py

@@ -43,17 +43,23 @@ def task(data):
 
 
 @contextlib.contextmanager
-def local_ftp_credential():
+def local_ftp_credential_data():
     with anonymous_ftp_server(dataset_name="cloudsync_remote") as ftp:
-        with credential({
+        yield {
             "provider": "FTP",
             "attributes": {
                 "host": "localhost",
                 "port": 21,
                 "user": ftp.username,
                 "pass": ftp.password,
             },
-        }) as c:
+        }
+
+
+@contextlib.contextmanager
+def local_ftp_credential():
+    with local_ftp_credential_data() as data:
+        with credential(data) as c:
             yield c
 
 

src/middlewared/middlewared/test/integration/utils/client.py

@@ -103,6 +103,8 @@ def client(self) -> Client:
                 self._client.close()
                 self._client = None
 
+        host()  # Has to be called in order for `truenas_server` global variable to be correctly initialized when
+                # running `runtest.py` with a single test name
         if (addr := self.ip) is None:
             raise RuntimeError('IP is not set')
 

tests/api2/test_cloud_sync.py

@@ -2,8 +2,9 @@
 import time
 
 import pytest
-from middlewared.test.integration.assets.cloud_sync import credential, task, local_ftp_credential
-from middlewared.test.integration.assets.cloud_sync import local_ftp_task, run_task
+from middlewared.test.integration.assets.cloud_sync import (
+    credential, task, local_ftp_credential, local_ftp_task, run_task,
+)
 from middlewared.test.integration.assets.ftp import anonymous_ftp_server, ftp_server_with_user_account
 from middlewared.test.integration.assets.pool import dataset
 from middlewared.test.integration.utils import call, pool, ssh

tests/api2/test_cloud_sync_credentials.py

@@ -0,0 +1,13 @@
+from middlewared.test.integration.assets.cloud_sync import local_ftp_credential_data
+from middlewared.test.integration.utils import call
+
+
+def test_verify_cloud_credential():
+    with local_ftp_credential_data() as data:
+        assert call("cloudsync.credentials.verify", data)["valid"]
+
+
+def test_verify_cloud_credential_fail():
+    with local_ftp_credential_data() as data:
+        data["attributes"]["user"] = "root"
+        assert not call("cloudsync.credentials.verify", data)["valid"]

tests/api2/test_cloud_sync_crud.py

@@ -0,0 +1,88 @@
+import pytest
+
+from middlewared.service_exception import CallError
+from middlewared.test.integration.assets.cloud_sync import credential as _credential, task as _task
+from middlewared.test.integration.assets.pool import dataset
+from middlewared.test.integration.utils import call, ssh
+
+try:
+    from config import (
+        AWS_ACCESS_KEY_ID,
+        AWS_SECRET_ACCESS_KEY,
+        AWS_BUCKET
+    )
+except ImportError:
+    Reason = 'AWS credential are missing in config.py'
+    pytestmark = pytest.mark.skip(reason=Reason)
+
+
+@pytest.fixture(scope='module')
+def credentials():
+    with _credential({
+        "provider": "S3",
+        "attributes": {
+            "access_key_id": AWS_ACCESS_KEY_ID,
+            "secret_access_key": AWS_SECRET_ACCESS_KEY,
+        }
+    }) as c:
+        yield c
+
+
+@pytest.fixture(scope='module')
+def task(credentials):
+    with dataset("cloudsync_local") as local_dataset:
+        with _task({
+            "direction": "PUSH",
+            "transfer_mode": "COPY",
+            "path": f"/mnt/{local_dataset}",
+            "credentials": credentials["id"],
+            "attributes": {
+                "bucket": AWS_BUCKET,
+                "folder": "",
+            },
+        }) as t:
+            yield t
+
+
+def test_update_cloud_credentials(credentials):
+    call("cloudsync.credentials.update", credentials["id"], {
+        "attributes": {
+            "access_key_id": "garbage",
+            "secret_access_key": AWS_SECRET_ACCESS_KEY,
+        }
+    })
+
+    assert call("cloudsync.credentials.get_instance", credentials["id"])["attributes"]["access_key_id"] == "garbage"
+
+    call("cloudsync.credentials.update", credentials["id"], {
+        "attributes": {
+            "access_key_id": AWS_ACCESS_KEY_ID,
+            "secret_access_key": AWS_SECRET_ACCESS_KEY,
+        },
+    })
+
+
+def test_update_cloud_sync(task):
+    assert call("cloudsync.update", task["id"], {"direction": "PULL"})
+
+
+def test_run_cloud_sync(task):
+    call("cloudsync.sync", task["id"], job=True)
+    print(ssh(f"ls {task['path']}"))
+    assert ssh(f"cat {task['path']}/freenas-test.txt") == "freenas-test\n"
+
+
+def test_restore_cloud_sync(task):
+    restore_task = call("cloudsync.restore", task["id"], {
+        "transfer_mode": "COPY",
+        "path": task["path"],
+    })
+
+    call("cloudsync.delete", restore_task["id"])
+
+
+def test_delete_cloud_credentials_error(credentials, task):
+    with pytest.raises(CallError) as ve:
+        call("cloudsync.credentials.delete", credentials["id"])
+
+    assert "This credential is used by cloud sync task" in ve.value.errmsg