Implement tests for lsps0 and bug-fixes

I've implemented an integration test for LSPS0 which replicates the
list-protocols behavior.

To ensure our test-case is reproducible I have removed the dependency on
`generate_random_rpc_id`. When writing tests you can use
`rpc_call_with_id`. In production the use of `rpc_call` is recommended.

This split is present in multiple methods for the creation of
JsonRpcRequests.

Previously, I made the mistake of misnaming the `jsonrpc` field.
The spurious underscore is removed everywhere.

I used the

```serde_json::from_value()```

in transport.rs when deserializing a JSON-rpc response. This did not
work as expected when the type has been erased.

I replaced this to
```
method.parse_json_response_value(,,.)
```

See transport.rs line 139. This ensures the underlying types are checked
even when a type-erased version is used.

libs/gl-client-py/Cargo.toml

@@ -20,3 +20,4 @@ prost = "0.11"
 pyo3 = {version = "0.18", features = ["extension-module", "serde", "abi3-py37"]}
 tokio = { version = "1", features = ["full"] }
 tonic = { version = "^0.8", features = ["tls", "transport"] }
+serde_json = "^1.0"

libs/gl-client-py/glclient/__init__.py

@@ -9,6 +9,7 @@
 from binascii import hexlify, unhexlify
 from typing import Optional, List, Union, Iterable, Any, Type, TypeVar
 import logging
+from glclient.lsps import LspClient
 
 
 # Keep in sync with the libhsmd version, this is tested in unit tests.
@@ -551,6 +552,13 @@ def list_datastore(
             bytes(self.inner.call(uri, bytes(req)))
         )
 
+    def get_lsp_client(
+        self,
+        peer_id : bytes
+    ) -> LspClient:
+        native_lsps = self.inner.get_lsp_client()
+        return LspClient(native_lsps, peer_id)
+
 
 def normalize_node_id(node_id, string=False):
     if len(node_id) == 66:

libs/gl-client-py/glclient/glclient.pyi

@@ -1,3 +1,5 @@
+import typing as t
+
 class TlsConfig:
     def __init__(self) -> None: ...
     def with_ca_certificate(self, ca: bytes) -> "TlsConfig": ...
@@ -24,6 +26,19 @@ class Scheduler:
     def schedule(self) -> bytes: ...
 
 class Node:
-    def __init__(self, node_id: bytes, network: str, tls: TlsConfig, grpc_uri: str) -> None: ...
+    def __init__(
+        self, node_id: bytes, network: str, tls: TlsConfig, grpc_uri: str
+    ) -> None: ...
     def stop(self) -> None: ...
     def call(self, method: str, request: bytes) -> bytes: ...
+    def get_lsp_client(self) -> LspClient: ...
+
+class LspClient:
+    def rpc_call(self, peer_id: bytes, method: str, params: bytes) -> bytes: ...
+    def rpc_call_with_json_rpc_id(
+        self,
+        peer_id: bytes,
+        method: str,
+        params: bytes,
+        json_rpc_id: t.Optional[str] = None,
+    ) -> bytes: ...

libs/gl-client-py/glclient/lsps.py

@@ -0,0 +1,119 @@
+from dataclasses import dataclass, is_dataclass, asdict, field
+
+import typing as t
+import json
+import time
+import binascii
+
+import glclient.glclient as native
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def parse_and_validate_peer_id(data: t.Union[str, bytes]) -> bytes:
+    if isinstance(data, bytes):
+        if len(data) == 33:
+            return data
+        else:
+            raise ValueError(
+                f"Invalid peer_id. Expected a byte-array of length 33 but received {len(data)} instead"
+            )
+    if isinstance(data, str):
+        if len(data) != 66:
+            raise ValueError(
+                f"Invalid peer_id. Must be a length 66 hex-string but received {len(data)}"
+            )
+        try:
+            return bytes.fromhex(data)
+        except Exception as e:
+            raise ValueError("Invalid peer_id. Failed to parse hex-string") from e
+
+
+class EnhancedJSONEncoder(json.JSONEncoder):
+    def default(self, o):
+        if is_dataclass(o):
+            return asdict(o)
+        elif isinstance(o, NoParams):
+            return dict()
+        elif isinstance(o, type) and o.__name__ == "NoParams":
+            return dict()
+        return super().default(o)
+
+
+class AsDataClassDescriptor:
+    """Descriptor that allows to initialize a nested dataclass from a nested directory"""
+
+    def __init__(self, *, cls):
+        self._cls = cls
+
+    def __set_name__(self, owner, name):
+        self._name = f"_{name}"
+
+    def __get__(self, obj, type):
+        return getattr(obj, self._name, None)
+
+    def __set__(self, obj, value):
+        if isinstance(value, self._cls):
+            setattr(obj, self._name, value)
+        else:
+            setattr(obj, self._name, self._cls(**value))
+
+
+def _dump_json_bytes(object: t.Any) -> bytes:
+    json_str: str = json.dumps(object, cls=EnhancedJSONEncoder)
+    json_bytes: bytes = json_str.encode("utf-8")
+    return json_bytes
+
+
+@dataclass
+class ProtocolList:
+    protocols: t.List[int]
+
+
+@dataclass
+class Lsps1Options:
+    minimum_channel_confirmations: t.Optional[int]
+    minimum_onchain_payment_confirmations: t.Optional[int]
+    supports_zero_channel_reserve: t.Optional[bool]
+    min_onchain_payment_size_sat: t.Optional[int]
+    max_channel_expiry_blocks: t.Optional[int]
+    min_initial_client_balance_sat: t.Optional[int]
+    min_initial_lsp_balance_sat: t.Optional[int]
+    max_initial_client_balance_sat: t.Optional[int]
+    min_channel_balance_sat: t.Optional[int]
+    max_channel_balance_sat: t.Optional[int]
+
+
+class NoParams:
+    pass
+
+
+class LspClient:
+    def __init__(self, native: native.LspClient, peer_id: t.Union[bytes, str]):
+        self._native = native
+        self._peer_id: bytes = parse_and_validate_peer_id(peer_id)
+
+    def _rpc_call(
+        self,
+        peer_id: bytes,
+        method_name: str,
+        param_json: bytes,
+        json_rpc_id: t.Optional[str] = None,
+    ) -> bytes:
+        logger.debug("Request lsp to peer %s and method %s", peer_id, method_name)
+        if json_rpc_id is None:
+            return self._native.rpc_call(peer_id, method_name, param_json)
+        else:
+            return self._native.rpc_call_with_json_rpc_id(
+                peer_id, method_name, param_json, json_rpc_id=json_rpc_id
+            )
+
+    def list_protocols(self, json_rpc_id: t.Optional[str] = None) -> ProtocolList:
+        json_bytes = _dump_json_bytes(NoParams)
+        result = self._rpc_call(
+            self._peer_id, "lsps0.listprotocols", json_bytes, json_rpc_id=json_rpc_id
+        )
+        response_dict = json.loads(result)
+        return ProtocolList(**response_dict)

libs/gl-client-py/src/lib.rs

@@ -9,11 +9,14 @@ mod runtime;
 mod scheduler;
 mod signer;
 mod tls;
+mod lsps;
 
 pub use node::Node;
 pub use scheduler::Scheduler;
 pub use signer::Signer;
 pub use tls::TlsConfig;
+pub use lsps::LspClient;
+
 
 #[pyfunction]
 pub fn backup_decrypt_with_seed(encrypted: Vec<u8>, seed: Vec<u8>) -> PyResult<Vec<u8>> {
@@ -36,6 +39,7 @@ fn glclient(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<Node>()?;
     m.add_class::<Scheduler>()?;
     m.add_class::<TlsConfig>()?;
+    m.add_class::<LspClient>()?;
 
     m.add_function(wrap_pyfunction!(backup_decrypt_with_seed, m)?)?;
 

libs/gl-client-py/src/lsps.rs

@@ -0,0 +1,93 @@
+use crate::runtime::exec;
+use gl_client::lsps::error::LspsError;
+use gl_client::lsps::json_rpc::{JsonRpcResponse, generate_random_rpc_id};
+use gl_client::lsps::message as lsps_message;
+use gl_client::lsps::transport::JsonRpcTransport;
+use gl_client::node::{Client, ClnClient};
+use pyo3::exceptions::{PyBaseException, PyConnectionError, PyTimeoutError, PyValueError};
+use pyo3::prelude::*;
+use pyo3::PyErr;
+use pyo3::types::PyBytes;
+
+#[pyclass]
+pub struct LspClient {
+    transport: JsonRpcTransport,
+}
+
+impl LspClient {
+    pub fn new(client: Client, cln_client: ClnClient) -> Self {
+        LspClient {
+            transport: JsonRpcTransport::new(client, cln_client),
+        }
+    }
+}
+
+fn lsps_err_to_py_err(err: &LspsError) -> PyErr {
+    match err {
+        LspsError::MethodUnknown(method_name) => {
+            PyValueError::new_err(format!("Unknown method {:?}", method_name))
+        }
+        LspsError::ConnectionClosed => PyConnectionError::new_err("Failed to connect"),
+        LspsError::GrpcError(status) => PyConnectionError::new_err(String::from(status.message())),
+        LspsError::Timeout => PyTimeoutError::new_err("Did not receive a response from the LSPS"),
+        LspsError::JsonParseRequestError(error) => {
+            PyValueError::new_err(format!("Failed to parse json-request, {:}", error))
+        }
+        LspsError::JsonParseResponseError(error) => {
+            PyValueError::new_err(format!("Failed to parse json-response, {:}", error))
+        }
+        LspsError::Other(error_message) => PyBaseException::new_err(String::from(error_message)),
+    }
+}
+
+#[pymethods]
+impl LspClient {
+    // When doing ffi with python we'de like to keep the interface as small as possible.
+    //
+    // We already have JSON-serialization and deserialization working because the underlying protocol uses JSON-rpc
+    //
+    // When one of the JSON-rpc method is called from python the user can just specify the peer-id and the serialized parameter they want to send
+    // The serialized result will be returned
+    pub fn rpc_call(
+        &mut self,
+        py : Python,
+        peer_id: &[u8],
+        method_name: &str,
+        value: &[u8],
+    ) -> PyResult<PyObject> {
+        let json_rpc_id = generate_random_rpc_id();
+        self.rpc_call_with_json_rpc_id(py, peer_id, method_name, value, json_rpc_id)
+    }
+
+
+    pub fn rpc_call_with_json_rpc_id(
+        &mut self,
+        py : Python,
+        peer_id: &[u8],
+        method_name: &str,
+        value: &[u8],
+        json_rpc_id : String
+    ) -> PyResult<PyObject> {
+        // Parse the method-name and call the rpc-request
+        let rpc_response: JsonRpcResponse<Vec<u8>, Vec<u8>> =
+            lsps_message::JsonRpcMethodEnum::from_method_name(method_name)
+                .and_then(|method| exec(self.transport.request_with_json_rpc_id(peer_id, &method, value.to_vec(), json_rpc_id)))
+                .map_err(|err| lsps_err_to_py_err(&err))?;
+
+        match rpc_response {
+            JsonRpcResponse::Ok(ok) => {
+                let response = ok.result;                               // response as byte-array
+                let py_object : PyObject = PyBytes::new(py, &response).into();
+                return Ok(py_object)
+            }
+            JsonRpcResponse::Error(err) => {
+                // We should be able to put the error-data in here
+                // Replace this by a custom exception type
+                return Err(PyBaseException::new_err(format!(
+                    "{:?} - {:?}",
+                    err.error.code, err.error.message
+                )));
+            }
+        }
+    }
+}

libs/gl-client-py/src/node.rs

@@ -1,5 +1,6 @@
 use crate::runtime::exec;
 use crate::tls::TlsConfig;
+use crate::lsps::LspClient;
 use gl_client as gl;
 use gl_client::bitcoin::Network;
 use gl_client::pb;
@@ -12,6 +13,7 @@ use tonic::{Code, Status};
 pub struct Node {
     client: gl::node::Client,
     gclient: gl::node::GClient,
+    cln_client : gl::node::ClnClient
 }
 
 #[pymethods]
@@ -29,21 +31,24 @@ impl Node {
 
         // TODO: Could be massively simplified by using a scoped task
         // from tokio_scoped to a
-        let (client, gclient) = exec(async {
+        let (client, gclient, cln_client, ) = exec(async {
             let i = inner.clone();
             let u = grpc_uri.clone();
             let h1 = tokio::spawn(async move { i.connect(u).await });
             let i = inner.clone();
             let u = grpc_uri.clone();
             let h2 = tokio::spawn(async move { i.connect(u).await });
+            let i = inner.clone();
+            let u = grpc_uri.clone();
+            let h3 = tokio::spawn(async move { i.connect(u).await });
 
-            Ok::<(gl::node::Client, gl::node::GClient), anyhow::Error>((h1.await??, h2.await??))
+            Ok::<(gl::node::Client, gl::node::GClient, gl::node::ClnClient), anyhow::Error>((h1.await??, h2.await??, h3.await??))
         })
         .map_err(|e| {
             pyo3::exceptions::PyValueError::new_err(format!("could not connect to node: {}", e))
         })?;
 
-        Ok(Node { client, gclient })
+        Ok(Node { client, gclient, cln_client })
     }
 
     fn call(&self, method: &str, payload: Vec<u8>) -> PyResult<Vec<u8>> {
@@ -77,6 +82,13 @@ impl Node {
             .map_err(error_starting_stream)?;
         Ok(CustommsgStream { inner: stream })
     }
+
+    fn get_lsp_client(&self) -> LspClient {
+        LspClient::new(
+            self.client.clone(),
+            self.cln_client.clone()
+        )
+    }
 }
 
 fn error_decoding_request<D: core::fmt::Display>(e: D) -> PyErr {