test_: add multiple status-backend instances (#6104)

* test_: add multiple status-backend instances

* test_: reliable schemas

_assets/scripts/run_functional_tests.sh

@@ -26,9 +26,12 @@ mkdir -p "${test_results_path}"
 all_compose_files="-f ${root_path}/docker-compose.anvil.yml -f ${root_path}/docker-compose.test.status-go.yml"
 project_name="status-go-func-tests-$(date +%s)"
 
+export STATUS_BACKEND_COUNT=10
+export STATUS_BACKEND_URLS=$(eval echo http://${project_name}-status-backend-{1..${STATUS_BACKEND_COUNT}}:3333 | tr ' ' ,)
+
 # Run functional tests
 echo -e "${GRN}Running tests${RST}, HEAD: $(git rev-parse HEAD)"
-docker compose -p ${project_name} ${all_compose_files} up -d --build --remove-orphans
+docker compose -p ${project_name} ${all_compose_files} up -d --build --scale status-backend=${STATUS_BACKEND_COUNT} --remove-orphans
 
 echo -e "${GRN}Running tests-rpc${RST}" # Follow the logs, wait for them to finish
 docker compose -p ${project_name} ${all_compose_files} logs -f tests-rpc > "${root_path}/tests-rpc.log"

tests-functional/README.MD

@@ -25,7 +25,7 @@ Functional tests for status-go
     * Status-im contracts will be deployed to the network
 
 ### Run tests
-- In `./tests-functional` run `docker compose -f docker-compose.anvil.yml -f docker-compose.test.status-go.yml -f docker-compose.status-go.local.yml up --build --remove-orphans`, as result:
+- In `./tests-functional` run `docker compose -f docker-compose.anvil.yml -f docker-compose.test.status-go.yml -f docker-compose.status-go.local.yml up --build --scale status-backend=10 --remove-orphans`, as result:
     * a container with [status-go as daemon](https://github.com/status-im/status-go/issues/5175) will be created with APIModules exposed on `0.0.0.0:3333`
     * status-go will use [anvil](https://book.getfoundry.sh/reference/anvil/) as RPCURL with ChainID 31337 
     * all Status-im contracts will be deployed to the network

tests-functional/clients/rpc.py

@@ -0,0 +1,66 @@
+import json
+import logging
+import jsonschema
+import requests
+
+from conftest import option
+from json import JSONDecodeError
+
+class RpcClient:
+
+    def __init__(self, rpc_url, client=requests.Session()):
+        self.client = client
+        self.rpc_url = rpc_url
+
+    def _check_decode_and_key_errors_in_response(self, response, key):
+        try:
+            return response.json()[key]
+        except json.JSONDecodeError:
+            raise AssertionError(
+                f"Invalid JSON in response: {response.content}")
+        except KeyError:
+            raise AssertionError(
+                f"Key '{key}' not found in the JSON response: {response.content}")
+
+    def verify_is_valid_json_rpc_response(self, response, _id=None):
+        assert response.status_code == 200, f"Got response {response.content}, status code {response.status_code}"
+        assert response.content
+        self._check_decode_and_key_errors_in_response(response, "result")
+
+        if _id:
+            try:
+                if _id != response.json()["id"]:
+                    raise AssertionError(
+                        f"got id: {response.json()['id']} instead of expected id: {_id}"
+                    )
+            except KeyError:
+                raise AssertionError(f"no id in response {response.json()}")
+        return response
+
+    def verify_is_json_rpc_error(self, response):
+        assert response.status_code == 200
+        assert response.content
+        self._check_decode_and_key_errors_in_response(response, "error")
+
+    def rpc_request(self, method, params=[], request_id=13, url=None):
+        url = url if url else self.rpc_url
+        data = {"jsonrpc": "2.0", "method": method, "id": request_id}
+        if params:
+            data["params"] = params
+        logging.info(f"Sending POST request to url {url} with data: {json.dumps(data, sort_keys=True, indent=4)}")
+        response = self.client.post(url, json=data)
+        try:
+            logging.info(f"Got response: {json.dumps(response.json(), sort_keys=True, indent=4)}")
+        except JSONDecodeError:
+            logging.info(f"Got response: {response.content}")
+        return response
+
+    def rpc_valid_request(self, method, params=[], _id=None, url=None):
+        response = self.rpc_request(method, params, _id, url)
+        self.verify_is_valid_json_rpc_response(response, _id)
+        return response
+
+    def verify_json_schema(self, response, method):
+        with open(f"{option.base_dir}/schemas/{method}", "r") as schema:
+            jsonschema.validate(instance=response,
+                                schema=json.load(schema))

tests-functional/clients/status_backend.py

@@ -1,88 +1,39 @@
 import json
 import logging
 import time
-from datetime import datetime
-from json import JSONDecodeError
-
-import jsonschema
+import random
+import threading
 import requests
 
 from clients.signals import SignalClient
+from clients.rpc import RpcClient
+from datetime import datetime
 from conftest import option
 from constants import user_1
 
 
-class RpcClient:
-
-    def __init__(self, rpc_url, client=requests.Session()):
-        self.client = client
-        self.rpc_url = rpc_url
-
-    def _check_decode_and_key_errors_in_response(self, response, key):
-        try:
-            return response.json()[key]
-        except json.JSONDecodeError:
-            raise AssertionError(
-                f"Invalid JSON in response: {response.content}")
-        except KeyError:
-            raise AssertionError(
-                f"Key '{key}' not found in the JSON response: {response.content}")
-
-    def verify_is_valid_json_rpc_response(self, response, _id=None):
-        assert response.status_code == 200, f"Got response {response.content}, status code {response.status_code}"
-        assert response.content
-        self._check_decode_and_key_errors_in_response(response, "result")
 
-        if _id:
-            try:
-                if _id != response.json()["id"]:
-                    raise AssertionError(
-                        f"got id: {response.json()['id']} instead of expected id: {_id}"
-                    )
-            except KeyError:
-                raise AssertionError(f"no id in response {response.json()}")
-        return response
-
-    def verify_is_json_rpc_error(self, response):
-        assert response.status_code == 200
-        assert response.content
-        self._check_decode_and_key_errors_in_response(response, "error")
+class StatusBackend(RpcClient, SignalClient):
 
-    def rpc_request(self, method, params=[], request_id=13, url=None):
-        url = url if url else self.rpc_url
-        data = {"jsonrpc": "2.0", "method": method, "id": request_id}
-        if params:
-            data["params"] = params
-        logging.info(f"Sending POST request to url {url} with data: {json.dumps(data, sort_keys=True, indent=4)}")
-        response = self.client.post(url, json=data)
+    def __init__(self, await_signals=[], url=None):
         try:
-            logging.info(f"Got response: {json.dumps(response.json(), sort_keys=True, indent=4)}")
-        except JSONDecodeError:
-            logging.info(f"Got response: {response.content}")
-        return response
+            url = url if url else random.choice(option.status_backend_urls)
+        except IndexError:
+            raise Exception("Not enough status-backend containers, please add more")
+        option.status_backend_urls.remove(url)
 
-    def rpc_valid_request(self, method, params=[], _id=None, url=None):
-        response = self.rpc_request(method, params, _id, url)
-        self.verify_is_valid_json_rpc_response(response, _id)
-        return response
-
-    def verify_json_schema(self, response, method):
-        with open(f"{option.base_dir}/schemas/{method}", "r") as schema:
-            jsonschema.validate(instance=response,
-                                schema=json.load(schema))
-
-
-class StatusBackend(RpcClient, SignalClient):
-
-    def __init__(self, await_signals=list()):
+        self.api_url = f"{url}/statusgo"
+        self.ws_url = f"{url}".replace("http", "ws")
+        self.rpc_url = f"{url}/statusgo/CallRPC"
 
-        self.api_url = f"{option.rpc_url_status_backend}/statusgo"
-        self.ws_url = f"{option.ws_url_status_backend}"
-        self.rpc_url = f"{option.rpc_url_status_backend}/statusgo/CallRPC"
 
         RpcClient.__init__(self, self.rpc_url)
         SignalClient.__init__(self, self.ws_url, await_signals)
 
+        websocket_thread = threading.Thread(target=self._connect)
+        websocket_thread.daemon = True
+        websocket_thread.start()
+
     def api_request(self, method, data, url=None):
         url = url if url else self.api_url
         url = f"{url}/{method}"

tests-functional/conftest.py

@@ -12,23 +12,21 @@ def pytest_addoption(parser):
         help="",
         default="http://0.0.0.0:3333",
     )
-    parser.addoption(
-        "--rpc_url_status_backend",
-        action="store",
-        help="",
-        default="http://0.0.0.0:3334",
-    )
     parser.addoption(
         "--ws_url_statusd",
         action="store",
         help="",
         default="ws://0.0.0.0:8354",
     )
     parser.addoption(
-        "--ws_url_status_backend",
+        "--status_backend_urls",
         action="store",
         help="",
-        default="ws://0.0.0.0:3334",
+        default=[
+            f"http://0.0.0.0:{3314 + i}" for i in range(
+                int(os.getenv("STATUS_BACKEND_COUNT", 10))
+            )
+        ],
     )
     parser.addoption(
         "--anvil_url",
@@ -43,7 +41,6 @@ def pytest_addoption(parser):
         default="Strong12345",
     )
 
-
 @dataclass
 class Option:
     pass
@@ -55,33 +52,6 @@ class Option:
 def pytest_configure(config):
     global option
     option = config.option
+    if type(option.status_backend_urls) is str:
+        option.status_backend_urls = option.status_backend_urls.split(",")
     option.base_dir = os.path.dirname(os.path.abspath(__file__))
-
-
-@pytest.fixture(scope="session", autouse=True)
-def init_status_backend():
-    await_signals = [
-
-        "mediaserver.started",
-        "node.started",
-        "node.ready",
-        "node.login",
-
-        "wallet",  # TODO: a test per event of a different type
-    ]
-
-    from clients.status_backend import StatusBackend
-    backend_client = StatusBackend(
-        await_signals=await_signals
-    )
-
-    websocket_thread = threading.Thread(
-        target=backend_client._connect
-    )
-    websocket_thread.daemon = True
-    websocket_thread.start()
-
-    backend_client.init_status_backend()
-    backend_client.restore_account_and_wait_for_rpc_client_to_start()
-
-    yield backend_client

tests-functional/docker-compose.status-go.local.yml

@@ -8,4 +8,4 @@ services:
       - 8354:8354
   status-backend:
     ports:
-      - 3334:3333
+      - 3314-3324:3333

tests-functional/docker-compose.test.status-go.yml

@@ -68,9 +68,8 @@ services:
       "-m", "rpc",
       "--anvil_url=http://anvil:8545",
       "--rpc_url_statusd=http://status-go:3333",
-      "--rpc_url_status_backend=http://status-backend:3333",
+      "--status_backend_urls=${STATUS_BACKEND_URLS}",
       "--ws_url_statusd=ws://status-go:8354",
-      "--ws_url_status_backend=ws://status-backend:3333",
       "--junitxml=/tests-rpc/reports/report.xml"
     ]
     volumes: