Integrate autobahn tests with pytest (#5809)

* Integrate autobahn tests with pytest

* Fix docker compose file paths

* Fix typo in CHANGES file

* Fix add python-on-whales dependency to .in file instead of .txt

* Use pathlib instead of os

* Use buildx instead of compose build

* Regenerate dev requirements

* Rename changes file

* Use request fspath instead of hard coded path

* Create a sepearte builder when building aiohttp

* Use subprocess instead of python-on-whales

* Extract failed tests and make assertions on them

* Fix lint issues

* Fix fixture scope

* Add ports to docker-compose files

* Add wait-for-it package

* Use xfail instead of fail

* Use wstest cmd tool instead of the docker image

* Fix lint issues

* Use assert statement with custom output

Co-authored-by: Sviatoslav Sydorenko <[email protected]>

* Code cleanup

* Use docker instead of docker-compose

* Add xfail decorator

* Add tmp_path

* Remove gitignore

* Skip tests only on macOS

* Check if docker is available

* Regenerate dev.txt

Co-authored-by: Sviatoslav Sydorenko <[email protected]>

.mypy.ini

@@ -41,3 +41,6 @@ ignore_missing_imports = True
 
 [mypy-uvloop]
 ignore_missing_imports = True
+
+[mypy-python_on_whales]
+ignore_missing_imports = True

CHANGES/4247.1.misc

@@ -0,0 +1 @@
+Automated running autobahn test suite by integrating with pytest.

requirements/dev.in

@@ -2,3 +2,5 @@
 -r test.txt
 -r doc.txt
 cherry_picker==2.0.0; python_version>="3.6"
+python-on-whales==0.19.0
+wait-for-it==2.2.0

requirements/dev.txt

@@ -59,6 +59,8 @@ click==7.1.2
     #   cherry-picker
     #   click-default-group
     #   towncrier
+    #   typer
+    #   wait-for-it
 click-default-group==1.2.2
     # via towncrier
 coverage[toml]==6.0
@@ -177,6 +179,8 @@ pycodestyle==2.7.0
     #   flake8
 pycparser==2.20
     # via cffi
+pydantic==1.8.2
+    # via python-on-whales
 pyflakes==2.3.0
     # via
     #   -r requirements/lint.txt
@@ -204,6 +208,8 @@ pytest-mock==3.6.1
     # via -r requirements/test.txt
 python-dateutil==2.8.1
     # via freezegun
+python-on-whales==0.19.0
+    # via -r requirements/dev.in
 pytz==2020.5
     # via babel
 pyyaml==5.4.1
@@ -220,6 +226,7 @@ regex==2020.11.13
 requests==2.25.1
     # via
     #   cherry-picker
+    #   python-on-whales
     #   sphinx
 setuptools-git==1.2
     # via -r requirements/test.txt
@@ -272,8 +279,12 @@ towncrier==21.3.0
     # via
     #   -r requirements/doc.txt
     #   sphinxcontrib-towncrier
+tqdm==4.62.2
+    # via python-on-whales
 trustme==0.9.0 ; platform_machine != "i686"
     # via -r requirements/test.txt
+typer==0.4.0
+    # via python-on-whales
 types-chardet==0.1.3
     # via
     #   -r requirements/lint.txt
@@ -285,6 +296,7 @@ typing-extensions==3.7.4.3
     #   async-timeout
     #   mypy
     #   proxy.py
+    #   pydantic
 uritemplate==3.0.1
     # via gidgethub
 urllib3==1.26.5
@@ -293,6 +305,8 @@ virtualenv==20.4.2
     # via
     #   -r requirements/lint.txt
     #   pre-commit
+wait-for-it==2.2.0
+    # via -r requirements/dev.in
 webcolors==1.11.1
     # via blockdiag
 yarl==1.6.3

tests/autobahn/Dockerfile.autobahn

@@ -0,0 +1,6 @@
+FROM crossbario/autobahn-testsuite:0.8.2
+
+RUN apt-get update && apt-get install python3 python3-pip -y
+RUN pip3 install wait-for-it
+
+CMD ["wstest", "--mode", "fuzzingserver", "--spec", "/config/fuzzingserver.json"]

tests/autobahn/client/client.py

@@ -38,4 +38,4 @@ async def run(url: str, name: str) -> None:
 
 
 if __name__ == "__main__":
-    asyncio.run(run("http://autobahn:9001", "aiohttp"))
+    asyncio.run(run("http://localhost:9001", "aiohttp"))

tests/autobahn/server/fuzzingclient.json

@@ -5,7 +5,7 @@
     "servers": [
         {
             "agent": "AutobahnServer",
-            "url": "ws://aiohttp:9001",
+            "url": "ws://localhost:9001",
             "options": { "version": 18 }
         }
     ],

tests/autobahn/server/server.py

@@ -13,6 +13,8 @@ async def wshandler(request: web.Request) -> web.WebSocketResponse:
 
     await ws.prepare(request)
 
+    request.app["websockets"].append(ws)
+
     while True:
         msg = await ws.receive()
 
@@ -40,6 +42,7 @@ async def on_shutdown(app: web.Application) -> None:
     )
 
     app = web.Application()
+    app["websockets"] = []
     app.router.add_route("GET", "/", wshandler)
     app.on_shutdown.append(on_shutdown)
     try:

tests/autobahn/test_autobahn.py

@@ -0,0 +1,132 @@
+import json
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any, Dict, Generator, List
+
+import pytest
+from pytest import TempPathFactory
+from python_on_whales import DockerException, docker
+
+
+@pytest.fixture(scope="session")
+def report_dir(tmp_path_factory: TempPathFactory) -> Path:
+    return tmp_path_factory.mktemp("reports")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def build_autobahn_testsuite() -> Generator[None, None, None]:
+
+    try:
+        docker.build(
+            file="tests/autobahn/Dockerfile.autobahn",
+            tags=["autobahn-testsuite"],
+            context_path=".",
+        )
+    except DockerException:
+        pytest.skip(msg="The docker daemon is not running.")
+
+    try:
+        yield
+    finally:
+        docker.image.remove(x="autobahn-testsuite")
+
+
+def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]:
+    path = Path(report_path)
+    result_summary = json.loads((path / "index.json").read_text())[name]
+    failed_messages = []
+    PASS = {"OK", "INFORMATIONAL"}
+    entry_fields = {"case", "description", "expectation", "expected", "received"}
+    for results in result_summary.values():
+        if results["behavior"] in PASS and results["behaviorClose"] in PASS:
+            continue
+        report = json.loads((path / results["reportfile"]).read_text())
+        failed_messages.append({field: report[field] for field in entry_fields})
+    return failed_messages
+
+
+@pytest.mark.skipif(sys.platform == "darwin", reason="Don't run on macOS")
+@pytest.mark.xfail
+def test_client(report_dir: Path, request: Any) -> None:
+    try:
+        print("Starting autobahn-testsuite server")
+        autobahn_container = docker.run(
+            detach=True,
+            image="autobahn-testsuite",
+            name="autobahn",
+            publish=[(9001, 9001)],
+            remove=True,
+            volumes=[
+                (f"{request.fspath.dirname}/client", "/config"),
+                (f"{report_dir}", "/reports"),
+            ],
+        )
+        print("Running aiohttp test client")
+        client = subprocess.Popen(
+            ["wait-for-it", "-s", "localhost:9001", "--"]
+            + [sys.executable]
+            + ["tests/autobahn/client/client.py"]
+        )
+        client.wait()
+    finally:
+        print("Stopping client and server")
+        client.terminate()
+        client.wait()
+        autobahn_container.stop()
+
+    failed_messages = get_failed_tests(f"{report_dir}/clients", "aiohttp")
+
+    assert not failed_messages, "\n".join(
+        "\n\t".join(
+            f"{field}: {msg[field]}"
+            for field in ("case", "description", "expectation", "expected", "received")
+        )
+        for msg in failed_messages
+    )
+
+
+@pytest.mark.skipif(sys.platform == "darwin", reason="Don't run on macOS")
+@pytest.mark.xfail
+def test_server(report_dir: Path, request: Any) -> None:
+    try:
+        print("Starting aiohttp test server")
+        server = subprocess.Popen(
+            [sys.executable] + ["tests/autobahn/server/server.py"]
+        )
+        print("Starting autobahn-testsuite client")
+        docker.run(
+            image="autobahn-testsuite",
+            name="autobahn",
+            remove=True,
+            volumes=[
+                (f"{request.fspath.dirname}/server", "/config"),
+                (f"{report_dir}", "/reports"),
+            ],
+            networks=["host"],
+            command=[
+                "wait-for-it",
+                "-s",
+                "localhost:9001",
+                "--",
+                "wstest",
+                "--mode",
+                "fuzzingclient",
+                "--spec",
+                "/config/fuzzingclient.json",
+            ],
+        )
+    finally:
+        print("Stopping client and server")
+        server.terminate()
+        server.wait()
+
+    failed_messages = get_failed_tests(f"{report_dir}/servers", "AutobahnServer")
+
+    assert not failed_messages, "\n".join(
+        "\n\t".join(
+            f"{field}: {msg[field]}"
+            for field in ("case", "description", "expectation", "expected", "received")
+        )
+        for msg in failed_messages
+    )