Refactor mock webserver into a fixture

pyproject.toml

@@ -114,6 +114,7 @@ module = [
     "core.util.worker_pools",
     "core.util.xmlparser",
     "tests.fixtures.authenticator",
+    "tests.fixtures.webserver",
     "tests.migration.*",
 ]
 no_implicit_reexport = true

tests/api/test_overdrive.py

@@ -59,9 +59,9 @@
 from core.util.http import BadResponseException
 from tests.api.mockapi.overdrive import MockOverdriveAPI
 from tests.core.mock import DummyHTTPClient, MockRequestsResponse
-from tests.core.util.test_mock_web_server import MockAPIServer, MockAPIServerResponse
 from tests.fixtures.database import DatabaseTransactionFixture
 from tests.fixtures.library import LibraryFixture
+from tests.fixtures.webserver import MockAPIServer, MockAPIServerResponse
 
 if TYPE_CHECKING:
     from tests.fixtures.api_overdrive_files import OverdriveAPIFilesFixture

tests/core/conftest.py

@@ -12,4 +12,5 @@
     "tests.fixtures.services",
     "tests.fixtures.time",
     "tests.fixtures.tls_server",
+    "tests.fixtures.webserver",
 ]

tests/core/test_http.py

@@ -1,57 +1,66 @@
-import logging
+import functools
+from dataclasses import dataclass
+from typing import Callable
 
 import pytest
+import requests
 
 from core.util.http import HTTP, RequestNetworkException
 from tests.core.util.test_mock_web_server import MockAPIServer, MockAPIServerResponse
 
 
+@dataclass
+class TestHttpFixture:
+    server: MockAPIServer
+    request_with_timeout: Callable[..., requests.Response]
+
+
 @pytest.fixture
-def mock_web_server():
-    """A test fixture that yields a usable mock web server for the lifetime of the test."""
-    _server = MockAPIServer("127.0.0.1", 10256)
-    _server.start()
-    logging.info(f"starting mock web server on {_server.address()}:{_server.port()}")
-    yield _server
-    logging.info(
-        f"shutting down mock web server on {_server.address()}:{_server.port()}"
+def test_http_fixture(mock_web_server: MockAPIServer):
+    # Make sure we don't wait for retries, as that will slow down the tests.
+    request_with_timeout = functools.partial(
+        HTTP.request_with_timeout, timeout=1, backoff_factor=0
+    )
+    return TestHttpFixture(
+        server=mock_web_server, request_with_timeout=request_with_timeout
     )
-    _server.stop()
 
 
 class TestHTTP:
-    def test_retries_unspecified(self, mock_web_server: MockAPIServer):
+    def test_retries_unspecified(self, test_http_fixture: TestHttpFixture):
         for i in range(1, 7):
             response = MockAPIServerResponse()
             response.content = b"Ouch."
             response.status_code = 502
-            mock_web_server.enqueue_response("GET", "/test", response)
+            test_http_fixture.server.enqueue_response("GET", "/test", response)
 
         with pytest.raises(RequestNetworkException):
-            HTTP.request_with_timeout("GET", mock_web_server.url("/test"))
+            test_http_fixture.request_with_timeout(
+                "GET", test_http_fixture.server.url("/test")
+            )
 
-        assert len(mock_web_server.requests()) == 6
-        request = mock_web_server.requests().pop()
+        assert len(test_http_fixture.server.requests()) == 6
+        request = test_http_fixture.server.requests().pop()
         assert request.path == "/test"
         assert request.method == "GET"
 
-    def test_retries_none(self, mock_web_server: MockAPIServer):
+    def test_retries_none(self, test_http_fixture: TestHttpFixture):
         response = MockAPIServerResponse()
         response.content = b"Ouch."
         response.status_code = 502
 
-        mock_web_server.enqueue_response("GET", "/test", response)
+        test_http_fixture.server.enqueue_response("GET", "/test", response)
         with pytest.raises(RequestNetworkException):
-            HTTP.request_with_timeout(
-                "GET", mock_web_server.url("/test"), max_retry_count=0
+            test_http_fixture.request_with_timeout(
+                "GET", test_http_fixture.server.url("/test"), max_retry_count=0
             )
 
-        assert len(mock_web_server.requests()) == 1
-        request = mock_web_server.requests().pop()
+        assert len(test_http_fixture.server.requests()) == 1
+        request = test_http_fixture.server.requests().pop()
         assert request.path == "/test"
         assert request.method == "GET"
 
-    def test_retries_3(self, mock_web_server: MockAPIServer):
+    def test_retries_3(self, test_http_fixture: TestHttpFixture):
         response0 = MockAPIServerResponse()
         response0.content = b"Ouch."
         response0.status_code = 502
@@ -64,24 +73,24 @@ def test_retries_3(self, mock_web_server: MockAPIServer):
         response2.content = b"OK!"
         response2.status_code = 200
 
-        mock_web_server.enqueue_response("GET", "/test", response0)
-        mock_web_server.enqueue_response("GET", "/test", response1)
-        mock_web_server.enqueue_response("GET", "/test", response2)
+        test_http_fixture.server.enqueue_response("GET", "/test", response0)
+        test_http_fixture.server.enqueue_response("GET", "/test", response1)
+        test_http_fixture.server.enqueue_response("GET", "/test", response2)
 
-        response = HTTP.request_with_timeout(
-            "GET", mock_web_server.url("/test"), max_retry_count=3
+        response = test_http_fixture.request_with_timeout(
+            "GET", test_http_fixture.server.url("/test"), max_retry_count=3
         )
         assert response.status_code == 200
 
-        assert len(mock_web_server.requests()) == 3
-        request = mock_web_server.requests().pop()
+        assert len(test_http_fixture.server.requests()) == 3
+        request = test_http_fixture.server.requests().pop()
         assert request.path == "/test"
         assert request.method == "GET"
 
-        request = mock_web_server.requests().pop()
+        request = test_http_fixture.server.requests().pop()
         assert request.path == "/test"
         assert request.method == "GET"
 
-        request = mock_web_server.requests().pop()
+        request = test_http_fixture.server.requests().pop()
         assert request.path == "/test"
         assert request.method == "GET"

tests/core/util/test_mock_web_server.py

@@ -1,189 +1,7 @@
-import logging
-import threading
-from http.server import BaseHTTPRequestHandler, HTTPServer
-from typing import Dict, List, Optional, Tuple
-
 import pytest
 
 from core.util.http import HTTP, RequestNetworkException
-
-
-class MockAPIServerRequest:
-    """A request made to a server."""
-
-    headers: Dict[str, str]
-    payload: bytes
-    method: str
-    path: str
-
-    def __init__(self):
-        self.headers = {}
-        self.payload = b""
-        self.method = "GET"
-        self.path = "/"
-
-
-class MockAPIServerResponse:
-    """A response returned from a server."""
-
-    status_code: int
-    content: bytes
-    headers: Dict[str, str]
-    close_obnoxiously: bool
-
-    def __init__(self):
-        self.status_code = 200
-        self.content = b""
-        self.headers = {}
-        self.close_obnoxiously = False
-
-    def set_content(self, data: bytes):
-        """A convenience method that automatically sets the correct content length for data."""
-        self.content = data
-        self.headers["content-length"] = str(len(data))
-
-
-class MockAPIServerRequestHandler(BaseHTTPRequestHandler):
-    """Basic request handler."""
-
-    def _send_everything(self, _response: MockAPIServerResponse):
-        if _response.close_obnoxiously:
-            return
-
-        self.send_response(_response.status_code)
-        for key in _response.headers.keys():
-            _value = _response.headers.get(key)
-            if _value:
-                self.send_header(key, _value)
-
-        self.end_headers()
-        self.wfile.write(_response.content)
-        self.wfile.flush()
-
-    def _read_everything(self) -> MockAPIServerRequest:
-        _request = MockAPIServerRequest()
-        _request.method = self.command
-        for k in self.headers.keys():
-            header = self.headers.get(k, None)
-            if header is not None:
-                _request.headers[k] = header
-        _request.path = self.path
-        _readable = int(self.headers.get("Content-Length") or 0)
-        if _readable > 0:
-            _request.payload = self.rfile.read(_readable)
-        return _request
-
-    def _handle_everything(self):
-        _request = self._read_everything()
-        _response = self.server.mock_api_server.dequeue_response(_request)
-        if _response is None:
-            logging.error(
-                f"failed to find a response for {_request.method} {_request.path}"
-            )
-            raise AssertionError(
-                f"No available response for {_request.method} {_request.path}!"
-            )
-        self._send_everything(_response)
-
-    def do_GET(self):
-        logging.info("GET")
-        self._handle_everything()
-
-    def do_POST(self):
-        logging.info("POST")
-        self._handle_everything()
-
-    def do_PUT(self):
-        logging.info("PUT")
-        self._handle_everything()
-
-    def version_string(self) -> str:
-        return ""
-
-    def date_time_string(self, timestamp: Optional[int] = 0) -> str:
-        return "Sat, 1 January 2000 00:00:00 UTC"
-
-
-class MockAPIInternalServer(HTTPServer):
-    mock_api_server: "MockAPIServer"
-
-    def __init__(self, server_address: Tuple[str, int], bind_and_activate: bool):
-        super().__init__(server_address, MockAPIServerRequestHandler, bind_and_activate)
-        self.allow_reuse_address = True
-
-
-class MockAPIServer:
-    """Embedded web server."""
-
-    _address: str
-    _port: int
-    _server: HTTPServer
-    _server_thread: threading.Thread
-    _responses: Dict[str, Dict[str, List[MockAPIServerResponse]]]
-    _requests: List[MockAPIServerRequest]
-
-    def __init__(self, address: str, port: int):
-        self._address = address
-        self._port = port
-        self._server = MockAPIInternalServer(
-            (self._address, self._port), bind_and_activate=True
-        )
-        self._server.mock_api_server = self
-        self._server_thread = threading.Thread(target=self._server.serve_forever)
-        self._responses = {}
-        self._requests = []
-
-    def start(self) -> None:
-        self._server_thread.start()
-
-    def stop(self) -> None:
-        self._server.shutdown()
-        self._server.server_close()
-        self._server_thread.join(timeout=10)
-
-    def enqueue_response(
-        self, request_method: str, request_path: str, response: MockAPIServerResponse
-    ):
-        _by_method = self._responses.get(request_method) or {}
-        _by_path = _by_method.get(request_path) or []
-        _by_path.append(response)
-        _by_method[request_path] = _by_path
-        self._responses[request_method] = _by_method
-
-    def dequeue_response(
-        self, request: MockAPIServerRequest
-    ) -> Optional[MockAPIServerResponse]:
-        self._requests.append(request)
-        _by_method = self._responses.get(request.method) or {}
-        _by_path = _by_method.get(request.path) or []
-        if len(_by_path) > 0:
-            return _by_path.pop(0)
-        return None
-
-    def address(self) -> str:
-        return self._address
-
-    def port(self) -> int:
-        return self._port
-
-    def url(self, path: str) -> str:
-        return f"http://{self.address()}:{self.port()}{path}"
-
-    def requests(self) -> List[MockAPIServerRequest]:
-        return list(self._requests)
-
-
-@pytest.fixture
-def mock_web_server():
-    """A test fixture that yields a usable mock web server for the lifetime of the test."""
-    _server = MockAPIServer("127.0.0.1", 10256)
-    _server.start()
-    logging.info(f"starting mock web server on {_server.address()}:{_server.port()}")
-    yield _server
-    logging.info(
-        f"shutting down mock web server on {_server.address()}:{_server.port()}"
-    )
-    _server.stop()
+from tests.fixtures.webserver import MockAPIServer, MockAPIServerResponse
 
 
 class TestMockAPIServer:
@@ -224,20 +42,14 @@ def test_server_post(self, mock_web_server: MockAPIServer):
 
     def test_server_get_no_response(self, mock_web_server: MockAPIServer):
         url = mock_web_server.url("/x/y/z")
-        try:
-            HTTP.request_with_timeout("GET", url)
-        except RequestNetworkException:
-            return
-        raise AssertionError("Failed to fail!")
+        with pytest.raises(RequestNetworkException):
+            HTTP.request_with_timeout("GET", url, timeout=1, backoff_factor=0)
 
     def test_server_get_dies(self, mock_web_server: MockAPIServer):
         _r = MockAPIServerResponse()
         _r.close_obnoxiously = True
         mock_web_server.enqueue_response("GET", "/x/y/z", _r)
 
         url = mock_web_server.url("/x/y/z")
-        try:
-            HTTP.request_with_timeout("GET", url)
-        except RequestNetworkException:
-            return
-        raise AssertionError("Failed to fail!")
+        with pytest.raises(RequestNetworkException):
+            HTTP.request_with_timeout("GET", url, timeout=1, backoff_factor=0)

tests/customlists/conftest.py

@@ -0,0 +1,3 @@
+pytest_plugins = [
+    "tests.fixtures.webserver",
+]

tests/customlists/test_export.py

@@ -1,5 +1,4 @@
 import json
-import logging
 from pathlib import Path
 
 import pytest
@@ -9,20 +8,7 @@
     CustomListExportFailed,
     CustomListExports,
 )
-from tests.core.util.test_mock_web_server import MockAPIServer, MockAPIServerResponse
-
-
-@pytest.fixture
-def mock_web_server():
-    """A test fixture that yields a usable mock web server for the lifetime of the test."""
-    _server = MockAPIServer("127.0.0.1", 10256)
-    _server.start()
-    logging.info(f"starting mock web server on {_server.address()}:{_server.port()}")
-    yield _server
-    logging.info(
-        f"shutting down mock web server on {_server.address()}:{_server.port()}"
-    )
-    _server.stop()
+from tests.fixtures.webserver import MockAPIServer, MockAPIServerResponse
 
 
 class TestExports: