Allow some tests to always mock HTTP requests, even when request mock…

…s are disable globally

backend/pyproject.toml

@@ -73,6 +73,9 @@ env = [
     "HTV_BACKEND_PUBLIC_URL=https://example.org/api",
     "HTV_SEARCH_INDEX_PREFIX=test",
 ]
+markers = [
+    "always_mock_requests: Always mock HTTP requests, even when request mocks are disabled globally"
+]
 addopts = [
     "--import-mode=importlib",
 ]

backend/tests/conftest.py

@@ -65,25 +65,29 @@ def add(self, response):
 
 
 @pytest.fixture
-def responses():
-    """Allows mocking HTTP requests made with requests."""
+def responses(request):
+    """Allows mocking HTTP requests made with `requests`. Request mocking can be
+    disabled globally using the `HTV_TEST_MOCK_REQUESTS=false` env variable to
+    run tests against the live sources rather than test fixtures. Individual tests
+    can be marked using `always_mock_requests` to mock them even if requests mocks
+    are disabled globally. However, it’s preferred to write tests that can be run
+    against the live data sources."""
 
     mock_requests = os.environ.get("HTV_TEST_MOCK_REQUESTS", "true").lower() in ["true", "1"]
+    marks = [m.name for m in request.node.iter_markers()]
+    always_mock_requests = "always_mock_requests" in marks
 
-    if not mock_requests:
-        # In most cases, we want HTTP requests in tests to be mocked. The `responses` package
-        # doesn’t seem to provide a global configuration option to disable all mocks and pass
-        # through the request.
-        #
-        # When calling `responses.get("http://...", body="Lorem ipsum")` in a test to register
-        # a mock response, the mock is stored in a registry. When the tested then tries to send
-        # a matching request, `responses` tries to find a matching mock in the registry. To
-        # disable all mocks, we simply pass a dummy registry that never actually registers any
-        # mocks and allow all unmatched requests to pass to the original source.
-        with RequestsMock(registry=DummyRegistry) as r:
-            r.add_passthru("http")
-            yield r
-    else:
-        # Return a "normal" requests mock that fails any request that isn’t explicitly mocked.
+    if always_mock_requests or mock_requests:
         with RequestsMock() as r:
+            # Yield a "normal" requests mock that fails any request that isn’t explicitly mocked.
             yield r
+        return
+
+    # When calling `responses.get("http://...", body="Lorem ipsum")` in a test to register
+    # a mock response, the mock is stored in a registry. When the tested then tries to send
+    # a matching request, `responses` tries to find a matching mock in the registry. To
+    # disable all mocks, we simply pass a dummy registry that never actually registers any
+    # mocks and allow all unmatched requests to pass to the original source.
+    with RequestsMock(registry=DummyRegistry) as r:
+        r.add_passthru("http")
+        yield r

backend/tests/pipelines/test_rcv_list.py

@@ -5,6 +5,7 @@
 from howtheyvote.pipelines import DataUnavailableError, RCVListPipeline
 
 
+@pytest.mark.always_mock_requests
 def test_run_source_not_available(responses, db_session):
     with pytest.raises(DataUnavailableError):
         pipe = RCVListPipeline(term=9, date=datetime.date(2024, 4, 10))

backend/tests/scrapers/test_common.py

@@ -1,9 +1,11 @@
+import pytest
 import requests
 from cachetools import LRUCache
 
 from howtheyvote.scrapers.common import get_url
 
 
+@pytest.mark.always_mock_requests
 def test_get_url(responses):
     mock = responses.get("https://example.org", body="Hello World")
 
@@ -16,13 +18,15 @@ def test_get_url(responses):
     assert mock.call_count == 2
 
 
+@pytest.mark.always_mock_requests
 def test_get_url_timeout(responses):
     responses.get("https://example.org", body=requests.RequestException())
 
     response = get_url("https://example.org", headers={})
     assert response is None
 
 
+@pytest.mark.always_mock_requests
 def test_get_url_cache(responses):
     mock = responses.get("https://example.org", body="Hello World")
     cache = LRUCache(maxsize=10)
@@ -36,6 +40,7 @@ def test_get_url_cache(responses):
     assert mock.call_count == 1
 
 
+@pytest.mark.always_mock_requests
 def test_get_url_retries(responses):
     error = responses.get("https://example.org", body="Internal Server Error", status=500)
     success = responses.get("https://example.org", body="Hello World")

backend/tests/scrapers/test_members.py

@@ -1,13 +1,16 @@
 from datetime import date
 from pathlib import Path
 
+import pytest
+
 from howtheyvote.scrapers import MemberGroupsScraper, MemberInfoScraper, MembersScraper
 
 from .helpers import load_fixture
 
 TEST_DATA_DIR = Path(__file__).resolve().parent / "data"
 
 
+@pytest.mark.always_mock_requests
 def test_members_scraper(responses):
     responses.get(
         "https://www.europarl.europa.eu/meps/en/directory/xml/?leg=9",

backend/tests/scrapers/test_votes.py

@@ -14,6 +14,7 @@
 from .helpers import load_fixture, record_to_dict
 
 
+@pytest.mark.always_mock_requests
 def test_rcv_list_scraper(responses):
     responses.get(
         "https://www.europarl.europa.eu/doceo/document/PV-9-2020-07-23-RCV_FR.xml",
@@ -59,6 +60,7 @@ def test_rcv_list_scraper(responses):
     assert record_to_dict(actual[0]) == record_to_dict(expected[0])
 
 
+@pytest.mark.always_mock_requests
 def test_rcv_list_scraper_incorrect_totals(responses):
     responses.get(
         "https://www.europarl.europa.eu/doceo/document/PV-9-2020-07-23-RCV_FR.xml",
@@ -81,6 +83,7 @@ def test_rcv_list_scraper_incorrect_totals(responses):
         scraper.run()
 
 
+@pytest.mark.always_mock_requests
 def test_rcv_list_scraper_did_not_vote(responses):
     responses.get(
         "https://www.europarl.europa.eu/doceo/document/PV-9-2020-07-23-RCV_FR.xml",
@@ -110,6 +113,7 @@ def test_rcv_list_scraper_did_not_vote(responses):
     assert actual == expected
 
 
+@pytest.mark.always_mock_requests
 def test_rcv_list_scraper_same_name(responses):
     responses.get(
         "https://www.europarl.europa.eu/doceo/document/PV-9-2020-09-15-RCV_FR.xml",
@@ -135,6 +139,7 @@ def test_rcv_list_scraper_same_name(responses):
     assert actual == expected
 
 
+@pytest.mark.always_mock_requests
 def test_rcv_list_scraper_pers_id(responses):
     responses.get(
         "https://www.europarl.europa.eu/doceo/document/PV-9-2023-12-12-RCV_FR.xml",
@@ -161,6 +166,7 @@ def test_rcv_list_scraper_pers_id(responses):
     assert actual == expected
 
 
+@pytest.mark.always_mock_requests
 def test_rcv_list_scraper_pers_id_unknown(responses):
     responses.get(
         "https://www.europarl.europa.eu/doceo/document/PV-9-2023-12-12-RCV_FR.xml",
@@ -177,6 +183,7 @@ def test_rcv_list_scraper_pers_id_unknown(responses):
         scraper.run()
 
 
+@pytest.mark.always_mock_requests
 def test_rcv_list_scraper_document_register(responses):
     doceo_mock = responses.get(
         "https://www.europarl.europa.eu/doceo/document/PV-9-2020-07-23-RCV_FR.xml",
@@ -204,6 +211,7 @@ def test_rcv_list_scraper_document_register(responses):
     assert register_mock.call_count == 1
 
 
+@pytest.mark.always_mock_requests
 def test_rcv_list_scraper_timestamp_from_text(responses):
     responses.get(
         "https://www.europarl.europa.eu/doceo/document/PV-9-2019-07-15-RCV_FR.xml",