Allow PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT setting to be 0

scrapy-plugins · Jan 13, 2022 · 6875f7c · 6875f7c
1 parent 0f1548f
commit 6875f7c
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -97,7 +97,7 @@ TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
     The arguments passed here take precedence over the ones defined in `PLAYWRIGHT_CONTEXT_ARGS`.
     See the docs for [`Browser.new_context`](https://playwright.dev/python/docs/api/class-browser#browsernew_contextkwargs).
 
-* `PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT` (type `Optional[int]`, default `None`)
+* `PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT` (type `Optional[float]`, default `None`)
 
     The timeout used when requesting pages by Playwright. If `None` or unset,
     the default value will be used (30000 ms at the time of writing this).

diff --git a/scrapy_playwright/handler.py b/scrapy_playwright/handler.py
@@ -2,6 +2,7 @@
 import logging
 import warnings
 from collections import defaultdict
+from contextlib import suppress
 from time import time
 from typing import Callable, Dict, Optional, Type, TypeVar
 from urllib.parse import urlparse
@@ -65,9 +66,12 @@ def __init__(self, crawler: Crawler) -> None:
 
         self.browser_type: str = crawler.settings.get("PLAYWRIGHT_BROWSER_TYPE") or "chromium"
         self.launch_options: dict = crawler.settings.getdict("PLAYWRIGHT_LAUNCH_OPTIONS") or {}
-        self.default_navigation_timeout: Optional[int] = (
-            crawler.settings.getint("PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT") or None
-        )
+        self.default_navigation_timeout: Optional[float] = None
+        if "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT" in crawler.settings:
+            with suppress(TypeError, ValueError):
+                self.default_navigation_timeout = float(
+                    crawler.settings.get("PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT")
+                )
 
         default_context_kwargs: dict = {}
         if "PLAYWRIGHT_CONTEXT_ARGS" in crawler.settings:
@@ -115,7 +119,7 @@ async def _create_browser_context(self, name: str, context_kwargs: dict) -> Brow
         context.on("close", self._make_close_browser_context_callback(name))
         logger.debug("Browser context started: '%s'", name)
         self.stats.inc_value("playwright/context_count")
-        if self.default_navigation_timeout:
+        if self.default_navigation_timeout is not None:
             context.set_default_navigation_timeout(self.default_navigation_timeout)
         return context
 
@@ -132,7 +136,7 @@ async def _create_page(self, request: Request) -> Page:
         page.on("request", _make_request_logger(context_name))
         page.on("request", self._increment_request_stats)
         self.stats.inc_value("playwright/page_count")
-        if self.default_navigation_timeout:
+        if self.default_navigation_timeout is not None:
             page.set_default_navigation_timeout(self.default_navigation_timeout)
         return page
 

diff --git a/tests/test_playwright_requests.py b/tests/test_playwright_requests.py
@@ -117,6 +117,41 @@ async def test_page_coroutine_infinite_scroll(self):
             assert "playwright" in resp.flags
             assert len(resp.css("div.quote")) == 30
 
+    @pytest.mark.asyncio
+    async def test_timeout_value(self):
+        settings_dict = {
+            "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
+        }
+        async with make_handler(settings_dict) as handler:
+            assert handler.default_navigation_timeout is None
+
+        settings_dict = {
+            "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
+            "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT": None,
+        }
+        async with make_handler(settings_dict) as handler:
+            assert handler.default_navigation_timeout is None
+
+        settings_dict = {
+            "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
+            "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT": 0,
+        }
+        async with make_handler(settings_dict) as handler:
+            assert handler.default_navigation_timeout == 0
+
+        settings_dict = {
+            "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
+            "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT": 123,
+        }
+        async with make_handler(settings_dict) as handler:
+            assert handler.default_navigation_timeout == 123
+        settings_dict = {
+            "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
+            "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT": 0.5,
+        }
+        async with make_handler(settings_dict) as handler:
+            assert handler.default_navigation_timeout == 0.5
+
     @pytest.mark.asyncio
     async def test_timeout(self):
         settings_dict = {