diff --git a/Dockerfile b/Dockerfile index 6bed088..5fd3059 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,11 +18,9 @@ RUN echo "Python version:" \ && poetry install --only=main --no-interaction --no-ansi \ && rm -rf /tmp/.poetry-cache \ && echo "All installed Python packages:" \ - && pip freeze \ - && echo "Installing Playwright dependencies:" \ - && poetry run playwright install chromium --with-deps + && pip freeze RUN python3 -m compileall -q ./jg/plucker ENV ACTOR_PATH_IN_DOCKER_CONTEXT="${ACTOR_PATH_IN_DOCKER_CONTEXT}" -CMD ["poetry", "run", "plucker", "--debug", "crawl", "--apify"] +CMD ["poetry", "run", "plucker", "crawl", "--apify"] diff --git a/README.md b/README.md index 36d8f7b..cbde6c6 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Maybe there is now, but the monitoring is already implemented, so… ## Notes on development - Use [Poetry](https://python-poetry.org/) for dependency management. - After `poetry install` run also `poetry run playwright install chromium` to enable browser scraping. + Run `poetry install`. - It is preferred to pin exact versions of dependencies, without `^`, and let GitHub's Dependabot to upgrade dependencies in Pull Requests. Unfortunately there is no setting in pyproject.toml, which would force this behavior, so once new dependencies are added, one needs to go and manually remove the `^` characters. - Run `pytest` to see if your code has any issues. diff --git a/jg/plucker/jobs_linkedin/spider.py b/jg/plucker/jobs_linkedin/spider.py index 860d460..6505a7a 100644 --- a/jg/plucker/jobs_linkedin/spider.py +++ b/jg/plucker/jobs_linkedin/spider.py @@ -28,12 +28,6 @@ class Spider(BaseSpider): name = "jobs-linkedin" download_delay = 5 - custom_settings = { - "DOWNLOAD_HANDLERS": { - "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", - "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", - }, - } search_params = { "f_TPR": "r2592000", # past month @@ -168,12 +162,11 @@ def verify_job( def _retry(self, url: str, request: Request | None = None) -> Request: if not request: raise ValueError(f"Request object is required to retry {url}") - # self.logger.warning(f"Retrying {url} using browser") return request.replace( url=url, dont_filter=True, headers=self.request_headers, - meta=request.meta, # | dict(playwright=True), + meta=request.meta, ) def _request( @@ -188,7 +181,7 @@ def _request( cookies=self.lang_cookies, callback=callback, cb_kwargs=cb_kwargs or {}, - meta=dict(max_retry_times=5), + meta=dict(max_retry_times=10), ) diff --git a/jg/plucker/scrapers.py b/jg/plucker/scrapers.py index 97905fb..52fadf3 100644 --- a/jg/plucker/scrapers.py +++ b/jg/plucker/scrapers.py @@ -6,7 +6,6 @@ from apify import Actor, Configuration from apify.scrapy.middlewares.apify_proxy import ApifyHttpProxyMiddleware from apify.scrapy.utils import apply_apify_settings -from playwright.sync_api import Error as PlaywrightError from scrapy import Item, Request, Spider from scrapy.crawler import Crawler, CrawlerProcess from scrapy.settings import BaseSettings, Settings @@ -30,15 +29,6 @@ async def run_actor(settings: Settings, spider_class: Type[Spider]) -> None: actor_input = await Actor.get_input() or {} proxy_config = actor_input.get("proxyConfig") settings = apply_apify_settings(settings=settings, proxy_config=proxy_config) - - # use custom proxy middleware - priority = settings["DOWNLOADER_MIDDLEWARES"].pop( - "apify.scrapy.middlewares.ApifyHttpProxyMiddleware" - ) - settings["DOWNLOADER_MIDDLEWARES"][ - "jg.plucker.scrapers.PlaywrightApifyHttpProxyMiddleware" - ] = priority - run_spider(settings, spider_class) @@ -121,59 +111,3 @@ def raise_for_stats(stats: dict[str, Any]): raise StatsError(f"Scraping finished with reason {reason!r}") if item_count := stats.get("item_dropped_reasons_count/MissingRequiredFields"): raise StatsError(f"Items missing required fields: {item_count}") - - -class PlaywrightApifyHttpProxyMiddleware(ApifyHttpProxyMiddleware): - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - Actor.log.info("Using customized ApifyHttpProxyMiddleware.") - return cls(super().from_crawler(crawler)._proxy_settings) - - async def process_request(self, request: Request, spider: Spider): - if request.meta.get("playwright"): - Actor.log.debug( - f"ApifyHttpProxyMiddleware.process_request: playwright=True, request={request}, spider={spider}" - ) - url = await self._get_new_proxy_url() - - if not (url.username and url.password): - raise ValueError( - "Username and password must be provided in the proxy URL." - ) - - proxy = url.geturl() - proxy_hash = hashlib.sha1(proxy.encode()).hexdigest()[0:8] - context_name = f"proxy_{proxy_hash}" - Actor.log.info(f"Using Playwright context {context_name}") - request.meta.update( - { - "playwright_context": f"proxy_{context_name}", - "playwright_context_kwargs": { - "proxy": { - "server": proxy, - "username": url.username, - "password": url.password, - }, - }, - } - ) - Actor.log.debug( - f"ApifyHttpProxyMiddleware.process_request: updated request.meta={request.meta}" - ) - else: - await super().process_request(request, spider) - - def process_exception( - self: ApifyHttpProxyMiddleware, - request: Request, - exception: Exception, - spider: Spider, - ) -> None | Request: - if request := super().process_exception(request, exception, spider): - return request - if isinstance(exception, PlaywrightError): - Actor.log.warning( - f'ApifyHttpProxyMiddleware: Playwright error occurred for request="{request}", reason="{exception}", skipping...' - ) - return request - return None diff --git a/jg/plucker/settings.py b/jg/plucker/settings.py index 8f70f1e..713c2c1 100644 --- a/jg/plucker/settings.py +++ b/jg/plucker/settings.py @@ -34,9 +34,6 @@ TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" -PLAYWRIGHT_BROWSER_TYPE = "chromium" - EXTENSIONS = { "scrapy.extensions.memusage.MemoryUsage": None, - "scrapy_playwright.memusage.ScrapyPlaywrightMemoryUsageExtension": 0, } diff --git a/poetry.lock b/poetry.lock index 0093243..6e30150 100644 --- a/poetry.lock +++ b/poetry.lock @@ -588,77 +588,6 @@ docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2. testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] typing = ["typing-extensions (>=4.12.2)"] -[[package]] -name = "greenlet" -version = "3.0.3" -description = "Lightweight in-process concurrent programming" -optional = false -python-versions = ">=3.7" -files = [ - {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, - {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, - {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, - {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, - {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, - {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, - {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, - {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, - {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, - {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, - {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, - {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, - {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, - {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, - {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, - {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, - {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, - {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, - {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, - {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, - {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, - {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, - {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, - {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, - {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, - {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, - {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, - {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, -] - -[package.extras] -docs = ["Sphinx", "furo"] -test = ["objgraph", "psutil"] - [[package]] name = "h11" version = "0.14.0" @@ -1216,26 +1145,6 @@ lxml = "*" packaging = "*" w3lib = ">=1.19.0" -[[package]] -name = "playwright" -version = "1.47.0" -description = "A high-level API to automate web browsers" -optional = false -python-versions = ">=3.8" -files = [ - {file = "playwright-1.47.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:f205df24edb925db1a4ab62f1ab0da06f14bb69e382efecfb0deedc4c7f4b8cd"}, - {file = "playwright-1.47.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7fc820faf6885f69a52ba4ec94124e575d3c4a4003bf29200029b4a4f2b2d0ab"}, - {file = "playwright-1.47.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:8e212dc472ff19c7d46ed7e900191c7a786ce697556ac3f1615986ec3aa00341"}, - {file = "playwright-1.47.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:a1935672531963e4b2a321de5aa59b982fb92463ee6e1032dd7326378e462955"}, - {file = "playwright-1.47.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0a1b61473d6f7f39c5d77d4800b3cbefecb03344c90b98f3fbcae63294ad249"}, - {file = "playwright-1.47.0-py3-none-win32.whl", hash = "sha256:1b977ed81f6bba5582617684a21adab9bad5676d90a357ebf892db7bdf4a9974"}, - {file = "playwright-1.47.0-py3-none-win_amd64.whl", hash = "sha256:0ec1056042d2e86088795a503347407570bffa32cbe20748e5d4c93dba085280"}, -] - -[package.dependencies] -greenlet = "3.0.3" -pyee = "12.0.0" - [[package]] name = "pluggy" version = "1.5.0" @@ -1702,21 +1611,6 @@ Twisted = ">=18.9.0" w3lib = ">=1.17.0" "zope.interface" = ">=5.1.0" -[[package]] -name = "scrapy-playwright" -version = "0.0.41" -description = "Playwright integration for Scrapy" -optional = false -python-versions = ">=3.8" -files = [ - {file = "scrapy_playwright-0.0.41-py3-none-any.whl", hash = "sha256:4979a72b9ecabc5434669c50600b712a9393ed9aed668d99e32c292e071a2d82"}, - {file = "scrapy_playwright-0.0.41.tar.gz", hash = "sha256:5cdf7f9cf256f0e640d569ba844a082af17cd9123a694be7c38ec33cc9abeb4e"}, -] - -[package.dependencies] -playwright = ">=1.15" -scrapy = ">=2.0,<2.4.0 || >2.4.0" - [[package]] name = "service-identity" version = "24.1.0" @@ -2109,4 +2003,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "3.12.*" -content-hash = "aaaf87c1568dcc4ea5c62f387417e707603379f7fd1e2ca71b0ee144a634ed10" +content-hash = "a12d5fd8b79daac3bbe4e9b8497ad8a96cb50af11d06b84a6a6de41529a3497d" diff --git a/pyproject.toml b/pyproject.toml index bd470ca..afa3588 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,6 @@ feedparser = "6.0.11" lxml = "5.3.0" nest-asyncio = "1.6.0" scrapy = "2.11.2" -scrapy-playwright = "0.0.41" [tool.poetry.group.dev.dependencies] cookiecutter = "2.6.0"