From 8c3ea7162e78dbdf417cc38695a574f3826e54bb Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 26 Mar 2021 21:15:16 -0300 Subject: [PATCH] Add example script about setting and getting local storage state --- examples/storage.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 examples/storage.py diff --git a/examples/storage.py b/examples/storage.py new file mode 100644 index 00000000..ae936f4b --- /dev/null +++ b/examples/storage.py @@ -0,0 +1,40 @@ +from scrapy import Spider, Request +from scrapy.crawler import CrawlerProcess +from scrapy_playwright.page import PageCoroutine + + +class StorageSpider(Spider): + """ + Set and get storage state + """ + name = "storage" + + def start_requests(self): + yield Request( + url="https://example.org", + meta={ + "playwright": True, + "playwright_include_page": True, + "playwright_page_coroutines": [ + PageCoroutine("evaluate_handle", "window.localStorage.setItem('foo', 'bar');"), + ], + }, + ) + + async def parse(self, response): + page = response.meta["playwright_page"] + return {"url": response.url, "storage_state": await page.context.storage_state()} + + +if __name__ == "__main__": + process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "DOWNLOAD_HANDLERS": { + "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + # "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + }, + } + ) + process.crawl(StorageSpider) + process.start()