Skip to content

Commit

Permalink
Add example script about setting and getting local storage state
Browse files Browse the repository at this point in the history
  • Loading branch information
elacuesta committed Mar 27, 2021
1 parent 87d3708 commit 8c3ea71
Showing 1 changed file with 40 additions and 0 deletions.
40 changes: 40 additions & 0 deletions examples/storage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
from scrapy import Spider, Request
from scrapy.crawler import CrawlerProcess
from scrapy_playwright.page import PageCoroutine


class StorageSpider(Spider):
"""
Set and get storage state
"""
name = "storage"

def start_requests(self):
yield Request(
url="https://example.org",
meta={
"playwright": True,
"playwright_include_page": True,
"playwright_page_coroutines": [
PageCoroutine("evaluate_handle", "window.localStorage.setItem('foo', 'bar');"),
],
},
)

async def parse(self, response):
page = response.meta["playwright_page"]
return {"url": response.url, "storage_state": await page.context.storage_state()}


if __name__ == "__main__":
process = CrawlerProcess(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
"DOWNLOAD_HANDLERS": {
"https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
# "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
},
}
)
process.crawl(StorageSpider)
process.start()

0 comments on commit 8c3ea71

Please sign in to comment.