importasyncio

fromcrawlee.playwright_crawlerimportPlaywrightCrawler,PlaywrightCrawlingContext


asyncdefmain()->None:
crawler=PlaywrightCrawler(
max_requests_per_crawl=5,# Limit the crawl to 5 requests.
headless=False,# Show the browser window.
browser_type='firefox',# Use the Firefox browser.
)

# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
asyncdefrequest_handler(context:PlaywrightCrawlingContext)->None:
context.log.info(f'Processing{context.request.url}...')

# Enqueue all links found on the page.
awaitcontext.enqueue_links()

# Extract data from the page using Playwright API.
data={
'url':context.request.url,
'title':awaitcontext.page.title(),
'content':(awaitcontext.page.content())[:100],
}

# Push the extracted data to the default dataset.
awaitcontext.push_data(data)

# Run the crawler with the initial list of URLs.
awaitcrawler.run(['https://crawlee.dev'])

# Export the entire dataset to a JSON file.
awaitcrawler.export_data('results.json')

# Or work with the data directly.
data=awaitcrawler.get_data()
crawler.log.info(f'Extracted data:{data.items}')


if__name__=='__main__':
asyncio.run(main())