[Python] Crawl4AI Not Extracting Data After Successful Login Using Playwright

I'm trying to use Crawl4AI along with Playwright to extract data from a website. The login process seems to work fine, but when Crawl4AI runs, it does not extract any information from the page.

code:

from playwright.async_api import async_playwright
from crawl4ai import AsyncWebCrawler
import asyncio

async def login_and_get_page(browser):
context = await browser.new_context(viewport={'width': 1920, 'height': 1080})
page = await context.new_page()

await page.goto('https://xxxx/xxxxx', timeout=1000000)
await page.fill('[data-testid="email_input"]', 'xxxx@xxxx.com')
await page.fill('[data-testid="password_input"]', 'xxxxxx')
await page.click('[data-testid="submit_button"]')

title = await page.title()
print(f"title: {title}")
print("login pass")
return page

async def main():
print("[HOOK] on_browser_created")

async with async_playwright() as p:
browser = await p.chromium.launch(headless=False)
page = await login_and_get_page(browser)

# Get cookies after login
cookies = await page.context.cookies()

# Use Crawl4AI to scrape data
async with AsyncWebCrawler(verbose=True, cookies=cookies, headless=False, magic=True) as crawler:
result = await crawler.arun(
url="https://xxxx/xxxxx",
)
print(result.markdown)

asyncio.run(main())



output:

[HOOK] on_browser_created
title: Social Bot
login pass
[INIT].... → Crawl4AI 0.5.0.post4
[FETCH]... ↓ https://xxxx/xxxxx | Status: True | Time: 35.94s
[SCRAPE].. ◆ https://xxxx/xxxxx | Time: 0.007s
[COMPLETE] ● https://xxxx/xxxxx | Status: True | Total: 35.95s



Additionally, I have an AsyncCrawlerStrategy class, which looks like this:

class AsyncCrawlerStrategy(ABC):
"""
Abstract base class for crawler strategies.
Subclasses must implement the crawl method.
"""

@abstractmethod
async def crawl(self, url: str, **kwargs) -> AsyncCrawlResponse:
async with async_playwright() as p:
browser = await p.chromium.launch(headless=kwargs.get("headless", True))
context = await browser.new_context(viewport={'width': 1920, 'height': 1080})
page = await context.new_page()

await page.goto('https://xxxx/xxxxx')
await page.wait_for_selector('[data-testid="email_input"]')
await page.fill('[data-testid="email_input"]', 'xxxx@xxxx.com')
await page.wait_for_selector('[data-testid="password_input"]')
await page.fill('[data-testid="password_input"]', 'xxxxxx')
await page.wait_for_selector('[data-testid="submit_button"]')
await page.click('[data-testid="submit_button"]')
await page.wait_for_load_state('networkidle')

await page.goto(url)
await page.wait_for_load_state('networkidle')
content = await page.content()

await browser.close()
return AsyncCrawlResponse(content=content, url=url)



assing cookies to Crawl4AI to maintain the session.

Continue reading...