[Python] Playwright Async Python: "target page context or browser has been closed" Error...

I am trying to crawl dynamic content from a website using Playwright in an asynchronous Python environment. However, I keep encountering the error: target page context or browser has been closed. Below is my code:

from playwright.async_api import async_playwright
from chatbot import init_logger
import datetime

logger = init_logger("DynamicCrawling")

class DynamicCrawler:
def __init__(self):
self.browser = None
self.context = None

async def get_browser(self):
if not self.browser:
async with async_playwright() as p:
self.browser = await p.chromium.launch()
self.context = await self.browser.new_context()
return self.browser, self.context

async def get_page(self, url):
try:
browser, context = await self.get_browser()
page = await context.new_page()
await page.goto(url)
await page.wait_for_load_state('networkidle')
dynamic_content = await page.content()
return {
'data': dynamic_content,
'success': True
}
except Exception as ex:
logger.error('Dynamic crawler exception msg ' + str(ex))
return {
'data': None,
'success': False
}

async def close_browser(self):
if self.browser:
await self.browser.close()

async def get_title(self, url):
async with await self.get_page(url) as page:
title = await page.title()
return title



@app.get('/chatbot/dynamic-crawl')
async def chatbot_test(request: Request, url: str = Query(alias='url')):
return await crawler.get_page(url)


Issue

target page context or browser has been closed

How can I fix this error and ensure that the browser and page context are properly managed? Any help would be greatly appreciated!

Continue reading...