[Python] Asyncio async funcitons hangs with asyncio.gather. (the code works without...

The following code runs well if did not put the async functions inside a asyncio.gather and let them run one after another (with awaits). But when I add them to an asyncio.gather, it prints "Scraping product data", and just hangs.

This is a nodriver + fast API project btw.

import asyncio

async def get_product_data(page):
print("Scraping product data", flush=True)
product_data = {}

async def safe_select(selector, attribute=None, index=0):
try:
if not page:
print(f"Page is None for selector: {selector}", flush=True)
return None
elements = await page.select_all(selector)
if elements and len(elements) > index:
if attribute:
return elements[index].attributes.get(attribute)
return elements[index].text.strip()
except Exception as e:
print(f"Error in {selector}: {str(e)}", flush=True)
return None

async def safe_select_all(selector, attribute=None):
try:
if not page:
print(f"Page is None for selector: {selector}", flush=True)
return []
elements = await page.select_all(selector)
if elements:
if attribute:
return [elem.attributes.get(attribute) for elem in elements]
return [elem.text.strip() for elem in elements]
except Exception as e:
print(f"Error in {selector}: {str(e)}", flush=True)
return []

# Concurrently fetch all data
try:
results = await asyncio.gather(
safe_select_all("#desktop-category-nav > div > div > ul > li > a"),
safe_select("div[data-buy-box-region='price'] > p:nth-child(1)"),
safe_select("div[data-buy-box-region='price'] > p:nth-child(2) > span:nth-child(2)"),
safe_select("#listing-page-cart h1"),
safe_select(".breadcrumb_nav ul > li:last-child > a", "href"),
safe_select(".reviews__shop-info h2"),
safe_select(".review-card > div > div > div:nth-child(3) p"),
safe_select("#shop_owners_content_toggle a", "href"),
safe_select("#shop_owners_content_toggle p"),
safe_select_all("div[data-page-type='recs_ribbon_same_shop'] > a", "href"),
safe_select_all("#product_details_content_toggle li div"),
safe_select(".image-carousel-container > ul > li:nth-child(1) > img", "src"),
safe_select("#listing-page-cart > div:nth-child(1) > p")
)
except Exception as e:
print(f"Error during concurrent fetching: {str(e)}", flush=True)
results = [None] * 13 # Ensure results has the correct length even if an error occurs

# Assign results to product_data
product_data['breadcrumb'] = " > ".join(results[0]) if results[0] else None
product_data['sales_price'] = results[1].split('\n')[-1].strip() if results[1] else None
product_data['original_price'] = results[2][1:] if results[2] else None
product_data['product_name'] = results[3]
product_data['category_url'] = results[4]
product_data['num_reviews'] = int(results[5].replace(',', '').replace('reviews', '').strip()) if results[5] else None
product_data['last_review_date'] = ' '.join(results[6].split()[-3:]) if results[6] else None
product_data['store_url'] = results[7]
product_data['brand'] = results[8]
product_data['more_from_this_shop_urls'] = results[9]
product_data['is_digital_download'] = any("digital" in detail.lower() for detail in results[10]) if results[10] else False
product_data['main_image'] = results[11]
product_data['last_24_hours'] = int(''.join(filter(str.isdigit, results[12].split('views')[0]))) if results[12] else 0

print("Final product data:", product_data, flush=True)
return product_data


This is how I call this function

@app.post("/product")
async def scrape_product(request: ScrapeRequest) -> Dict[str, Any]:
try:
async with get_browser() as browser:
page = await browser.get(request.url)
product_data = await get_product_data(page)
return product_data
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))


Any kind of help is much appreciated!

Continue reading...