[Python] trouble using playwright for data scrapping

#This code is not getting me to scrape the articles for my scoping review. i want a summary of the 3960 articles in excel which i will transfer to EndNote

'from playwright.sync_api import sync_playwright import csv import time import random from urllib.parse import urlparse

def scrape_article_page(page, article_url): """Scrapes data from a single AJOL article page with enhanced reliability.""" article_data = { 'title': 'N/A', 'authors': 'N/A', 'abstract': 'N/A', 'keywords': 'N/A', 'date_published': 'N/A', 'doi': 'N/A', 'journal': 'N/A', 'issue': 'N/A', 'url': article_url }

try:
print(f" Navigating to: {article_url}")

# Retry mechanism with timeout handling
for attempt in range(3):
try:
page.goto(article_url, timeout=40000)
break
except Exception as e:
if attempt < 2:
print(f" Retry {attempt+1}/3 for {article_url}")
time.sleep(2)
else:
print(f"❌ Failed to load {article_url}")
return None

# Core content detection with multiple fallbacks
try:
page.wait_for_selector(
"h1.page-header, h1.article-title, section.abstract, div.article-meta",
timeout=20000
)
except:
print("⏳ Timeout waiting for article content")
return None

# Title extraction with priority to visible elements
title_selector = "h1.page-header, h1.article-title, h1.title"
title_element = page.query_selector(title_selector)
if title_element:
article_data['title'] = title_element.inner_text().strip()

# Authors extraction
author_selector = "div.authors, div.author, div.article-author"
authors_element = page.query_selector(author_selector)
if authors_element:
article_data['authors'] = authors_element.inner_text().strip()

# Abstract extraction with content validation
abstract_selector = "section.abstract, div.abstract, div#abstract"
abstract_element = page.query_selector(abstract_selector)
if abstract_element:
abstract_text = abstract_element.inner_text().strip()
if len(abstract_text) > 50:
article_data['abstract'] = abstract_text

# Keyword extraction
keyword_element = page.query_selector('meta[name="keywords"], .keywords, `your `your text`text`[class*="keyword"]')
if keyword_element:
if keyword_element.get_attribute('name') == 'keywords':
article_data['keywords'] = keyword_element.get_attribute('content') or 'N/A'
else:
article_data['keywords'] = keyword_element.inner_text().replace('Keywords:', '').strip()

# Publication date
date_element = page.query_selector('[class*="date"], [class*="published"]')
if date_element:
article_data['date_published'] = date_element.inner_text().replace('Published:', '').strip()

# DOI extraction
doi_element = page.query_selector('a[href*="doi.org"], [class*="doi"], meta[name="citation_doi"]')
if doi_element:
article_data['doi'] = doi_element.get_attribute('href') or doi_element.get_attribute('content') or 'N/A'

# Journal/issue information
breadcrumbs = page.query_selector_all('nav.breadcrumbs a, .breadcrumb a, .journal-info a')
if len(breadcrumbs) > 1:
article_data['journal'] = breadcrumbs[1].inner_text().strip()
if len(breadcrumbs) > 2:
article_data['issue'] = breadcrumbs[2].inner_text().strip()

print(f"✅ Successfully scraped: {article_url}")
return article_data

except Exception as e:
print(f"❌ Critical error scraping {article_url}: {str(e)}")
return None


def scrape_ajol_gcs(base_url, max_pages=10): """Main scraping function with comprehensive fixes""" all_articles = []

with sync_playwright() as p:
browser = p.chromium.launch(
headless=False,
args=[
'--disable-blink-features=AutomationControlled',
'--disable-dev-shm-usage',
'--no-sandbox',
f'--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{random.randint(90,115)}.0.0.0 Safari/537.36'
]
)

context = browser.new_context(
viewport={'width': 1920, 'height': 1080},
java_script_enabled=True,
locale='en-US'
)

page = context.new_page()

try:
print(" Initializing browser...")

# Enhanced anti-detection measures
page.add_init_script("""
delete Object.getPrototypeOf(navigator).webdriver;
window.chrome = {app: {isInstalled: false}};
Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3]});
""")

# Smart resource blocking
def route_handler(route):
blocked_resources = ['image', 'stylesheet', 'font', 'media']
if route.request.resource_type in blocked_resources:
route.abort()
else:
route.continue_()
page.route('**/*', route_handler)

# Navigate with error handling
try:
page.goto(base_url, timeout=120000)
except Exception as e:
print(f"❌ Initial navigation failed: {str(e)}")
return []

# Security check handling
try:
page.wait_for_selector('text="Checking your browser"', timeout=10000)
print("⚠️ Complete security check in browser window...")
start_time = time.time()
while time.time() - start_time < 120: # 2 minute timeout
if not page.is_visible('text="Checking your browser"'):
print("✅ Security check completed")
break
time.sleep(5)
else:
print("❌ Security check timed out")
return []
except:
pass

# Search results detection
try:
page.wait_for_function("""() => {
const resultsContainer = document.querySelector('.gsc-results') ||
document.querySelector('.gsc-wrapper') ||
document.querySelector('.gs-result');
return resultsContainer && resultsContainer.children.length > 0;
}""", timeout=60000)
print(" Search results loaded")
except:
print("❌ Search results failed to load")
page.screenshot(path='debug_search_failure.png')
return []

# Get results using multiple selectors
results = page.locator(".gsc-webResult.gsc-result, .gsc-result, .gs-result")
if results.count() == 0:
print("⚠️ No results found with combined selectors")
return []

print(f" Found {results.count()} articles")

current_page = 1
while current_page <= max_pages:
print(f"\n Processing page {current_page}/{max_pages}")

# Process articles
for i in range(results.count()):
try:
result = results.nth(i)
link = result.locator("a[data-ctorig], a.gs-title").first
article_url = link.evaluate("""
el => el.getAttribute('data-ctorig') || el.href
""")

if not article_url or "/download/" in article_url:
print(f"⚠️ Skipping invalid/download link: {article_url}")
continue

print(f" Processing article {i+1}/{results.count()}")

# Open in new tab with middle-click simulation
with context.expect_page(timeout=15000) as new_page_info:
link.dispatch_event('click', button="middle")
article_page = new_page_info.value

# Scrape article data
try:
article_page.wait_for_load_state('networkidle', timeout=20000)
article_data = scrape_article_page(article_page, article_url)
finally:
article_page.close()

if article_data:
all_articles.append(article_data)

time.sleep(random.uniform(1, 3))

except Exception as e:
print(f"⚠️ Error processing article {i+1}: {str(e)}")
continue

# Enhanced pagination handling
try:
next_btn = page.locator("div.gsc-cursor-next-page:visible").first
if next_btn.is_enabled():
print("⏩ Clicking next page")
next_btn.click()
# Wait for new results to load
page.wait_for_function("""() => {
const oldCount = arguments[0];
const newCount = document.querySelectorAll('.gsc-webResult.gsc-result').length;
return newCount !== oldCount;
}""", args=[results.count()], timeout=30000)
current_page += 1
time.sleep(2)
# Refresh results locator
results = page.locator(".gsc-webResult.gsc-result, .gsc-result, .gs-result")
else:
print("⏹ No more pages available")
break
except Exception as e:
print(f"⛔ Pagination failed: {str(e)}")
break

finally:
browser.close()

return all_articles


def save_to_csv(articles, filename): """Saves data to CSV with error handling""" if not articles: print(" No articles to save") return

try:
with open(filename, "w", newline="", encoding="utf-8") as csvfile:
fieldnames = ["title", "authors", "abstract", "keywords",
"date_published", "doi", "journal", "issue", "url"]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(articles)
print(f" Saved {len(articles)} articles to {filename}")
except Exception as e:
print(f"❌ CSV save failed: {str(e)}")


if name == "main": # IMPORTANT: Verify this URL matches EXACTLY what's in your browser SEARCH_URL = ( "https://www.ajol.info/index.php/ajol/Gsearch/google?q=%22HIV%22+OR+%22Human+Immunodeficiency+Virus%22%29+AND+%28%22People+living+with+HIV%22+OR+PLHIV+OR+%22HIV-positive+persons%22+OR+%22HIV+infected+persons%22+OR+%22HIV+patients%22+OR+%22HIV+positive+individuals%22+OR+%22AIDS+patients%22+OR+%22Communities+of+PLHIV%22+OR+PLWHA+OR+%22HIV+peers%22%29+AND+%28involvement+OR+engagement+OR+participation+OR+%22community+participation%22+OR+%22patient+involvement%22+OR+%22community+engagement%22+OR+%22decision-making%22+OR+%22policy+development%22+OR+%22program+implementation%22+OR+%22monitoring+and+evaluation%22+OR+%22peer+support%22+OR+advocacy+OR+%22health+promotion%22+OR+%22capacity+building%22+OR+empowerment#gsc.tab=0&gsc.page=1" )

# Configuration
MAX_PAGES = 1 # Start with 1 page for testing
HEADLESS_MODE = False # Keep browser visible for debugging

articles = scrape_ajol_gcs(SEARCH_URL, max_pages=MAX_PAGES)
save_to_csv(articles, "ajol_articles.csv")


`

Continue reading...