[Python] How can I optimise this Python script to scrape pages as quickly as possible?

I’m currently using Playwright with Pandas to scrape data from a paginated website. The script dynamically goes through each page until it encounters an empty page or one without valid data. While it works well, it’s taking quite a bit of time to go through all the pages.

I’m looking for ways to really speed this up so that it can query the pages within seconds rather than minutes. I’m open to adjustments in how Playwright is being used, optimisations for the scraping logic, or even exploring alternative libraries or frameworks if they can achieve the same results faster.

Here’s the script:

import asyncio
import nest_asyncio
from playwright.async_api import async_playwright
import pandas as pd
from io import StringIO
import random

# Apply nest_asyncio to handle nested event loops
nest_asyncio.apply()

# Updated URL with items=100
URL = "https://www.coingecko.com/en/coins/1/markets/spot?items=100"
UA = (
"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/130.0.0.0 Mobile Safari/537.3"
)
OUTPUT_PATH = r"C:\Users\Gaming\Downloads\scraped_data_optimized.csv"

# Function to add random delay
async def random_delay():
await asyncio.sleep(random.uniform(0.5, 2.0))

# Function to scrape a single page
async def scrape_page(context, page_num):
# Construct the URL with the page parameter
page_url = f"{URL}&page={page_num}"
page = await context.new_page()
await page.goto(page_url, wait_until="commit")
html = await page.content()
await page.close()

try:
tables = pd.read_html(StringIO(html))
return tables if tables else None
except ValueError:
return None

# Function to scrape a single page with retries
async def scrape_page_with_retries(context, page_num, retries=3):
for attempt in range(retries):
try:
result = await scrape_page(context, page_num)
if result:
return result # Success
print(f"No tables found on page {page_num}.")
return None
except Exception as e:
print(f"Retry {attempt + 1} for page {page_num} due to error: {e}")
await random_delay()
print(f"Failed to scrape page {page_num} after {retries} attempts.")
return None

# Function to scrape multiple pages dynamically
async def scrape_all_pages(consecutive_empty_pages=3, concurrency=5):
markets = []
empty_page_count = 0
page_num = 1

async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
context = await browser.new_context(user_agent=UA, java_script_enabled=False)

semaphore = asyncio.Semaphore(concurrency)

while empty_page_count < consecutive_empty_pages:
async def scrape_page_with_semaphore():
async with semaphore:
return await scrape_page_with_retries(context, page_num)

result = await scrape_page_with_semaphore()
if result:
markets.extend(result)
empty_page_count = 0
print(f"Page {page_num} scraped successfully.") # Only prints here
else:
empty_page_count += 1
print(f"Page {page_num} was empty.")

page_num += 1

await browser.close()

return pd.concat(markets, ignore_index=True) if markets else pd.DataFrame()

# Main function to handle scraping and saving data
async def main_async():
max_consecutive_empty_pages = 3 # Stop after 3 consecutive empty pages
concurrency = 10 # Number of concurrent tasks
df = await scrape_all_pages(consecutive_empty_pages=max_consecutive_empty_pages, concurrency=concurrency)

if not df.empty:
df = df.dropna(how="all")
df.to_csv(OUTPUT_PATH, index=False)
print(f"Data saved to '{OUTPUT_PATH}'")
else:
print("No data was scraped.")

# Entry point for the script
def main():
asyncio.run(main_async())

if __name__ == "__main__":
main()


Currently, it processes one page at a time in sequence. Is there a way to make this faster, perhaps by parallelising requests or optimising the way Playwright interacts with the browser? Any advice on how to achieve the fastest possible scraping speed would be greatly appreciated.

UPDATE AS OF 14/01/2025

Here is the latest code which runs quite fast. Just curious to know if anyone can make it even faster:

import asyncio
import nest_asyncio
from playwright.async_api import async_playwright
import pandas as pd
from io import StringIO

# Apply nest_asyncio to handle nested event loops
nest_asyncio.apply()

# Define the token ID dynamically
token_id = "1" # Change this to the desired token ID
URL = f"https://www.coingecko.com/en/coins/{token_id}/markets/spot?items=100"

UA = (
"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/130.0.0.0 Mobile Safari/537.3"
)
OUTPUT_PATH = r"C:\Users\Gaming\Downloads\scraped_data_parallel_fixed.csv"


async def scrape_page(context, page_num):
"""Scrape a single page."""
page_url = f"{URL}&page={page_num}"
page = await context.new_page()
try:
print(f"Scraping page {page_num}...")
await page.goto(page_url, wait_until="domcontentloaded")
await page.wait_for_selector("table", timeout=5000)
html = await page.content()
tables = pd.read_html(StringIO(html))
return tables if tables else None
except Exception as e:
print(f"Error scraping page {page_num}: {e}")
return None
finally:
await page.close()


async def scrape_all_pages_parallel(max_concurrent_tasks=10):
"""Scrape pages in parallel until no data is found."""
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
context = await browser.new_context(user_agent=UA, java_script_enabled=False)

page_num = 1
tasks = []
results = [] # Use a list to collect results
max_empty_pages = 3
empty_page_count = 0

while empty_page_count < max_empty_pages:
if len(tasks) < max_concurrent_tasks:
# Create a task for the next page and add it to the tasks list
tasks.append(asyncio.create_task(scrape_page(context, page_num)))
page_num += 1
else:
# Wait for at least one task to complete
done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
tasks = list(pending) # Update the list of pending tasks
for task in done:
page_data = task.result()
if page_data:
results.extend(page_data)
empty_page_count = 0
else:
empty_page_count += 1
print(f"No data found on a page. Empty count: {empty_page_count}")

await browser.close()
return pd.concat(results, ignore_index=True) if results else pd.DataFrame()


async def main_async():
df = await scrape_all_pages_parallel(max_concurrent_tasks=10)
if not df.empty:
df.to_csv(OUTPUT_PATH, index=False)
print(f"Data saved to {OUTPUT_PATH}")
else:
print("No data scraped.")


def main():
asyncio.run(main_async())


if __name__ == "__main__":
main()

# Initialize a variable to track the last successful page
last_successful_page = None

async def scrape_page(page, page_num, retries=3):
global last_successful_page
page_url = f"{URL}&page={page_num}"
for attempt in range(retries):
try:
print(f"Scraping page {page_num} (Attempt {attempt + 1})...")
await page.goto(page_url, wait_until="domcontentloaded")
html = await page.content()
tables = pd.read_html(StringIO(html))
if tables: # Data found
last_successful_page = page_num # Update the last successful page
return tables
except Exception as e:
print(f"Error scraping page {page_num}: {e}")
if attempt == retries - 1:
return None
await asyncio.sleep(1) # Backoff before retrying
return None

# After the scraping loop:
print(f"Last successful page with data: {last_successful_page}")

Continue reading...