[Python] I need to speed the code up for a python scraper [closed]

I'm trying to make a temporary program that will:

• get the classes from a website
• append any new classes not already found in a list "all_classes" TO all_classes

for a list of length ~150k words.

I do have some code, but it just:

1. sucks
2. seems to be riddled with annoying bugs and inconsistancies
3. is so slow that it takes a day or more to complete, and even then the results returned are uselessly bug-infested

so it'd be better to just start from the ground up honestly.

Here it is anyway though:

import random
import aiohttp as aio
import asyncio as asnc
import logging
from diccionario_de_todas_las_palabras_del_español import c
from diskcache import Cache

# init
cache = Cache('scrape_cache')
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
all_classes = set()
words_to_retry = [] # For slow requests
pattern = re.compile(r'''class=["']((?:[A-Za-z0-9_]{8}\s*)+)["']''')


async def fetch_page(session, word, retry=3):
if word in cache:
return cache[word]
try:
start_time = time.time()
await asnc.sleep(random.uniform(0.1, 0.5))
async with session.get(
f"https://www.spanishdict.com/translate/{word}",
headers={'User-Agent': 'Mozilla/5.0'},
timeout=aio.ClientTimeout(total=10)
) as response:
if response.status == 429:
await asnc.sleep(random.uniform(5, 15))
return await fetch_page(session, word, retry - 1)

html = await response.text()
elapsed = time.time() - start_time

if elapsed > 1: # Too slow
logging.warning(f"Slow request ({elapsed:.2f}s): {word}")
return None

cache.set(word, html, expire=86400)
return html
except Exception as e:
if retry > 0:
await asnc.sleep(random.uniform(1, 3))
return await fetch_page(session, word, retry - 1)
logging.error(f"Failed {word}: {str(e)}")
return None


async def process_page(html):
return {' '.join(match.group(1).split()) for match in pattern.finditer(html)} if html else set()


async def worker(session, word_queue, is_retry_phase=False):
while True:
word = await word_queue.get()
try:
html = await fetch_page(session, word)

if html is None and not is_retry_phase:
words_to_retry.append(word)
print(f"Added to retry list: {word}")
word_queue.task_done()
continue

if html:
new_classes = await process_page(html)
if new_classes:
all_classes.update(new_classes)

logging.info(f"Processed {word} | Total classes: {len(all_classes)}")
finally:
word_queue.task_done()


async def main():
connector = aio.TCPConnector(limit_per_host=20, limit=200, enable_cleanup_closed=True)
async with aio.ClientSession(connector=connector) as session:
# First pass - normal processing
word_queue = asnc.Queue()
workers = [asnc.create_task(worker(session, word_queue)) for _ in range(100)]

for word in random.sample(c, len(c)):
await word_queue.put(word)

await word_queue.join()
for task in workers:
task.cancel()

# second pass to retry slow words
if words_to_retry:
print(f"\nStarting retry phase for {len(words_to_retry)} slow words")
retry_queue = asnc.Queue()
retry_workers = [asnc.create_task(worker(session, retry_queue, is_retry_phase=True))
for _ in range(25)] # Fewer workers for retries

for word in words_to_retry:
await retry_queue.put(word)

await retry_queue.join()
for task in retry_workers:
task.cancel()

return all_classes


if __name__ == "__main__":
result = asnc.run(main())
print(f"\nScraping complete. Found {len(result)} unique classes: {result}")
if words_to_retry:
print(f"Note: {len(words_to_retry)} words were too slow and may need manual checking. {words_to_retry}")

Continue reading...