[Python] web scouring and data fetching [closed]

# ai_firefox_scraper.py (Fixed)

import asyncio

import json

import os

import csv

import time

import random

from pathlib import Path

from playwright.async_api import async_playwright

SAVE_DIR = Path("scraped_data")

SAVE_DIR.mkdir(exist_ok=True)

class FirefoxSmartScraper:

def __init__(self, max_pages=5, throttle=(4, 8)):

self.max_pages = max_pages

self.throttle = throttle

async def search_and_scrape(self, topic: str):

async with async_playwright() as p:

browser = await p.firefox.launch(headless=False)

context = await browser.new_context()

page = await context.new_page()

print(f" Searching DuckDuckGo for: {topic}")

await page.goto("https://duckduckgo.com", timeout=30000)

await page.wait_for_selector("input[name='q']")

# Type like a human

for c in topic:

await page.type("input[name='q']", c, delay=random.randint(100, 200))

await page.keyboard.press("Enter")

await page.wait_for_selector("a.result__a", timeout=20000)

await asyncio.sleep(random.uniform(*self.throttle))

# Extract real links only

items = await page.query_selector_all("a.result__a")

urls = []

for item in items[:self.max_pages]:

try:

title = await item.inner_text()

href = await item.get_attribute("href")

# Ensure it's a valid URL

if href and href.startswith("http"):

urls.append({"title": title.strip(), "url": href})

except Exception as e:

print(f"[!] Failed to parse link: {e}")

continue

if not urls:

print("❌ No links found.")

await browser.close()

return

print(f" Visiting {len(urls)} pages...")

scraped = []

for idx, link in enumerate(urls):

print(f"\n [{idx+1}] {link['title']}")

try:

await page.goto(link["url"], timeout=30000)

await asyncio.sleep(random.uniform(*self.throttle))

content = await page.text_content("body")

scraped.append({

"title": link["title"],

"url": link["url"],

"content": content[:1500] # Limit content

})

except Exception as e:

print(f"[!] Failed to scrape: {link['url']}\nReason: {e}")

continue

await browser.close()

self.save_data(topic, scraped)

def save_data(self, topic: str, data: list):

filename_json = SAVE_DIR / f"{topic.replace(' ', '_')}_data.json"

filename_csv = SAVE_DIR / f"{topic.replace(' ', '_')}_data.csv"

# Save as JSON

with open(filename_json, "w", encoding="utf-8") as f:

json.dump(data, f, ensure_ascii=False, indent=2)

# Save as CSV

with open(filename_csv, "w", newline="", encoding="utf-8") as f:

writer = csv.DictWriter(f, fieldnames=["title", "url", "content"])

writer.writeheader()

for entry in data:

writer.writerow(entry)

print(f"\n✅ Saved {len(data)} entries to:\n- {filename_json}\n- {filename_csv}")

def main():

topic = input(" Enter topic to crawl web for data: ").strip()

if not topic:

print("❌ No topic entered.")

return

scraper = FirefoxSmartScraper()

asyncio.run(scraper.search_and_scrape(topic))

if __name__ == "__main__":

main()

this is my code in making an overall web scrapping.. i don't know whats wrong, it doesn't fetch data in the internet. or maybe websites are really protected

Continue reading...