[Python] ('Connection aborted.', RemoteDisconnected('Remote end closed connection without...

Currently I have this issue in my dev server its running in parallel scraping but it will always encounter ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')). Its currently deployed in ec2 t3a.large

def scrape_services(master_list, excluded_list, master_list_filename, request_id, max_workers=3):
results = []
output_csv_path = None
total_items = len(master_list)

with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {executor.submit(process_row, index, row, request_id, excluded_list, queue_status, total_items): index for index, row in master_list.iterrows()}
done, _ = concurrent.futures.wait(futures.keys(), return_when=concurrent.futures.ALL_COMPLETED)

results = [future.result() for future in done]

# Save results
if results:
output_csv_path = save_results_to_csv(results, master_list_filename)

return jsonify({"message": "Scraping completed", "csv_file": output_csv_path})

def process_row(index, row, request_id, excluded_list, queue_status, total_items):
""" Worker function that processes a single item """
with driver_lock: # Ensure only one thread initializes the driver at a time
driver = get_chrome_driver()
try:
# Add sleep to prevent Google from blocking
time.sleep(random.uniform(2, 5))

keyword = f"{row.get('業種_検索用_', '')} {row.get('顧客名', '')} {row.get('住所', '')} {row.get('電話番号', '')}".strip()
print(f"[{index+1}/{total_items}] Searching for: {keyword}")

# Introduce a random delay to prevent being blocked
time.sleep(random.uniform(2, 5)) # Wait between 2 to 5 seconds

url, top_url = scrape_google_search(driver, keyword, excluded_list)

return {
"業種_検索用_": row.get("業種_検索用_", ""),
"顧客名": row.get("顧客名", ""),
"住所": row.get("住所", ""),
"電話番号": row.get("電話番号", ""),
"url": url,
"top_url": top_url
}
except Exception as e:
print(f"Error processing {keyword}: {e}")
return {
"業種_検索用_": row.get("業種_検索用_", ""),
"顧客名": row.get("顧客名", ""),
"住所": row.get("住所", ""),
"電話番号": row.get("電話番号", ""),
"url": None,
"top_url": None,
"error": str(e)
}
finally:
driver.quit()

Continue reading...