[Python] Huge internet usage by chrome and chromedriver [closed]

I am using selenium version 4.31.0 with chromedriver and chrome version 129 for scraping through python. I run out simple scraper which scrape from multiple urls. But within 5 minutes it consumed up to 1GB data from my internet connection, and for 9 hours in scraping it has comsumed up to 80GB. There for I changed into chromedriver and chrome version 135. It has consumed 20GB data from my internet connection within 9 hours. Following I have attached rough structure of one of my scraping script

def __init__(self):
self.options = webdriver.ChromeOptions()
self.options.add_argument("start-maximized")
self.options.add_argument("disable-infobars")
self.options.add_argument("--disable-extensions")
self.options.add_argument("--no-sandbox")
self.options.add_argument('--headless')
self.helper = Helper()
self.site_name ='gomedicalmarket'
self.csv_headers = [
'site_name', 'city_name', 'category_name','product_name','item_name', 'key_', 'additional_day', 'additional_week', 'description',
'first_week','1day_price', '2day_price', '3day_price','4day_price', '5day_price', '6day_price', '7day_price',
'8day_price', '9day_price', '10day_price', '11day_price', '12day_price', '13day_price', '14day_price',
'15day_price', '16day_price', '17day_price', '18day_price', '19day_price', '20day_price', '21day_price',
'22day_price', '23day_price', '24day_price', '25day_price', '26day_price', '27day_price', '28day_price',
'29day_price', '30day_price','refined_scrap_prices'
]
try:
item_name = self.helper.get_item(map_item['item_name'])
city = self.helper.get_city(map_item['city'])
print(item_name, city)
except:
traceback.print_exc()
try:
driver.get(url)
msg.good("Page loaded successfully")
except Exception as e:
msg.fail("Page load error")
print(e)
print(traceback.format_exc())
continue
price_dict = {}
description = []

#getting price ele
try:
listed_name =map_item["short_name"]
name_ele = wait.until(EC.visibility_of_element_located((By.XPATH, f"//span[contains(text(),'{listed_name}')]")))
first_parent = name_ele.find_element(By.XPATH, "..")
for i in range (0, 5):
first_parent = first_parent.find_element(By.XPATH, "..")
price_elements = first_parent.find_elements(By.XPATH, './/span[contains(text(), "$")]')
weekly_price = float(re.search(r'\d+.\d+',price_elements[0].text).group(0))
monthly_price = float(re.search(r'\d+.\d+',price_elements[1].text).group(0))
additional_week = float(re.search(r'\d+.\d+',price_elements[2].text).group(0))
print("Weekly price: ", weekly_price)
print("Monthly price: ", monthly_price)
except Exception as e:
print(f"Error getting Daily price")
print(traceback.format_exc())



I am expecting a proper solution for huge internet usage.

Continue reading...