[Python] undetected_chromedriver error: Too many open files

I am running this selenium-script. After 10 hours I see an error

Too many open files
​

My code:

logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler("my_parser.log"),
logging.StreamHandler()
]
)
logger = logging.getLogger('ParserLogger')

def get_random_chrome_user_agent():
user_agent = UserAgent(browsers='chrome', os='windows', platforms='pc')
return user_agent.random


def get_chromedriver(proxy_data: dict, USE_GUI=False):
chrome_options = uc.ChromeOptions()
if not USE_GUI:
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--start-maximized')
chrome_options.add_argument('--disable-blink-features=AutomationControlled')
chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36")

manifest_json = """
{
"version": "1.0.0",
"manifest_version": 2,
"name": "Chrome Proxy",
"permissions": [
"proxy",
"tabs",
"unlimitedStorage",
"storage",
"<all_urls>",
"webRequest",
"webRequestBlocking"
],
"background": {
"scripts": ["background.js"]
},
"minimum_chrome_version":"22.0.0"
}
"""

background_js = """
var config = {
mode: "fixed_servers",
rules: {
singleProxy: {
scheme: "http",
host: "%s",
port: parseInt(%s)
},
bypassList: ["localhost"]
}
};

chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});

function callbackFn(details) {
return {
authCredentials: {
username: "%s",
password: "%s"
}
};
}

chrome.webRequest.onAuthRequired.addListener(
callbackFn,
{urls: ["<all_urls>"]},
['blocking']
);
""" % (proxy_data["proxy"], proxy_data["proxy_port"], proxy_data["proxy_login"], proxy_data["proxy_password"])
if proxy_data:
pluginfile = 'proxy_auth_plugin.zip'

with zipfile.ZipFile(pluginfile, 'w') as zp:
zp.writestr("manifest.json", manifest_json)
zp.writestr("background.js", background_js)
zp.close()
chrome_options.add_extension(pluginfile)

driver = uc.Chrome(chrome_options=chrome_options)

# Apply stealth mode for Russian Linux user
stealth(driver,
languages=["ru-RU", "ru", "en-US", "en"],
vendor="Google Inc.",
platform="Linux x86_64",
webgl_vendor="Intel Inc.",
renderer="Intel Open Source Technology Center Mesa DRI Intel(R) UHD Graphics 620 (KBL GT2)",
fix_hairline=True,
)

return driver


class YandexParser():
def __init__(self, proxy_data, USE_GUI=False, SAVE_SCRIN=False):
# Use native headless mode for Chrome
print(f"USE_GUI={USE_GUI} SAVE_SCRIN={SAVE_SCRIN}")
self.SAVE_SCRIN = SAVE_SCRIN
self.proxy_data = proxy_data
self.USE_GUI = USE_GUI
self.driver = None
self.display = None
self._initialize_driver()

def _initialize_driver(self):
if platform.system() == 'Darwin': # macOS
self.chrome_options = uc.ChromeOptions()
self.chrome_options.add_argument('--headless=new')
self.chrome_options.add_argument('--disable-gpu')
self.chrome_options.add_argument('--no-sandbox')
self.chrome_options.add_argument('--disable-dev-shm-usage')
self.driver = get_chromedriver(proxy_data=self.proxy_data, USE_GUI=self.USE_GUI)
else: # Linux
from pyvirtualdisplay import Display
self.display = Display(visible=0, size=(800, 600))
self.display.start()
self.driver = get_chromedriver(proxy_data=self.proxy_data, USE_GUI=self.USE_GUI)

self.driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
'source': '''
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
'''
})
self.driver.get("https://ya.ru/")
time.sleep(random.uniform(1.5, 3.5))

def __enter__(self):
return self

def __exit__(self, exc_type, exc_val, exc_tb):
self.close()

def close(self):
try:
if self.driver:
self.driver.quit()
self.driver = None
except Exception as e:
logger.error(f"Error closing driver: {e}")

try:
if self.display:
self.display.stop()
self.display = None
except Exception as e:
logger.error(f"Error stopping display: {e}")

def check_captcha(self):
""" Try to overcome isBot button"""
cur_time = str(datetime.now()).replace(' ', '_')
if "showcaptcha" in self.driver.current_url:
logger.info("Captcha found")
if self.SAVE_SCRIN:
self.driver.save_screenshot(f'screens/img_captcha_{cur_time}.png')

raw_button = self.driver.find_elements(By.XPATH, "//input[@class='CheckboxCaptcha-Button']")
if raw_button:
raw_button[0].click()
logger.info("Button clicked")
time.sleep(15)
if self.SAVE_SCRIN:
self.driver.save_screenshot(f'screens/img_captcha_afterclick_{cur_time}.png')
elif self.SAVE_SCRIN:
self.driver.save_screenshot(f'screens/img_{cur_time}.png')


def parse(self, film_name: str):
logger.info(f"Start parse {film_name}")
result_urls = []
try:
self.driver.get(f"https://ya.ru/search/?text={film_name}&lr=213&search_source=yaru_desktop_common&search_domain=yaru")
self.check_captcha()
for i in range(1, 5):
result_urls.extend(self.parse_page(page_id=i))
self.get_next_page()
self.check_captcha()
# Human-like random delay
time.sleep(random.uniform(2, 5))
except Exception:
logger.error(f"Exception in {traceback.format_exc()}")
finally:
logger.info(f"Found {len(result_urls)} for film {film_name}: {result_urls}")

def parse_page(self, page_id):
res = []
urls_raw = self.driver.find_elements(By.XPATH, value='//a[@class="Link Link_theme_normal OrganicTitle-Link organic__url link"]')
for url_raw in urls_raw:
href = url_raw.get_attribute("href")
if href and "yabs.yandex.ru" not in href:
res.append(href)
logger.info(f"Found {len(res)} urls on page {page_id}")
return res

def get_next_page(self):
next_link_raw = self.driver.find_elements(By.XPATH, '//div[@class="Pager-ListItem Pager-ListItem_type_next"]')
if next_link_raw:
next_link_raw[0].click()
# Human-like random delay
time.sleep(random.uniform(3, 6))


if __name__ == "__main__":
proxies = [
# Proxy datya
]
films = ["Терминатор смотреть", "Саша Таня смотреть", "Джон Уик смотреть онлайн"]
idx = 0
while True:
try:
with YandexParser(USE_GUI=False, proxy_data=proxies[idx % 2]) as parser:
film = films[idx]
idx = (idx + 1) % len(films)
parser.parse(film)

time.sleep(random.uniform(8, 15))
except Exception as e:
logger.error(f"Exception {traceback.format_exc()}")
time.sleep(1) # Add delay after error to prevent rapid retries


Full trace:

[ERROR] Exception Traceback (most recent call last):
File "/home/myuser/yandex-urls-fetcher/03_selenium_stels.py", line 259, in <module>
with YandexParser(USE_GUI=False, proxy_data=proxies[idx % 2]) as parser:
File "/home/myuser/yandex-urls-fetcher/03_selenium_stels.py", line 139, in __init__
self._initialize_driver()
File "/home/myuser/yandex-urls-fetcher/03_selenium_stels.py", line 151, in _initialize_driver
self.display = Display(visible=0, size=(800, 600))
File "/home/myuser/yandex-urls-fetcher/venv/lib/python3.9/site-packages/pyvirtualdisplay/display.py", line 54, in __init__
self._obj = cls(
File "/home/myuser/yandex-urls-fetcher/venv/lib/python3.9/site-packages/pyvirtualdisplay/xvfb.py", line 44, in __init__
AbstractDisplay.__init__(
File "/home/myuser/yandex-urls-fetcher/venv/lib/python3.9/site-packages/pyvirtualdisplay/abstractdisplay.py", line 85, in __init__
helptext = get_helptext(program)
File "/home/myuser/yandex-urls-fetcher/venv/lib/python3.9/site-packages/pyvirtualdisplay/util.py", line 13, in get_helptext
p = subprocess.Popen(
File "/usr/lib/python3.9/subprocess.py", line 951, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/usr/lib/python3.9/subprocess.py", line 1722, in _execute_child
errpipe_read, errpipe_write = os.pipe()
OSError: [Errno 24] Too many open files

Continue reading...