[Python] How to improve selenium script so that the captcha does not appear

My aim is to fetch search-engine results (urls) from yandex.ru

I am running this this selenium script. The script is working fine on my PC, but on the server I see sophisticated captch (see Image below)

How to improve the script so that the captcha does not appear

import undetected_chromedriver as uc
import random
from selenium.webdriver.common.by import By
from datetime import datetime
import time
import logging
import traceback
import pathlib


logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler("yandex_parser.log"),
logging.StreamHandler()
]
)

logger = logging.getLogger('ParserLogger')


class YandexParser():
def __init__(self, USE_GUI=True):
self.chrome_options = uc.ChromeOptions()
if not USE_GUI:
self.chrome_options.add_argument('--headless')
self.chrome_options.add_argument('--no-sandbox')
self.chrome_options.add_argument('--disable-dev-shm-usage')
self.chrome_options.add_argument('--start-maximized')
self.chrome_options.add_argument('--disable-blink-features=AutomationControlled')
self.chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36")
# self.chrome_options.add_argument('--proxy-server=http://your-proxyort') # Optional
self.driver = uc.Chrome(options=self.chrome_options)
self.driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
'source': '''
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
'''
})
self.driver.get("https://ya.ru/")
time.sleep(random.uniform(1.5, 3.5))

def close(self):
self.driver.quit()

def check_captcha(self):
""" Try to overcome isBot button"""
cur_time = str(datetime.now()).replace(' ', '_')
if "showcaptcha" in self.driver.current_url:
logger.info("Captcha found")
self.driver.save_screenshot(f'screens/img_captcha_{cur_time}.png')

raw_button = self.driver.find_elements(By.XPATH, "//input[@class='CheckboxCaptcha-Button']")
if raw_button:
raw_button[0].click()
logger.info("Button clicked")
time.sleep(random.uniform(1, 2))
self.driver.save_screenshot(f'screens/img_captcha_afterclick_{cur_time}.png')
else:
self.driver.save_screenshot(f'screens/img_{cur_time}.png')


def parse(self, film_name: str):
logger.info(f"Start parse {film_name}")
result_urls = []
try:
self.driver.get(f"https://ya.ru/search/?text={film_name}&lr=213&search_source=yaru_desktop_common&search_domain=yaru")
self.check_captcha()
for i in range(1, 5):
result_urls.extend(self.parse_page(page_id=i))
self.get_next_page()
self.check_captcha()
# Human-like random delay
time.sleep(random.uniform(2, 5))
except Exception:
logger.error(f"Exception in {traceback.format_exc()}")
finally:
logger.info(f"Found {len(result_urls)} for film {film_name}: {result_urls}")

def parse_page(self, page_id):
res = []
urls_raw = self.driver.find_elements(By.XPATH, value='//a[@class="Link Link_theme_normal OrganicTitle-Link organic__url link"]')
for url_raw in urls_raw:
href = url_raw.get_attribute("href")
if href and "yabs.yandex.ru" not in href:
res.append(href)
logger.info(f"Found {len(res)} urls on page {page_id}")
return res

def get_next_page(self):
next_link_raw = self.driver.find_elements(By.XPATH, '//div[@class="Pager-ListItem Pager-ListItem_type_next"]')
if next_link_raw:
next_link_raw[0].click()
# Human-like random delay
time.sleep(random.uniform(3, 6))


if __name__ == "__main__":
pathlib.Path('screens/').mkdir(exist_ok=True)
parser = YandexParser(USE_GUI=False) # Default to GUI mode for stealth
films = ["Терминатор смотреть", "Саша Таня смотреть", "Джон Уик смотреть онлайн"]
idx = 0
while True:
try:
film = films[idx]
idx = (idx + 1) % len(films)
parser.parse(film)
time.sleep(random.uniform(8, 15))
except Exception as e:
parser = YandexParser(USE_GUI=False)



Log of work on the server

2025-05-17 14:53:26,295 [INFO] patching driver executable /home/MY_USER/.local/share/undetected_chromedriver/undetected_chromedriver
2025-05-17 14:53:30,904 [INFO] Start parse Терминатор смотреть
2025-05-17 14:53:33,448 [INFO] Captcha found
2025-05-17 14:53:35,291 [INFO] Button clicked
2025-05-17 14:53:42,194 [INFO] Found 0 urls on page 1
2025-05-17 14:53:42,284 [INFO] Captcha found
2025-05-17 14:53:43,363 [INFO] Button clicked
2025-05-17 14:53:53,826 [INFO] Found 0 urls on page 2
2025-05-17 14:53:53,848 [INFO] Captcha found
2025-05-17 14:53:54,659 [INFO] Button clicked
2025-05-17 14:54:06,023 [INFO] Found 0 urls on page 3




Continue reading...