[Python] Bypass Captcha to scrape through Selenium [duplicate]

I want to scrape a webpage using Selenium, I am unable to bypass cloudflare bot check. Previously I used the code below to get the data table from here. The code below fails due to cloud flare bot check. I am using undetected_chromedriver but that also doesn't seems to work.

import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait as wait
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support import expected_conditions as EC
import time as t
import pandas as pd
from datetime import datetime

pd.options.mode.chained_assignment = None

dates = pd.bdate_range('2024-02-28', '2024-02-28')

for d in dates:

try:

date = d.strftime('%d-%m-%y')
conv_date = datetime.strptime(date, '%d-%m-%y')
day = conv_date.strftime("%d").lstrip('0')
date = '{}/{}/{}'.format(day, conv_date.strftime("%m"), conv_date.strftime("%Y"))

# url = 'https://www.nccpl.com.pk/en/market-information/fipi-lipi/fipi-sector-wise'
url = 'https://www.nccpl.com.pk/en/portfolio-investments/fipi-sector-wise'

options = Options()
options.add_argument('--headless')
browser = uc.Chrome(options=options)
browser.get(url)
picker = wait(browser, 10).until(EC.presence_of_element_located((By.ID, 'popupDatepicker')))
t.sleep(2)
browser.execute_script('arguments[0].scrollIntoView();', picker)
picker.click()
t.sleep(3)
browser.execute_script(f'arguments[0].value = "{date}";', picker)
# print('clicked the day!')
t.sleep(5)

picker = wait(browser, 10).until(EC.presence_of_element_located((By.ID, 'popupDatepicker1')))
t.sleep(2)
browser.execute_script('arguments[0].scrollIntoView();', picker)
picker.click()
t.sleep(4)
browser.execute_script(f'arguments[0].value = "{date}";', picker)
# print('clicked the second day!')
t.sleep(1)
search_button = browser.find_element(By.XPATH, '//button[@class="search_btn"]/parent::div')
t.sleep(2)
search_button.click()
# print('clicked search!')
t.sleep(10)
dfs = pd.read_html(browser.page_source)
df_lipi = dfs[0]
browser.quit()

todays_date = conv_date

df_lipi.columns = ['CLIENT_TYPE', 'SEC_CODE', 'SECTOR_NAME', 'MARKET_TYPE',
'BUY_VOLUME', 'BUY_VALUE', 'SELL_VOLUME', 'SELL_VALUE',
'NET_VOLUME', 'NET_VALUE', 'USD']

df_lipi.loc[:, 'Date'] = todays_date

df_lipi = df_lipi[['CLIENT_TYPE', 'Date', 'SEC_CODE', 'SECTOR_NAME', 'MARKET_TYPE',
'BUY_VOLUME', 'BUY_VALUE', 'SELL_VOLUME', 'SELL_VALUE',
'NET_VOLUME', 'NET_VALUE', 'USD']]

lipi_columns = ['BUY_VOLUME', 'BUY_VALUE', 'SELL_VOLUME', 'SELL_VALUE',
'NET_VOLUME', 'NET_VALUE', 'USD']

df_lipi.drop(df_lipi[df_lipi.MARKET_TYPE == 'TOTAL'].index, inplace=True)

df_lipi.iloc[-1:, 0] = 'FIPI_NET'

df_lipi.iloc[-1:, 2:4] = ['x9999', 'All']

for string in lipi_columns:
df_lipi[[string]] = (df_lipi[[string]].replace('[\$,)]', '', regex=True)
.replace('[(]', '-', regex=True).astype(float))


except:

print(d)


Is there a way to bypass cloudflare? I've added an argument of headless but that doesn't seems to resolve the issue.

Open to paid solutions that can assist bypassing like 2Captcha.

Continue reading...