[Python] CAPTCHA Issues in Headless Mode for TikTok URL Fetching

I'm trying to fetch URLs from a TikTok user page for a research purpose. It works fine with headless=false, but when I set it to headless=true, it triggers a captcha verification. Any ideas on how to fix this? I'm using playwright

import time
import random
from playwright.sync_api import sync_playwright

def fetch_tiktok_video_urls(profile_url, max_videos):
video_urls = []
try:
with sync_playwright() as p:
browser = p.chromium.launch_persistent_context(
user_data_dir="./user_data", headless=False
)
page = browser.new_page()
page.goto(profile_url)
page.wait_for_load_state('load')
# page.screenshot(path="tiktok_profile_screenshot.png")
simulate_human_browsing(page)
# page.screenshot(path="tiktok_profile_scroll.png")
while len(video_urls) < max_videos:
page.mouse.wheel(0, random.randint(200, 500))
simulate_human_browsing(page)
# page.screenshot(path="tiktok_profile_video_scroll.png")
video_elements = page.locator("a[href*='/video/']")
links = video_elements.evaluate_all("elements => elements.map(e => e.href)")
for link in links:
if link not in video_urls:
video_urls.append(link)
if len(video_urls) >= max_videos:
break

browser.close()
except Exception as e:
print(f"Error: {e}")
return video_urls[:max_videos]

def simulate_human_browsing(page):
# Random scrolling
scroll_steps = random.randint(3, 8)
for _ in range(scroll_steps):
page.mouse.wheel(0, random.randint(200, 500))
time.sleep(random.uniform(1.5, 3.5))

# Random mouse movements
for _ in range(random.randint(2, 5)):
page.mouse.move(
random.randint(0, 800), random.randint(0, 600),
steps=random.randint(5, 15)
)
time.sleep(random.uniform(0.5, 1.5))

if __name__ == "__main__":
tiktok_url = "https://www.tiktok.com/@nail_videos2024"
urls = fetch_tiktok_video_urls(tiktok_url, max_videos=30)
print("Fetched URLs:", urls)


I tried adding user agents like this:

browser = p.chromium.launch_persistent_context(
user_data_dir="./user_data", headless=True,
args=[
"--no-sandbox",
'--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
]
)

Continue reading...