[Python] Web scraping trip advisor for reviews [closed]

Im trying to scrape data from trip advisor for a NLP project

Although I add many delays , random delays in my codebase still I got detected as a bot

We detected unusual activity from your device or network.

Reasons may include:

Rapid taps or clicks
JavaScript disabled or not working
Automated (bot) activity on your network (IP 112.135.68.237)
Use of developer or inspection tools
Need help? Submit feedback.

ID: 0b92e96d-dcc5-8c23-0d66-7b5ba1e11e7c


Any workaround to avoid this?

Here im using

from playwright.sync_api import sync_playwright


And I have added many random delays as well

import time
import random
import csv
from playwright.sync_api import sync_playwright

# -------------------------------------
# HELPERS FOR HUMAN-LIKE BEHAVIOR
# -------------------------------------

def random_delay(a=5, b=15):
"""Wait like a human reading."""
time.sleep(random.uniform(a, b))

def micro_pause():
"""Tiny human-like pauses to mimic hesitation."""
time.sleep(random.uniform(0.5, 2))

def human_scroll(page):
"""Scrolls down a bit like a real user would."""
for _ in range(random.randint(2, 5)):
scroll_y = random.randint(200, 800)
page.mouse.wheel(0, scroll_y)
micro_pause()

def safe_text(element):
"""Extract text safely."""
try:
return element.inner_text().strip()
except:
return ""

# -------------------------------------
# START PLAYWRIGHT
# -------------------------------------

with sync_playwright() as p:
browser = p.chromium.launch(headless=False, slow_mo=500) # slow_mo helps mimic real usage
context = browser.new_context(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36..."
)
page = context.new_page()

hotel_urls = [
"https://www.tripadvisor.com/Hotel_Review-g293962-d14106301-Reviews-Marino_Beach_Colombo-Colombo_Western_Province.html",
# Add your 120 hotel URLs here
]

all_reviews = []

for hotel_url in hotel_urls:
hotel_name = "Marino Beach Colombo"

print(f"➡️ Visiting hotel: {hotel_url}")
page.goto(hotel_url)
page.wait_for_load_state("networkidle")
random_delay(10, 20)
human_scroll(page)

reviews_scraped = 0

while reviews_scraped < 50:
human_scroll(page)
review_blocks = page.query_selector_all('xpath=//*[@id="REVIEWS"]/div[1]')
print(f" Found {len(review_blocks)} reviews.")

for block in review_blocks:
try:
# Read More
read_more = block.query_selector(
'xpath=//*[@id="lithium-root"]/main/span/div[4]/div[2]/div[2]/div/div[1]/div[3]/div[3]/div/div[1]/div/div[2]/div[4]/div[1]/div[2]/div/span[1]'
)
if read_more:
read_more.click()
micro_pause()
random_delay(2, 4)

# Extract all fields
title_element = block.query_selector(
'xpath=//*[@id="lithium-root"]/main/span/div[4]/div[2]/div[2]/div/div[1]/div[3]/div[3]/div/div[1]/div/div[2]/div[2]/a/span/span/span'
)
text_element = block.query_selector(
'xpath=//*[@id="lithium-root"]/main/span/div[4]/div[2]/div[2]/div/div[1]/div[3]/div[3]/div/div[1]/div/div[2]/div[4]/div[1]/div[1]'
)
rating_element = block.query_selector('xpath=//*[@id=":lithium-r6f:"]')
trip_type_element = page.query_selector(
'xpath=//*[@id="lithium-root"]/main/span/div[4]/div[2]/div[2]/div/div[1]/div[3]/div[3]/div/div[11]/div/div[2]/div[4]/div[3]/span[2]'
)

reviewer_name = "Name Placeholder" # Replace with actual XPath if you have it
all_reviews.append({
'hotel_name': hotel_name,
'reviewer_name': reviewer_name,
'trip_type': safe_text(trip_type_element),
'rating': safe_text(rating_element),
'title': safe_text(title_element),
'text': safe_text(text_element)
})

reviews_scraped += 1
print(f"[{reviews_scraped}] {safe_text(title_element)[:50]}...")

micro_pause()
human_scroll(page)

if reviews_scraped >= 50:
break

except Exception as e:
print(f"⚠️ Error: {e}")
continue

# ▶️ Try Next Page
try:
next_button = page.query_selector(
'xpath=//*[@id="lithium-root"]/main/span/div[4]/div[2]/div[2]/div/div[1]/div[3]/div[3]/div/div[12]/div/div/div[2]/div/div[2]'
)
if next_button and next_button.is_enabled():
next_button.click()
page.wait_for_load_state("networkidle")
random_delay(10, 20)
human_scroll(page)
else:
break
except Exception as e:
print(f"❌ Next page error: {e}")
break

random_delay(5, 10)

print(f"✅ Done scraping {reviews_scraped} reviews for this hotel.")
random_delay(20, 30) # Big pause between hotels

# -------------------------------------
# SAVE TO CSV
# -------------------------------------
if all_reviews:
keys = all_reviews[0].keys()
with open('tripadvisor_reviews.csv', 'w', newline='', encoding='utf-8') as output_file:
dict_writer = csv.DictWriter(output_file, keys)
dict_writer.writeheader()
dict_writer.writerows(all_reviews)

print("Data saved.")
browser.close()

Continue reading...