[Python] Playwright can't detect text that appears in DevTools Cmd+F — how to match...

I'm using Playwright with Python to search Instagram bio links for a specific URL. For most URLs it works fine, but on some (e.g. Beacons.ai pages), the link is visibly present in the DOM when I inspect manually, yet Playwright cannot detect it.

What I tried: page.content() → didn’t work

document.body.innerText → didn’t work

document.documentElement.outerHTML → also didn’t work

Extracting and scanning the JSON → still didn’t match

Here’s the key problem: When I manually open this link in Chrome: https://beacons.ai/(onlyfans model)

I can inspect the page, open the Elements panel, hit Cmd+F, and find "onlyfans.com" in a structured or another hidden tag.

But when using Playwright to scan everything (including document.documentElement.outerHTML), "onlyfans.com" is missing.

What I need: A guaranteed way to:

Match DevTools Cmd+F

Detect my keyword no matter where it's rendered

Works reliably in headless or headed Playwright sessions

Please someone help!!

Full code:


import csv
import urllib.parse
import asyncio
import os
from playwright.async_api import async_playwright

INPUT_FILE = "input_links.csv"
OUTPUT_FILE = "onlyfans_deepsearch.csv"
SCREENSHOT_FOLDER = "screenshots"
CONCURRENT_TASKS = 5
WAIT_TIME = 3 # seconds

os.makedirs(SCREENSHOT_FOLDER, exist_ok=True)

def resolve_redirect(link: str) -> str:
if "l.instagram.com" in link and "?u=" in link:
try:
parsed = urllib.parse.urlparse(link)
real_url = urllib.parse.parse_qs(parsed.query).get("u", [None])[0]
return urllib.parse.unquote(real_url) if real_url else link
except Exception:
pass
return link

async def check_link(context, identifier: str, raw_link: str) -> dict:
if not raw_link.lower().startswith("http"):
return {"ID": identifier, "Link": raw_link, "Onlyfans": ""}

urls = [u for u in raw_link.split() if u.startswith("http")]
onlyfans_found = ""

for url in urls:
resolved = resolve_redirect(url)
try:
page = await context.new_page()
print(f" {identifier} → {resolved}")
await page.goto(resolved, timeout=30000)
await page.wait_for_load_state("networkidle")
await asyncio.sleep(WAIT_TIME)

# True DevTools-style search of full live DOM
full_dom = await page.evaluate("() => document.documentElement.outerHTML.toLowerCase()")
if "onlyfans.com" in full_dom:
onlyfans_found = "x"
safe_id = "".join(c for c in identifier if c.isalnum() or c in ("_", "-"))
await page.screenshot(path=f"{SCREENSHOT_FOLDER}/{safe_id}.png")

await page.close()
if onlyfans_found:
break
except Exception as e:
print(f"❌ Error: {identifier} → {resolved}: {e}")
continue

return {"ID": identifier, "Link": raw_link, "Onlyfans": onlyfans_found}

async def process_all(rows):
results = []
sem = asyncio.Semaphore(CONCURRENT_TASKS)

async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
context = await browser.new_context()

async def sem_task(row):
async with sem:
return await check_link(context, row[0].strip(), row[1].strip())

tasks = [sem_task(row) for row in rows]
for future in asyncio.as_completed(tasks):
results.append(await future)

await browser.close()
return results

def load_input_csv(filepath):
with open(filepath, 'r', encoding='utf-8') as f:
reader = csv.reader(f)
rows = []
for row in reader:
if len(row) >= 2:
cleaned = [col.strip() for col in row]
if len(cleaned) == 3 and cleaned[2].lower().strip() != "x":
cleaned[2] = ""
rows.append(cleaned)
return rows

def save_output_csv(filepath, data):
with open(filepath, 'w', newline='', encoding='utf-8') as f:
writer = csv.DictWriter(f, fieldnames=["ID", "Link", "Onlyfans"])
writer.writeheader()
writer.writerows(data)

rows = load_input_csv(INPUT_FILE)
final_results = asyncio.run(process_all(rows))
save_output_csv(OUTPUT_FILE, final_results)
print(f"✅ Done. Results saved to {OUTPUT_FILE}")

Continue reading...