[Python] Why am i getting is interrupted by another navigation to...

I am scraping a page to get links of games to save them in a file, the script works perfectly, it is made so that if it finds an error in a link that does not exist it forgets it and goes to the next one, the thing is that when I get to the link 162, the program crashes saying this

program otuput


link 160, 338 leftlink 161, 337 leftError going to the download pageUnexpected error occurred: Page.goto: Navigation to “https://nintendoproject.com/vip/five-nights-at-freddys-2” is interrupted by another navigation to “chrome-error://chromewebdata/”Call log:navigating to “https://nintendoproject.com/vip/five-nights-at-freddys-2”, waiting until “load”.


and if I open the link, it is a fallen one, that means that I should avoid it and go to the other, the thing is that it does not happen, just gives the error and crashes, and now comes the worst, it happens with all the links that come after that, whether fallen or not, before I had a try that opened all no matter the error, opened all the links, and then I realized that happens with all the links after 162.

Complete code ( i removed my credentials because i need to be logged in the page)

from playwright.sync_api import sync_playwright
import json
import time


def login(page):
# login into the page
page.goto("https://nintendoproject.com/login/")
page.fill("input[name='username-20797']", "empty")
page.fill("input[name='user_password-20797']", "empty")
page.click("#um-submit-btn")
page.wait_for_load_state('networkidle')


def save_links_to_file(page):
# get links and save them in a file
page.goto("https://nintendoproject.com/vip/catalogo-completo-nintendo-switch/")
page.wait_for_selector("span > a")

download_links = page.query_selector_all("span > a")
download_links_href = [link.get_attribute("href") for link in download_links]

with open("links.txt", "a+", encoding="utf-8") as file:
file.seek(0)
existing_links = {link.strip() for link in file.readlines()}

for href in download_links_href:
if href not in existing_links:
file.write(href + "\n")


def process_links(page):
# append every link
download_info_list = []

with open("links.txt", "r", encoding="utf-8") as file:
links = [link.strip() for link in file.readlines()]

for index, link in enumerate(links):
# try to go to the page
page.goto(link)
print(f"link {index + 1}, quedan {len(links) - index - 1}")
page.wait_for_load_state('networkidle')

download_info = {"nombre": "", "juego": "", "dlc": [], "update": []}

# locate the following link to get its href
try:
second_link = page.query_selector(".elementor-button.elementor-button-link")
second_link_href = second_link.get_attribute("href")
page.goto(second_link_href)
page.wait_for_selector(".close-button", state="visible")

except Exception as e:
print("Error while going to the page")
continue # Si no se encuentra el segundo link, continuar con el siguiente

# locate the download links
download_links = page.query_selector_all(".wp-block-button__link.wp-element-button")
if not download_links:
print("No download links")
continue

# get the game title
try:
game_title = page.query_selector("h1").inner_text()
download_info["nombre"] = game_title
download_info["juego"] = download_links[0].get_attribute("href") # Primer link es el juego
except Exception as e:
print("Error getting the game title")
continue

# process other links (dlc and update)
for download_link in download_links:
span_element = download_link.query_selector("span")
if span_element:
span_text = span_element.inner_text().lower()
link_href = download_link.get_attribute("href")

if "dlc" in span_text:
download_info["dlc"].append(link_href)
elif "update" in span_text:
download_info["update"].append(link_href)

download_info_list.append(download_info)
time.sleep(3)

return download_info_list


def main():
with sync_playwright() as p:
browser = p.chromium.launch(headless=False)
context = browser.new_context()
page = context.new_page()

try:
login(page) # login
save_links_to_file(page) # save links
download_info_list = process_links(page) # process links and get the information

# save the information in a json
with open("download_info.json", "w", encoding="utf-8") as saving_file:
json.dump(download_info_list, saving_file, indent=2)

except Exception as e:
print(f"Ocurrió un error inesperado: {e}")
finally:
browser.close()


if __name__ == "__main__":
main()


I don't know what I'm doing wrong, I don't know how to fix it, I don't know if it's my problem or playwright's problem. It does not matter how much times i re run it, it happens the same

Continue reading...