[Python] Issue with pagination on my PyCharm script

I have been trying to use scrape this website (https://www.datacenters.com/locations/united-states/virginia) and the pagination is not working. I have tried all sorts of things but can't get it to work. it will only scrape the first page. Can anyone help? I am using pycharm.

Sure, here's the exact code you provided:

import requests
from bs4 import BeautifulSoup
import csv

# Base URL of the website
base_url = 'https://www.datacenters.com'
# URL of the main webpage to scrape
main_url = f'{base_url}/locations/united-states/virginia'

def get_data_from_page(url, writer):
response = requests.get(url)
if response.status_code == 200:
soup = BeautifulSoup(response.content, 'html.parser')
location_tiles = soup.find_all('div', class_='LocationTile__details__sXkB0')

if not location_tiles:
print(f"No location tiles found on {url}")

for tile in location_tiles:
name = tile.find('div', class_='LocationTile__name__NrDKr').text
address = tile.find('div', class_='LocationTile__address__Utj30').text

parent_anchor = tile.find_parent('a', href=True)
if parent_anchor:
relative_link = parent_anchor['href']
link = f'{base_url}{relative_link}'

detail_response = requests.get(link)
if detail_response.status_code == 200:
detail_soup = BeautifulSoup(detail_response.content, 'html.parser')

power_div = detail_soup.find('div', id='power')
power = power_div.find('strong').text.strip() if power_div else 'N/A'

sqf_div = detail_soup.find('div', id='statInfo')
sqf = sqf_div.find('strong').text.strip() if sqf_div else 'N/A'

# Write the data to the CSV file
writer.writerow([name, address, power, sqf, link])
print(f"Scraped data for {name}")
else:
print(f"Failed to retrieve details from https://stackoverflow.com/questions/79227787/issue-with-pagination-on-my-pycharm-script. Status code: {detail_response.status_code}")
else:
print(f"No link found for {name}")
else:
print(f"Failed to retrieve the webpage. Status code: {response.status_code}")

# Open a CSV file to write the data
with open('datacenters.csv', mode='w', newline='', encoding='utf-8') as file:
writer = csv.writer(file)
# Write the header row
writer.writerow(['Name', 'Address', 'Power', 'SQF', 'Link'])

# Start with the first page
page_url = main_url
while True:
print(f"Scraping page: {page_url}")
get_data_from_page(page_url, writer)

# Check if there is a next page link
response = requests.get(page_url)
soup = BeautifulSoup(response.content, 'html.parser')
next_page_button = soup.find('button', class_='Control__control__ijHLR Pagination__pageItem__NsQSw Pagination__symbol__KHv6r')
if next_page_button and not 'Pagination__disabled__FbUC6' in next_page_button['class']:
next_page_button.click()
next_page_link_tag = soup.find('a', href=True)
if next_page_link_tag:
next_page_link = next_page_link_tag['href']
page_url = f"{base_url}{next_page_link}"
print(f"Next page link found: {page_url}")
else:
print("Next page link not found.")
break
else:
print("No more pages.")
break


Feel free to ask if you need any further assistance or modifications!

I have tried multiple configurations but I am consistently getting errors indicating there is no next button.

Continue reading...