[Python] why doesn't pagination work using Selenium

Most websites display data across multiple pages. This is done to improve user experience and reduce loading times. but , I want to automate the data extraction process using Selenium, I noticed that my script only retrieves information from page one and then stops. what are mine doing wrong

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
import pandas as pd # Ensure you import pandas
import time

# Initialize WebDriver
driver = webdriver.Chrome()

# Open URL
url = "https://www.zoopla.co.uk/house-prices/england/?new_homes=include&q=england+&orig_q=united+kingdom&view_type=list&pn=1"
driver.get(url)

# Initialize result list to store data
result = []

# Loop for pagination
while True:
# Wait for the main content to load (adjust time as needed)
WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, "_17smgnt0"))
)

# Find all house elements on the current page
houses = driver.find_elements(By.CLASS_NAME, "_1hzil3o0")

# Extract and store addresses and dates
for house in houses:
try:
item = {
"address": house.find_element(By.XPATH, './/a/h2').text,
"DateLast_sold": house.find_element(By.CSS_SELECTOR, "._1hzil3o9._1hzil3o8._194zg6t7").text
}
result.append(item) # Append to the result list
except Exception as e:
print(f"Error extracting address or date: {e}")

# Check for the "Next" button and move to the next page
try:
# Find the "Next" button using its XPath or CSS selector
next_button = driver.find_element(By.CLASS_NAME, '_14xj7k74')
next_button.click()
time.sleep(3) # Wait for the next page to load
except Exception as e:
print("No more pages to scrape or error:", e)
break # No more pages to scrape or error encountered

# Store the result into a dataframe after the loop
df = pd.DataFrame(result)

# Show the result
print(df)

# Close the driver
driver.quit()

Continue reading...