[Python] How to scrape movie titles on a dynamic site?

How can I parse movie titles, the site is dynamic and movies are loaded in 25 increments. Using bs4 I can only parse the first 25 https://www.imdb.com/chart/top/

import requests
from bs4 import BeautifulSoup


def parse_movies(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}

response = requests.get(url, headers=headers)

if response.status_code != 200:
print(f"Ошибка при запросе страницы: {response.status_code}")
return []

soup = BeautifulSoup(response.text, 'html.parser')
movies_info = []

for movie in soup.find_all('li', class_='ipc-metadata-list-summary-item sc-4929eaf6-0 DLYcv cli-parent'):
title_element = movie.find('h3', class_='ipc-title__text')

metadata_elements = movie.find_all('span', class_='sc-300a8231-7 eaXxft cli-title-metadata-item')

if title_element and metadata_elements:
title = title_element.text.strip()
year = metadata_elements[0].text.strip() if len(metadata_elements) > 0 else 'N/A'
duration = metadata_elements[1].text.strip() if len(metadata_elements) > 1 else 'N/A'

rating_element = movie.find('span', class_='ipc-rating-star--rating')
rating = rating_element.text.strip() if rating_element else 'N/A'

age_rating = metadata_elements[-1].text.strip() if metadata_elements else 'N/A'

movies_info.append({
'title': title,
'year': year,
'duration': duration,
'rating': rating,
'age_rating': age_rating
})

return movies_info

url = 'https://www.imdb.com/chart/moviemeter/'
movies = parse_movies(url)

for movie in movies:
print(
f"Название: {movie['title']}, Год: {movie['year']}, Длительность: {movie['duration']}, Рейтинг: {movie['rating']}, Возрастное ограничение: {movie['age_rating']}")

Continue reading...