[Python] Failed to identify the reason why my script is missing a few results while scraping a...

I've created a script in Python to scrape consultant links from this webpage based on the country filter United States, located in the left sidebar. The webpage shows 2,025 results. However, when I run the script, I always get 2,016 results—9 fewer than expected. How can I retrieve the missing results?

import time
import json
import requests
from pprint import pprint
from bs4 import BeautifulSoup

url = 'https://appexchange.salesforce.com/consulting'
link = 'https://appexchange.salesforce.com/appxstore'

filter_payload = {
'AJAXREQUEST': '_viewRoot',
'j_id0:AppxLayout:j_id1007:j_id1008:filtersForm': 'j_id0:AppxLayout:j_id1007:j_id1008:filtersForm',
'store-certified experts-option1': 'on',
'store-certified experts-option2': 'on',
'store-certified experts-option3': 'on',
'store-certified experts-option4': 'on',
'store-certified experts-option5': 'on',
'store-certified experts-option6': 'on',
'store-certified experts-option7': 'on',
'store-ratings-option1': 'on',
'store-ratings-option2': 'on',
'store-ratings-option3': 'on',
'store-ratings-option4': 'on',
'store-ratings-option5': 'on',
'store-ratings-option6': 'on',
'com.salesforce.visualforce.ViewState': '',
'com.salesforce.visualforce.ViewStateVersion': '',
'com.salesforce.visualforce.ViewStateMAC': '',
'j_id0:AppxLayout:j_id1007:j_id1008:filtersForm:j_id1019': 'j_id0:AppxLayout:j_id1007:j_id1008:filtersForm:j_id1019',
'isReset': 'false',
'filtersUrl': '40,41,42,43,44,45,46,rt5,rt4,rt3,rt2,rt1,rt0,co=co-US,Choose...'
}
payload = {
'AJAXREQUEST': '_viewRoot',
'j_id0:AppxLayout:actionsForm': 'j_id0:AppxLayout:actionsForm',
'com.salesforce.visualforce.ViewState': '',
'com.salesforce.visualforce.ViewStateVersion': '',
'com.salesforce.visualforce.ViewStateMAC': '',
'j_id0:AppxLayout:actionsForm:j_id5036': 'j_id0:AppxLayout:actionsForm:j_id5036',
}

headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'
}


def update_viewstate(soup, target_payload):
for key in ['ViewState', 'ViewStateVersion', 'ViewStateMAC']:
target_payload[f'com.salesforce.visualforce.{key}'] = soup.select_one(f"[id='com.salesforce.visualforce.{key}']")['value']


with requests.Session() as session:
session.headers.update(headers)
res = session.get(url)
soup = BeautifulSoup(res.text,"xml")
update_viewstate(soup, filter_payload)

while True:
resp = session.post(link,data=filter_payload)
soup_obj = BeautifulSoup(resp.text,"xml")
if not soup_obj.select_one("a.appx-tile-consultant"):
print("No more consultant links found.")
break

for item in soup_obj.select("a.appx-tile-consultant"):
print(item.get('href'))

update_viewstate(soup_obj, payload)
filter_payload = payload.copy()

Continue reading...