[Python] Post Request Not working for scraping a website

Step 1: Send a GET request to fetch the page and extract hidden form values We'll first send a GET request to the page and extract necessary hidden form values like __VIEWSTATE, __VIEWSTATEGENERATOR, and __EVENTVALIDATION which are needed for subsequent POST requests.

import requests
from bs4 import BeautifulSoup
import re

# Define the URL and headers for the initial GET request
url = "https://jamabandi.punjab.gov.in/default.aspx"
headers = {
'Accept': '*/*',
'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8',
'Cache-Control': 'no-cache',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Origin': 'https://jamabandi.punjab.gov.in',
'Referer': 'https://jamabandi.punjab.gov.in/default.aspx',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-origin',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest'
}

# Send GET request to fetch the page
response = requests.get(url, headers=headers)

# Extract hidden form fields (VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
viewstate = re.search(r'__VIEWSTATE" value="([^"]+)', response.text).group(1)
viewstategenerator = re.search(r'__VIEWSTATEGENERATOR" value="([^"]+)', response.text).group(1)
eventvalidation = re.search(r'__EVENTVALIDATION" value="([^"]+)', response.text).group(1)

# Print the extracted values to verify
print("VIEWSTATE:", viewstate)
print("VIEWSTATEGENERATOR:", viewstategenerator)
print("EVENTVALIDATION:", eventvalidation)


Step 2: Post request to select the district Now that we have the necessary hidden form values, we can post the district selection by sending a POST request.


Define the payload for the POST request to select a district
payload = {
'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlDistrict',
'ctl00$SelectRegion$rdPeriod': '1',
'ctl00$SelectRegion$ddlDistrict': '10', # Choose district here (example: 10)
'ctl00$SelectRegion$ddlTehsil': '',
'ctl00$SelectRegion$ddlVillage': '',
'ctl00$SelectRegion$ddlYear': '',
'ctl00$ddlLang': 'en-US',
'ctl00$ContentPlaceHolder1$rdPeriod': '1',
'ctl00$ContentPlaceHolder1$ddlDistrict': '10', # Same district
'__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlDistrict',
'__EVENTARGUMENT': '',
'__LASTFOCUS': '',
'__VIEWSTATE': viewstate,
'__VIEWSTATEGENERATOR': viewstategenerator,
'__EVENTVALIDATION': eventvalidation,
'__ASYNCPOST': 'true'
}

# Send the POST request to select the district
post_url = "https://jamabandi.punjab.gov.in/default.aspx"
response = requests.post(post_url, headers=headers, data=payload)

# Print response to check if the request was successful
print(response.text)


Step 3: Select Tehsil Next, we move to select the Tehsil, following the same approach as the district selection. We need to extract the hidden form values again from the updated page (the one that follows the district selection) and post the Tehsil selection.

# Extract hidden form fields after the district selection (VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
viewstate = re.search(r'__VIEWSTATE" value="([^"]+)', response.text).group(1)
viewstategenerator = re.search(r'__VIEWSTATEGENERATOR" value="([^"]+)', response.text).group(1)
eventvalidation = re.search(r'__EVENTVALIDATION" value="([^"]+)', response.text).group(1)

# Define the payload for the POST request to select the Tehsil
payload = {
'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlTehsil',
'ctl00$SelectRegion$rdPeriod': '1',
'ctl00$SelectRegion$ddlDistrict': '10',
'ctl00$SelectRegion$ddlTehsil': '63', # Choose Tehsil here (example: 63)
'ctl00$SelectRegion$ddlVillage': '',
'ctl00$SelectRegion$ddlYear': '',
'ctl00$ddlLang': 'en-US',
'ctl00$ContentPlaceHolder1$rdPeriod': '1',
'ctl00$ContentPlaceHolder1$ddlDistrict': '10',
'ctl00$ContentPlaceHolder1$ddlTehsil': '63',
'__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlTehsil',
'__EVENTARGUMENT': '',
'__LASTFOCUS': '',
'__VIEWSTATE': viewstate,
'__VIEWSTATEGENERATOR': viewstategenerator,
'__EVENTVALIDATION': eventvalidation,
'__ASYNCPOST': 'true'
}



# Send the POST request to select the Tehsil
response = requests.post(post_url, headers=headers, data=payload)

# Print response to check if the request was successful
print(response.text)


Step 4: Select Village We follow the same approach as before, but now we need to select the village. The hidden form values should be extracted again after selecting the Tehsil.

# Extract hidden form fields after the Tehsil selection (VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
viewstate = re.search(r'__VIEWSTATE" value="([^"]+)', response.text).group(1)
viewstategenerator = re.search(r'__VIEWSTATEGENERATOR" value="([^"]+)', response.text).group(1)
eventvalidation = re.search(r'__EVENTVALIDATION" value="([^"]+)', response.text).group(1)

# Define the payload for the POST request to select the Village
payload = {
'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlVillage',
'ctl00$SelectRegion$rdPeriod': '1',
'ctl00$SelectRegion$ddlDistrict': '10',
'ctl00$SelectRegion$ddlTehsil': '63',
'ctl00$SelectRegion$ddlVillage': '1E36A812-C218-DD11-8334-000E0CA49FC8', # Choose Village here (example village ID)
'ctl00$SelectRegion$ddlYear': '',
'ctl00$ddlLang': 'en-US',
'ctl00$ContentPlaceHolder1$rdPeriod': '1',
'ctl00$ContentPlaceHolder1$ddlDistrict': '10',
'ctl00$ContentPlaceHolder1$ddlTehsil': '63',
'ctl00$ContentPlaceHolder1$ddlVillage': '1E36A812-C218-DD11-8334-000E0CA49FC8',
'__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlVillage',
'__EVENTARGUMENT': '',
'__LASTFOCUS': '',
'__VIEWSTATE': viewstate,
'__VIEWSTATEGENERATOR': viewstategenerator,
'__EVENTVALIDATION': eventvalidation,
'__ASYNCPOST': 'true'
}

# Send the POST request to select the Village
response = requests.post(post_url, headers=headers, data=payload)

# Print response to check if the request was successful
print(response.text)


Step 5: Select Year Now, we move to select the year in the same manner.

# Extract hidden form fields after the Village selection (VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
viewstate = re.search(r'__VIEWSTATE" value="([^"]+)', response.text).group(1)
viewstategenerator = re.search(r'__VIEWSTATEGENERATOR" value="([^"]+)', response.text).group(1)
eventvalidation = re.search(r'__EVENTVALIDATION" value="([^"]+)', response.text).group(1)

# Define the payload for the POST request to select the Year
payload = {
'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlYear',
'ctl00$SelectRegion$rdPeriod': '1',
'ctl00$SelectRegion$ddlDistrict': '10',
'ctl00$SelectRegion$ddlTehsil': '63',
'ctl00$SelectRegion$ddlVillage': '1E36A812-C218-DD11-8334-000E0CA49FC8',
'ctl00$SelectRegion$ddlYear': '3', # Choose Year here (example: 3)
'ctl00$ddlLang': 'en-US',
'ctl00$ContentPlaceHolder1$rdPeriod': '1',
'ctl00$ContentPlaceHolder1$ddlDistrict': '10',
'ctl00$ContentPlaceHolder1$ddlTehsil': '63',
'ctl00$ContentPlaceHolder1$ddlVillage': '1E36A812-C218-DD11-8334-000E0CA49FC8',
'ctl00$ContentPlaceHolder1$ddlYear': '3',
'__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlYear',
'__EVENTARGUMENT': '',
'__LASTFOCUS': '',
'__VIEWSTATE': viewstate,
'__VIEWSTATEGENERATOR': viewstategenerator,
'__EVENTVALIDATION': eventvalidation,
'__ASYNCPOST': 'true'
}

# Send the POST request to select the Year
response = requests.post(post_url, headers=headers, data=payload)

# Print response to check if the request was successful
print(response.text)


Step 6: Send GET Request to the Mutation Page

The first part of code sends a GET request to the mutation page to retrieve necessary information, such as cookies and hidden fields like __VIEWSTATE, __VIEWSTATEGENERATOR, and __EVENTVALIDATION.

import requests
cookie1 = response.cookies.get_dict()
cookie2 = cookie1['dgrLAndrecordPLRS']

# Define the URL
url = "https://jamabandi.punjab.gov.in/Mutation.aspx?itemPID=4"

# Define the headers
headers = {
"authority": "jamabandi.punjab.gov.in",
"method": "GET",
"scheme": "https",
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-encoding": "gzip, deflate, br, zstd",
"accept-language": "en-GB,en;q=0.9",
"referer": "https://jamabandi.punjab.gov.in/default.aspx",
"sec-ch-ua": '"Not A(Brand";v="8", "Chromium";v="132", "Google Chrome";v="132"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"macOS"',
"sec-fetch-dest": "document",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "same-origin",
"sec-fetch-user": "?1",
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36"
}

# Define the cookies
cookies = {
"dgrLAndrecordPLRS": cookie2
}

# Make the GET request
response = requests.get(url, headers=headers, cookies=cookies)

# Check the response status
if response.status_code == 200:
print("Request was successful!")
print(response.text) # Print the HTML content
else:
print(f"Failed to fetch the page. Status Code: {response.status_code}")
Step 2: Extract Hidden Fields from the HTML


After successfully sending the GET request, you use BeautifulSoup to parse the HTML response and extract necessary hidden fields (__VIEWSTATE, __VIEWSTATEGENERATOR, and __EVENTVALIDATION) that will be required in the subsequent POST request.This is where it is not working.

from bs4 import BeautifulSoup

# Initialize a session to persist cookies
session = requests.Session()

# Step 1: Send a GET request to the page to get necessary hidden fields
url = "https://jamabandi.punjab.gov.in/Mutation.aspx?itemPID=4"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
}
headers.update({
"Referer": url,
"X-Requested-With": "XMLHttpRequest"
})

# Send GET request to fetch the page content
response = session.get(url, headers=headers)

# Parse HTML to extract __VIEWSTATE, __VIEWSTATEGENERATOR, and __EVENTVALIDATION
soup = BeautifulSoup(response.text, "html.parser")
viewstate = soup.find("input", {"name": "__VIEWSTATE"})["value"]
viewstategenerator = soup.find("input", {"name": "__VIEWSTATEGENERATOR"})["value"]
eventvalidation = soup.find("input", {"name": "__EVENTVALIDATION"})["value"]

# Debug: Print extracted values
print("VIEWSTATE:", viewstate)
print("VIEWSTATEGENERATOR:", viewstategenerator)
print("EVENTVALIDATION:", eventvalidation)
Step 3: Prepare Payload for POST Request

Next, you define the payload for the POST request. This includes the hidden fields you extracted (__VIEWSTATE, __VIEWSTATEGENERATOR, and __EVENTVALIDATION), as well as any form fields required by the mutation form.

# Step 8: Prepare the payload for the POST request
payload = {
"ctl00$ScriptManager1": "ctl00$ContentPlaceHolder1$updMutationPanel|ctl00$ContentPlaceHolder1$ddlMutationNumber",
"__EVENTTARGET": "ctl00$ContentPlaceHolder1$ddlMutationNumber",
"__EVENTARGUMENT": "",
"__LASTFOCUS": "",
"__VIEWSTATE": viewstate,
"__VIEWSTATEGENERATOR": viewstategenerator,
"__SCROLLPOSITIONX": "0",
"__SCROLLPOSITIONY": "0",
"__EVENTVALIDATION": eventvalidation,
"ctl00$SelectRegion$rdPeriod": "1",
"ctl00$SelectRegion$ddlDistrict": "",
"ctl00$SelectRegion$ddlTehsil": "",
"ctl00$SelectRegion$ddlVillage": "",
"ctl00$SelectRegion$ddlYear": "",
"ctl00$ddlLang": "en-US",
"ctl00$ContentPlaceHolder1$ddlMutationNumber": "5672",
"__ASYNCPOST": "true"
}


Step 4: Send POST Request to Select Mutation

After preparing the payload, you send the POST request to select the mutation number. This step may trigger CAPTCHA handling if a mutation is selected.

# Step 3: Send the POST request
post_url = "https://jamabandi.punjab.gov.in/Mutation.aspx?itemPID=4"
response = session.post(post_url, headers=headers, data=payload)

# Print response to verify
print(response.text)

Continue reading...