[Python] Is it possible to screen scrape a "GET" Url from a website using python?

I'm writing a program that automates downloads from an FCC website. Link to the website. My code currently Selects the state, then on the left side of the website I want to download the csv files for cable, copper, fiber, etc.

My problem is that since there is no download URL within the HTML of the download button, I'm having a hard time automating the downloads. However, I realized through the network tab of the chrome debugger that when I click a download button, it gets a request URL. I am having a hard time figuring out how to have my program "click" each button and search for the request URL to pop up, then download the file. Screenshot of the chrome debugger with the link I need

import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

path_to_chromedriver = r"C:\Users\stackoverflow\Documents\chromedriver_win32"


def main():
chrome_options = Options()
chrome_options.add_argument("--headless")

# chrome driver path
service = Service(path_to_chromedriver)

driver = webdriver.Chrome(service=service, options=chrome_options)

driver.get('https://broadbandmap.fcc.gov/data-download/nationwide-data?version=dec2022')

# Wait for the page to fully load
WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, 'state')))

# select the state from dropdown
select = Select(driver.find_element(By.ID, 'state'))

# Iterate through states
for option in select.options[1:]: # Skipping the first option (Select...)
select.select_by_value(option.get_attribute('value')) # Select the state
time.sleep(5) # Wait for the page to update after state selection

# Download the files
download_files(driver)

driver.quit()


def download_files(driver):
# Everything I want to download
technologies = ['Cable', 'Copper', 'Fiber to the Premises', 'LBR Fixed Wireless', 'Licensed Fixed Wireless',
'Unlicensed Fixed Wireless']

# Wait for the page to update
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'btn-outline-primary')))

for tech in technologies:
try:
# Find the row element of the technology
row = driver.find_element(By.XPATH, f"//td[normalize-space(.)='{tech}']/ancestor::tr")

# Find the download button within the row and click it
download_button = row.find_element(By.CSS_SELECTOR, "button.btn.btn-outline-primary.border-0")
download_button.click()

time.sleep(5) # Wait for the download to finish.
except Exception as e:
print(f"An error occurred while trying to download '{tech}': {str(e)}")


if __name__ == '__main__':
main()


When I run this, it presses the button but it doesn't actually download anything (which is expected). I just do not understand how I can scrape the URL from the network tab after the button is clicked.

Continue reading...