[Python] WeatherUnderground Web Scraping - choosing different measurements systems

I've been trying to change measurements systems. Default measurement system on the WeatherUnderground is set on imperial system.

from selenium import webdriver
from selenium.webdriver.common.by import By
from datetime import datetime

def scrape_weather_data(url, time_selector, temperature_selector,
humidity_selector, precip_selector, wind_speed_selector,
pressure_selector, amount_selector):
driver = webdriver.Chrome()

driver.get(url) # Enter The website

driver.find_element(By.CSS_SELECTOR, '#wuSettings > i').click()

driver.find_element(By.CSS_SELECTOR, '#wuSettings-quick > div >
a:nth-child(2)').click()

# Generate text file name based on day and hour
current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
filename = f'dane_pogodowe_{current_time}.txt'

# Saving data into a .txt file
with open(filename, 'a') as file:
time_elements = driver.find_elements(By.CSS_SELECTOR,time_selector)
temperature_elements =
driver.find_elements(By.CSS_SELECTOR,temperature_selector)
humidity_elements =
driver.find_elements(By.CSS_SELECTOR,humidity_selector)
precip_elements =
driver.find_elements(By.CSS_SELECTOR,precip_selector)
wind_elements = driver.find_elements(By.CSS_SELECTOR,
wind_speed_selector)
pressure_elements = driver.find_elements(By.CSS_SELECTOR,
pressure_selector)
amount_elements = driver.find_elements(By.CSS_SELECTOR,
amount_selector)

for i in range(len(time_elements)):
time = time_elements.text.strip()
temperature = temperature_elements.text.strip()
humidity = humidity_elements.text.strip()
precip = precip_elements.text.strip()
wind = wind_elements.text.strip()
pressure = pressure_elements.text.strip()
amount = amount_elements.text.strip()

file.write(f"Czas: {time}\n")
file.write(f"Temperatura: {temperature}\n")
file.write(f"Wilgotność: {humidity}\n")
file.write(f"Szansa opadów: {precip}\n")
file.write(f"Wiatr: {wind}\n")
file.write(f"Ciśnienie: {pressure}\n")
file.write(f"Ilość opadów: {amount}\n")
file.write("\n")
file.write("------------------------------\n")

driver.quit() # Close the Web browser

print("Dane pogodowe zostały zapisane.")

# URL
url = 'https://www.wunderground.com/hourly/pl/kucoby'

# Selectors CSS
time_selector = '#hourly-forecast-table > tbody > tr:nth-child(1)
> td.mat-cell.cdk-cell.cdk-column-timeHour.mat-column-
timeHour.ng-star-inserted > span'
temperature_selector = '#hourly-forecast-table > tbody > tr:nth-
child(1) > td.mat-cell.cdk-cell.cdk-column-temperature.mat-
column-temperature.ng-star-inserted'
humidity_selector = '#hourly-forecast-table > tbody > tr:nth-
child(1) > td.mat-cell.cdk-cell.cdk-column-humidity.mat-column-
humidity.ng-star-inserted'
precip_selector = '#hourly-forecast-table > tbody > tr:nth-
child(1) > td.mat-cell.cdk-cell.cdk-column-precipitation.mat-
column-precipitation.ng-star-inserted'
wind_speed_selector = '#hourly-forecast-table > tbody > tr:nth-
child(1) > td.mat-cell.cdk-cell.cdk-column-wind.mat-column-
wind.ng-star-inserted'
pressure_selector = '#hourly-forecast-table > tbody > tr:nth-
child(1) > td.mat-cell.cdk-cell.cdk-column-pressure.mat-column-
pressure.ng-star-inserted'
amount_selector = '#hourly-forecast-table > tbody > tr:nth-
child(1) > td.mat-cell.cdk-cell.cdk-column-
liquidPrecipitation.mat-column-liquidPrecipitation.ng-star-
inserted'


# Calling a function and saving the Data
scrape_weather_data(url, time_selector, temperature_selector,
humidity_selector, precip_selector, wind_speed_selector,
pressure_selector, amount_selector)


So basically the most important 2 lines are :

driver.find_element(By.CSS_SELECTOR, '#wuSettings > i').click()

driver.find_element(By.CSS_SELECTOR, '#wuSettings-quick > div >
a:nth-child(2)').click()


The first one is responsible to enter the settings[the gear icon] (which from where we can change the measurements systems), and the other one is responsible to click the 2nd option which is the metric system.

The problem is that when i try do that the output in my textfile is "------------------------------". I don't know why. It seems that is only working with selectors, that don't have such a path "a:nth-child". Because when I change that to celsius, then the imperial system becomes the selector that is"#wuSettings-quick > div > a:nth-child(1)". And the chosen selector is "#wuSettings-quick > div > a.button.selected" which works perfectly fine.

How do I solve that?

Continue reading...