[Python] Beginner in Python and Web Scraping – Looking for Feedback on My Script

I’m a software engineering student currently doing an internship in the Business Intelligence area at a university. As part of a project, I decided to create a script that scrapes job postings from a website to later use in data analysis.

Here’s my situation:

• 
  I’m completely new to both Python and web scraping.
  
  
• 
  I’ve been learning through documentation, tutorials, and by asking ChatGPT :'( .
  
  
• 
  After some effort, I managed to put together a semi-functional script, but it still contains many errors and inefficiencies.
  

import os
import csv
import time
import threading
import tkinter as tk

from datetime import datetime

from selenium import webdriver

from selenium.common.exceptions import NoSuchElementException

from selenium.webdriver import Chrome
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from webdriver_manager.chrome import ChromeDriverManager

# Variables globales
URL = "https://www.elempleo.com/co/ofertas-empleo/?Salaries=menos-1-millon:10-125-millones&PublishDate=hoy"
ofertas_procesadas = set()

# Configuración carpeta y archivo
now = datetime.now()
fecha = now.strftime("%Y-%m-%d - %H-%M")
CARPETA_DATOS = "datos"
ARCHIVO_CSV = os.path.join(CARPETA_DATOS, f"ofertas_elempleo - {fecha}.csv")

if not os.path.exists(CARPETA_DATOS):
os.makedirs(CARPETA_DATOS)

if not os.path.exists(ARCHIVO_CSV):
with open(ARCHIVO_CSV, "w", newline="", encoding="utf-8") as file:
# Cambiar delimiter al predeterminado
writer = csv.writer(file, delimiter="|")
writer.writerow(["id", "Titulo", "Salario", "Ciudad", "Fecha", "Detalle", "Cargo", "Tipo de puesto", "Nivel de educación", "Sector", "Experiencia", "Tipo de contrato", "Vacantes", "Areas", "Profesiones", "Nombre empresa", "Descripcion empresa", "Habilidades", "Cargos"])

# Ventana emnergente
root = tk.Tk()
root.title("Ejecución en proceso")
root.geometry("350x100")
root.resizable(False, False)
label = tk.Label(root, text="Ejecutando script...", font=("Arial", 12))
label.pack(pady=20)

def setup_driver():
# Configuracion del navegador
service = Service(ChromeDriverManager().install())
option=webdriver.ChromeOptions()
## option.add_argument('--headless')
option.add_argument("--ignore-certificate-errors")
driver = Chrome(service=service, options=option)
return driver

def cerrar_cookies(driver):
# Cerrar ventana cookies
try:
btn_cookies = WebDriverWait(driver, 5).until(
EC.presence_of_element_located((By.XPATH, "//div[@class='col-xs-12 col-sm-4 buttons-politics text-right']//a"))
)
btn_cookies.click()
except NoSuchElementException:
pass

def extraer_info_oferta(driver):
label.config(text="Escrapeando ofertas...")

try:
# Elementos sencillos
titulo_oferta_element = driver.find_element(By.XPATH, "//div[@class='eeoffer-data-wrapper']//h1")
salario_oferta_element = driver.find_element(By.XPATH, "//div[@class='eeoffer-data-wrapper']//span[contains(@class,'js-joboffer-salary')]")
ciudad_oferta_element = driver.find_element(By.XPATH, "//div[@class='eeoffer-data-wrapper']//span[contains(@class,'js-joboffer-city')]")
fecha_oferta_element = driver.find_element(By.XPATH, "//i[contains(@class,'fa-clock-o')]/following-sibling::span[2]")
detalle_oferta_element = driver.find_element(By.XPATH, "//div[@class='description-block']//p//span")
cargo_oferta_element = driver.find_element(By.XPATH, "//i[contains(@class,'fa-sitemap')]/following-sibling::span")
tipo_puesto_oferta_element = driver.find_element(By.XPATH, "//i[contains(@class,'fa-user-circle')]/parent:")
sector_oferta_element = driver.find_element(By.XPATH, "//i[contains(@class,'fa-building')]/following-sibling::span")
experiencia_oferta_element = driver.find_element(By.XPATH, "//i[contains(@class,'fa-list')]/following-sibling::span")
tipo_contrato_oferta_element = driver.find_element(By.XPATH, "//i[contains(@class,'fa-file-text')]/following-sibling::span")
vacantes_oferta_element = driver.find_element(By.XPATH, "//i[contains(@class,'fa-address-book')]/parent:")

# Limpiar el texto de detalle_oferta_element
detalle_oferta_texto = detalle_oferta_element.text.replace("\n", " ").replace("|", " ").replace(" ", " ").replace(" ", " ").replace(" ", " ").replace("\t", " ").replace(";" , " ").strip()

# Campo Id
try:
id_oferta_element = WebDriverWait(driver, 5).until(
EC.presence_of_element_located((By.XPATH, "//div[contains(@class,'offer-data-additional')]//p//span[contains(@class,'js-offer-id')]"))
)
id_oferta_texto = id_oferta_element.get_attribute("textContent").strip()
except:
if not id_oferta_texto:
id_oferta_texto = WebDriverWait(driver, 1).until(
EC.presence_of_element_located((By.XPATH, "//div[contains(@class,'offer-data-additional')]//p//span[contains(@class,'js-offer-id')]"))
)
id_oferta_texto = id_oferta_element.get_attribute("textContent").strip()

# Campos sensibles
try:
nivel_educacion_oferta_element = driver.find_element(By.XPATH, "//i[contains(@class,'fa-graduation-cap')]/following-sibling::span")
nivel_educacion_oferta_texto = nivel_educacion_oferta_element.text
except:
nivel_educacion_oferta_texto = ""

# Elementos con menú desplegable
try:
boton_area_element = driver.find_element(By.XPATH, "//i[contains(@class,'fa-users')]/following-sibling::a")
driver.execute_script("arguments[0].click();", boton_area_element)
areas = WebDriverWait(driver, 1).until(
EC.presence_of_all_elements_located((By.XPATH, "//div[@class='modal-content']//div[@class='modal-body']//li[@class='js-area']"))
)
areas_texto = [area.text.strip() for area in areas]
driver.find_element(By.XPATH, "//div[@id='AreasLightBox']//i[contains(@class,'fa-times-circle')]").click()
except:
area_oferta = driver.find_element(By.XPATH, "//i[contains(@class,'fa-users')]/following-sibling::span")
areas_texto = [area_oferta.text.strip()]

areas_oferta = ", ".join(areas_texto)

try:
boton_profesion_element = driver.find_element(By.XPATH, "//i[contains(@class,'fa-briefcase')]/following-sibling::a")
driver.execute_script("arguments[0].click();", boton_profesion_element)
profesiones = WebDriverWait(driver, 1).until(
EC.presence_of_all_elements_located((By.XPATH, "//div[@class='modal-content']//div[@class='modal-body']//li[@class='js-profession']"))
)
profesiones_texto = [profesion.text.strip() for profesion in profesiones]
driver.find_element(By.XPATH, "//div[@id='ProfessionLightBox']//i[contains(@class,'fa-times-circle')]").click()
except:
profesion_oferta = driver.find_element(By.XPATH, "//i[contains(@class,'fa-briefcase')]/following-sibling::span")
profesiones_texto = [profesion_oferta.text.strip()]

profesiones_oferta = ", ".join(profesiones_texto)

# Información de la empresa
try:
nombre_empresa_oferta_element = driver.find_element(By.XPATH, "//div[contains(@class,'ee-header-company')]//strong")
except:
nombre_empresa_oferta_element = driver.find_element(By.XPATH, "//div[contains(@class,'data-company')]//span//span//strong")

try:
descripcion_empresa_oferta_element = driver.find_element(By.XPATH, "//div[contains(@class,'eeoffer-data-wrapper')]//div[contains(@class,'company-description')]//div")
except:
descripcion_empresa_oferta_element = driver.find_element(By.XPATH, "//div[contains(@class,'eeoffer-data-wrapper')]//span[contains(@class,'company-sector')]")

# Información adicional
try:
habilidades = driver.find_elements(By.XPATH, "//div[@class='ee-related-words']//div[contains(@class,'ee-keywords')]//li//span")

habilidades_texto = [habilidad.text.strip() for habilidad in habilidades if habilidad.text.strip()]
except:
try:
habilidades = driver.find_elements(By.XPATH, "//div[contains(@class,'ee-related-words')]//div[contains(@class,'ee-keywords')]//li//span")
habilidades_texto = [habilidad.text.strip() for habilidad in habilidades if habilidad.text.strip()]
except:
habilidades_texto = []

if habilidades_texto:
habilidades_oferta = ", ".join(habilidades_texto)
else:
habilidades_oferta = ""

try:
cargos = driver.find_elements(By.XPATH, "//div[@class='ee-related-words']//div[contains(@class,'ee-container-equivalent-positions')]//li")
cargos_texto = [cargo.text.strip() for cargo in cargos if cargo.text.strip()]
except:
try:
cargos = driver.find_elements(By.XPATH, "//div[contains(@class,'ee-related-words')]//div[contains(@class,'ee-equivalent-positions')]//li//span")
cargos_texto = [cargo.text.strip() for cargo in cargos if cargo.text.strip()]
except:
cargos_texto = []

if cargos_texto:
cargos_oferta = ", ".join(cargos_texto)
else:
cargos_oferta = ""

# Tratamiento fecha invisible
fecha_oferta_texto = fecha_oferta_element.get_attribute("textContent").strip()
return id_oferta_texto, titulo_oferta_element, salario_oferta_element, ciudad_oferta_element, fecha_oferta_texto, detalle_oferta_texto, cargo_oferta_element, tipo_puesto_oferta_element, nivel_educacion_oferta_texto, sector_oferta_element, experiencia_oferta_element, tipo_contrato_oferta_element, vacantes_oferta_element, areas_oferta, profesiones_oferta, nombre_empresa_oferta_element, descripcion_empresa_oferta_element, habilidades_oferta, cargos_oferta
except Exception:
return label.config(text=f"Error al obtener la información de la oferta")

def escritura_datos(id_oferta_texto,
titulo_oferta_element,
salario_oferta_element,
ciudad_oferta_element,
fecha_oferta_texto,
detalle_oferta_texto,
cargo_oferta_element,
tipo_puesto_oferta_element,
nivel_educacion_oferta_texto,
sector_oferta_element,
experiencia_oferta_element,
tipo_contrato_oferta_element,
vacantes_oferta_element,
areas_oferta,
profesiones_oferta,
nombre_empresa_oferta_element,
descripcion_empresa_oferta_element,
habilidades_oferta,
cargos_oferta
):
datos = [id_oferta_texto,
titulo_oferta_element.text,
salario_oferta_element.text,
ciudad_oferta_element.text,
fecha_oferta_texto,
detalle_oferta_texto,
cargo_oferta_element.text,
tipo_puesto_oferta_element.text,
nivel_educacion_oferta_texto,
sector_oferta_element.text,
experiencia_oferta_element.text,
tipo_contrato_oferta_element.text,
vacantes_oferta_element.text,
areas_oferta,
profesiones_oferta,
nombre_empresa_oferta_element.text,
descripcion_empresa_oferta_element.text,
habilidades_oferta,
cargos_oferta
]
label.config(text="Escrapeando ofertas..")
with open(ARCHIVO_CSV, "a", newline="", encoding="utf-8") as file:
writer = csv.writer(file, delimiter="|")
writer.writerow(datos)

def procesar_ofertas_pagina(driver):
global ofertas_procesadas
while True:
try:
WebDriverWait(driver, 10).until(
EC.presence_of_all_elements_located((By.XPATH, "//div[contains(@class, 'js-results-container')]"))
)
except Exception as e:
print(f"No se encontraron ofertas: {str(e)}")
return

ofertas = WebDriverWait(driver, 5).until(
EC.presence_of_all_elements_located((By.XPATH, "//div[contains(@class,'result-item')]//a[contains(@class,'js-offer-title')]"))
)
print(f"Ofertas encontradas en la página: {len(ofertas)}")

for index in range(len(ofertas)):
try:
ofertas_actulizadas = WebDriverWait(driver, 5).until(
EC.presence_of_all_elements_located((By.XPATH, "//div[contains(@class,'result-item')]//a[contains(@class,'js-offer-title')]"))
)
oferta = ofertas_actulizadas[index]

enlace = oferta.get_attribute("href")
label.config(text="Ofertas encontradas.")

if not enlace:
label.config(text="Error al obtener el enlace de la oferta")
continue

label.config(text="Escrapeando ofertas...")
driver.execute_script(f"window.open('{enlace}', '_blank')")
time.sleep(2)
driver.switch_to.window(driver.window_handles[-1])

try:
datos_oferta = extraer_info_oferta(driver)
if datos_oferta:
id_oferta = datos_oferta[0]
if id_oferta not in ofertas_procesadas:
escritura_datos(*datos_oferta)
ofertas_procesadas.add(id_oferta)
print(f"Oferta numero {index + 1} de {len(ofertas)}.")

except Exception as e:
print(f"Error en la oferta: {str(e)}")

driver.close()
driver.switch_to.window(driver.window_handles[0])
except Exception as e:
print(f"Error procesando laoferta {index}: {str(e)}")
return False

label.config(text="Cambiando página de ofertas...")
if not siguiente_pagina(driver):
break

def siguiente_pagina(driver):
try:
btn_siguiente = driver.find_element(By.XPATH, "//ul[contains(@class,'pagination')]//li//a//i[contains(@class,'fa-angle-right')]")
li_contenedor = driver.find_element(By.XPATH, "//ul[contains(@class,'pagination')]//li//a//i[contains(@class,'fa-angle-right')]/ancestor::li")
if "disabled" in li_contenedor.get_attribute("class").split():
return False
else:
driver.execute_script("arguments[0].click();", btn_siguiente)
WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//div[@class='result-item']//a"))
)
return True
except NoSuchElementException:
return False



def main():
global root
driver = setup_driver()
try:
driver.get(URL)
cerrar_cookies(driver)

while True:
procesar_ofertas_pagina(driver)

# label.config(text="Cambiando página de ofertas...")
# if not siguiente_pagina(driver):
# break
finally:
driver.quit()
root.destroy()

def run_scraping():
main()

threading.Thread(target=run_scraping).start()
root.mainloop()


I would really appreciate it if someone with more experience in Python/web scraping could take a look and give me advice on what I could improve in my code (best practices, structure, libraries, etc.).

Thank you in advance!

Continue reading...