[Python] the beautiful soup returns a none when i go to any other pages other than 1 in a...

Discussão em 'Python' iniciado por Stack, Junho 14, 2025.

  1. Stack

    Stack Membro Participativo

    so im making a python project where user can search about the internship they want and the program will display a dataframe of all the internships from internshala website but im having a problem that beautiful soup only reads page 1 and after that it returns none i cant seem to find the sollution to this help pls


    import pandas as pd
    from bs4 import BeautifulSoup
    import requests
    import time
    a=input("enter the internship position:")
    def get_soup(a, pageno="1"):
    url = "https://internshala.com/internships/keywords-"
    l = a.split()
    for n, i in enumerate(l):
    if n != len(l) - 1:
    url += i + "%20"
    else:
    url += i + "/"
    url += f"page-{pageno}/"

    head = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
    }

    req = requests.get(url, headers=head)
    soup = BeautifulSoup(req.content, "html.parser")
    return soup


    def get_pageno():
    num=get_soup(a).find("div",class_="page_number heading_6").find_all("a")[-1].text.strip()
    return num




    def get_info(no):
    info={"company":[],"role":[],"location":[],"stipend":[],"duration":[]}
    hlo=get_soup(a,pageno=no).find(id="internship_list_container_1")
    internships=hlo.find_all(class_="container-fluid individual_internship view_detail_button visibilityTrackerItem")
    for i in internships:
    company=i.find("p",class_="company-name").text.strip()
    role=i.find("a",class_="job-title-href").text.strip()
    location=i.find("div",class_="row-1-item locations").find("a").text.strip()
    stipend=i.find("span",class_="stipend").text.strip()
    duration=i.find_all("div",class_="row-1-item")[2].find("span").text.strip()
    info["company"].append(company)
    info["role"].append(role)
    info["location"].append(location)
    info["stipend"].append(stipend)
    info["duration"].append(duration)

    df=pd.DataFrame(info)
    return df
    print(get_soup(a))
    info={"company":[],"role":[],"location":[],"stipend":[],"duration":[]}
    dfm=pd.DataFrame(info)
    for i in range(1,int(get_pageno())+1):
    dfm=pd.concat([dfm,get_info(i)])
    time.sleep(1)
    print(dfm)

    Continue reading...

Compartilhe esta Página