[Python] How is this Python script retrieving Youtube URLs from Discogs?

Discussão em 'Python' iniciado por Stack, Abril 1, 2025.

  1. Stack

    Stack Membro Participativo

    The purpose of the following code from here is to extract Youtube URLs from the Discogs API.

    It provides a JSON version of a list of releases on Discogs according to a particular search query, and an HTML example of this page would be:


    It then looks at each release and writes the Youtube URL to a text file.

    How does the following code extract Youtube URLs, given that data does not the contain the key videos?

    # Copy in your search URL, replace the parameters after /search?
    SEARCH_URL = 'https://api.discogs.com/database/search?sort=title&sortorder=desc&style="Dub+Techno"&format=Vinyl&type=release&year=1999&per_page=50'

    # Generate an access token at https://www.discogs.com/settings/developers
    ACCESS_TOKEN = #insert your own access token here

    # Some queries have a lot of results, limit the number by setting this
    PAGE_LIMIT = 25

    import requests
    import time
    import sys
    from urllib.parse import parse_qs, urlparse
    from queue import Queue

    INITIAL_URL = SEARCH_URL + '&token=' + ACCESS_TOKEN

    q = Queue()
    q.put(INITIAL_URL)

    videos = []
    file = open('video_urls.txt', 'w')


    def discogs_request(req_url):
    response = requests.get(
    req_url, headers={'User-agent': 'SearchToVideoList/1.0'})

    if response.status_code == 429:
    time.sleep(60)
    return discogs_request(req_url)
    elif response.status_code == 200:
    return response.json()


    while not q.empty():
    url = q.get()
    parsed_url = parse_qs(urlparse(url).query, keep_blank_values=True)

    data = discogs_request(url)

    if data is None:
    continue

    if 'page' in parsed_url and int(parsed_url['page'][0]) > PAGE_LIMIT:
    print('Page limit reached, exiting')
    continue

    if url is INITIAL_URL:
    print('Crawling %s releases in %s pages' % (
    data['pagination']['items'], data['pagination']['pages']))

    if 'results' in data:
    for release in data['results']:
    q.put(release['resource_url'] + '?token=' + ACCESS_TOKEN)

    if 'videos' in data:
    print(data['videos'])
    for video in data['videos']:
    file.write("%s\n" % video['uri'])
    print("Writing the following video to text: {}".format(video['uri']))
    file.flush()

    if 'pagination' in data:
    print('Current page: %s' % data['pagination']['page'])

    if 'next' in data['pagination']['urls']:
    q.put(data['pagination']['urls']['next'])

    q.task_done()



    As I have only recently been acquainted with the mentioned Python modules, I tried to understand the code by abstracting away the queue in the code, so I began by running instead,

    # Copy in your search URL, replace the parameters after /search?
    SEARCH_URL = 'https://api.discogs.com/database/search?sort=title&sortorder=desc&style="Dub+Techno"&format=Vinyl&type=release&year=1999&per_page=50'

    # Generate an access token at https://www.discogs.com/settings/developers
    ACCESS_TOKEN = #insert your own access token here.

    # Some queries have a lot of results, limit the number by setting this
    PAGE_LIMIT = 25

    import requests
    import time
    import sys
    from urllib.parse import parse_qs, urlparse
    from queue import Queue

    INITIAL_URL = SEARCH_URL + '&token=' + ACCESS_TOKEN

    # This function is a wrapper for the requests module, which is accessing the HTTP gateway.
    # requests.get() returns a response object, indicating whether the HTTP resource is available.
    def discogs_request(req_url):
    response = requests.get(
    req_url, headers={'User-agent': 'SearchToVideoList/1.0'})

    if response.status_code == 429:
    time.sleep(60)
    return discogs_request(req_url)
    elif response.status_code == 200:
    return response.json()

    # The first URL in the queue will be the initial Discogs search for releases that satisfy our criteria/query.
    url = INITIAL_URL
    print("The URL we are working on is: {}".format(url))

    # This line uses urllib.parse to split the given URL into 6 components.
    # The general structure of a URL is `scheme://netloc/path;parameters?query#fragment`
    print("The parsed URL is: {}".format(urlparse(url)))

    # We now only want the 'query' component of the URL, so we use a method to call the values of 'query'.
    print("The query components of the URL are: {}".format(urlparse(url).query))

    # We now convert the value of `query` to a dictionary,
    parsed_url = parse_qs(urlparse(url).query, keep_blank_values=True)
    print("The query components of the URL as a dictionary are: {}".format(parsed_url))

    # Now store the retrieved data JSON.
    data = discogs_request(url)


    However, the following is returned False after I run,

    'videos' in data


    Here is a sample of data from my script:

    {'pagination': {'page': 1,
    'pages': 3,
    'per_page': 50,
    'items': 113,
    'urls': {'last': 'https://api.discogs.com/database/search?sort=title&sortorder=desc&style=%22Dub+Techno%22&format=Vinyl&type=release&year=1999&per_page=50&token=###&page=3',
    'next': 'https://api.discogs.com/database/search?sort=title&sortorder=desc&style=%22Dub+Techno%22&format=Vinyl&type=release&year=1999&per_page=50&token=###&page=2'}},
    'results': [{'country': 'Germany',
    'year': '1999',
    'format': ['Vinyl', '12"', 'White Label'],
    'label': ['Force Inc. Music Works', 'SST Brüggemann GmbH', 'MPO'],
    'type': 'release',
    'genre': ['Electronic'],
    'style': ['Dub Techno', 'Minimal'],
    'id': 1136089,
    'barcode': [],
    'user_data': {'in_wantlist': False, 'in_collection': False},
    'master_id': 83020,
    'master_url': 'https://api.discogs.com/masters/83020',
    'uri': '/Exos-Yellow-Yard/release/1136089',
    'catno': 'FIM 174',
    'title': 'Exos - Yellow Yard',
    'thumb': 'https://i.discogs.com/-Jz0cMoRi-g0vgwmUZOs3P6i_HyFva_fobHjY303-II/rs:fit/g:sm/q:40/h:150/w:150/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTExMzYw/ODktMTI0NDY3MjIz/My5qcGVn.jpeg',
    'cover_image': 'https://i.discogs.com/s5-uqKQaR-CXrpj8dSwWsl3Z0K9ZJs1G9w_ZwgUrsVQ/rs:fit/g:sm/q:90/h:600/w:596/czM6Ly9kaXNjb2dz/LWRhdGFiYXNlLWlt/YWdlcy9SLTExMzYw/ODktMTI0NDY3MjIz/My5qcGVn.jpeg',
    'resource_url': 'https://api.discogs.com/releases/1136089',
    'community': {'want': 270, 'have': 28},
    'format_quantity': 1,
    'formats': [{'name': 'Vinyl',
    'qty': '1',
    'descriptions': ['12"', 'White Label']}]},
    {'country': 'Germany',
    'year': '1999',
    'format': ['Vinyl', '12"', '33 ⅓ RPM', 'White Label'],
    'label': ['Profan', 'SST Brüggemann GmbH', 'MPO'],
    'type': 'release',
    'genre': ['Electronic'],
    'style': ['Dub Techno', 'Minimal Techno', 'Tech House'],
    'id': 9133409,
    'barcode': ['MPO PRO 028 A 33 RPM K SST', 'MPO PRO 028 B 33 RPM K SST'],
    'user_data': {'in_wantlist': False, 'in_collection': False},
    'master_id': 327526,
    'master_url': 'https://api.discogs.com/masters/327526',
    'uri': '/Wassermann-W-I-R-Das-Original-Sven-V%C3%A4th-Mix-Thomas-Mayer-Mix/release/9133409',
    'catno': 'PROFAN 028',
    'title': 'Wassermann - W. I. R. (Das Original + Sven Väth Mix Thomas / Mayer Mix)',
    'thumb': '',
    'cover_image': 'https://st.discogs.com/1504bf7e69cad5ced79c9e7b6cf62bda18dce7eb/images/spacer.gif',
    'resource_url': 'https://api.discogs.com/releases/9133409',
    'community': {'want': 129, 'have': 23},
    'format_quantity': 1,
    'formats': [{'name': 'Vinyl',
    'qty': '1',
    'descriptions': ['12"', '33 ⅓ RPM', 'White Label']}]},


    But when I put print statements to see what is being written to the .txt file in the original 1st code block, it's clear that it's writing videos to the file. So what is going on?

    Continue reading...
    Stack, Abril 1, 2025
    #1

Compartilhe esta Página