[Python] Asyncio Proxy Checking

I am writing a script to determine working proxy servers. Everything runs smoothly with the Synchronous class, however, when I want to do this with asynchronous programming, I get very different results. Effectively nothing else has changed as far as I'm aware.

Async Class:

class FreeProxyList(Proxy):
def __init__(self, testEach=False):
self.testEach = testEach
self.proxies = []
self.session = None

async def setup(self):
async with aiohttp.ClientSession() as session:
async with session.get("https://free-proxy-list.net/") as response:
html = await response.text()
souped = BeautifulSoup(html, "html.parser")
table = souped.find_all("table")[0]
headers = [header.text for header in table.find_all("th")]

proxies_list = []
for row in table.find_all("tr")[1:]:
columns = row.find_all("td")
if columns:
proxy_info = {headers: columns.text for i in range(len(headers))}
proxies_list.append(proxy_info)

# Filter proxies
proxies_list = list(filter(lambda x: x['Https'] == "yes", proxies_list))
# proxies_list = list(filter(lambda x: x['Code'] in ["US", "CA", "GB", "AU", "NZ"], proxies_list))

if self.testEach:
self.proxies = []
tasks = [self.testOne(session, proxy, index, len(proxies_list)) for index, proxy in enumerate(proxies_list)]
results = await asyncio.gather(*tasks)
self.proxies = [proxy for proxy, result in zip(proxies_list, results) if result]

async def testOne(self, session, proxy, index, total):
import time

start = time.time()
try:
print(f"{index + 1} / {total :<5} Testing {proxy['IP Address']}:{proxy['Port']}")
async with session.get("https://reedgraff.com", proxy=f"http://{proxy['IP Address']}:{proxy['Port']}", timeout=5) as response:
if response.status == 200:
return time.time() - start
else:
return None
except KeyboardInterrupt:
raise KeyboardInterrupt
except:
return None


Synchronous Class:

class FreeProxyList(Proxy):
def __init__(self, testEach=False):
self.testEach = testEach
self.proxies = []
self.session = None

def setup(self):
import bs4
import requests

html = requests.get("https://free-proxy-list.net/")
souped = bs4.BeautifulSoup(html.text, "html.parser")
table = souped.find_all("table")[0]
headers = [header.text for header in table.find_all("th")]

proxies_list = []
for row in table.find_all("tr")[1:]:
columns = row.find_all("td")
if columns:
proxy_info = {headers: columns.text for i in range(len(headers))}
proxies_list.append(proxy_info)

# Filter proxies
proxies_list = list(filter(lambda x: x['Https'] == "yes", proxies_list))
# proxies_list = list(filter(lambda x: x['Code'] in ["US", "CA", "GB", "AU", "NZ"], proxies_list))

if self.testEach:
self.proxies = []
for index, proxy in enumerate(proxies_list):
print(f"{index + 1} / {len(proxies_list) :<5} Testing {proxy['IP Address']}:{proxy['Port']}")
if self.testOne(proxy):
self.proxies.append(proxy)

def testOne(self, proxy):
import requests
import time

start = time.time()
try:
response = requests.get("https://reedgraff.com", proxies={"https": f"{proxy['IP Address']}:{proxy['Port']}", "http": f"{proxy['IP Address']}:{proxy['Port']}"}, timeout=5)
if response.status_code == 200:
return time.time() - start
else:
return None
except KeyboardInterrupt:
raise KeyboardInterrupt
except:
return None


When running the synchronous program almost all of the IPs work, however, when running the async program none or one work.

Continue reading...