Skip to content
This repository has been archived by the owner on Jun 18, 2022. It is now read-only.

Update crawl_proxy_servers.py #4

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Update crawl_proxy_servers.py #4

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0a17218
Update crawl_proxy_servers.py
nubotz Nov 11, 2016
bda46de
Update crawl_proxy_servers.py
nubotz Feb 26, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions crawl_proxy_servers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def crawl_proxy_servers():
if page.status_code == requests.codes.ok:
bsobj = BeautifulSoup(page.text, 'html.parser')
try:
proxy_table = bsobj.find('div', {'class': 'table-container'})
proxy_table = bsobj.findAll('div', {'class': 'table-container'})[1]
proxy_body = proxy_table.find('table').find('tbody')
for tr in proxy_body.findAll('tr'):
proxy_info = tr.findAll('td')
Expand All @@ -47,7 +47,6 @@ def crawl_proxy_servers():

def test_proxy_server(proxy):
proxies = {'http': proxy}

try:
response = requests.get(test_url, proxies=proxies, timeout=10)
if response.status_code == requests.codes.ok:
Expand Down