Using Proxies in Python 3

Question

i am using proxies to make crawlers:

import requests
from bs4 import BeautifulSoup
import time 
time.sleep(3)  
import random

proxy_list = [
'66.82.144.29:8080', 
'47.75.0.253:3129',    
'217.119.82.14:8080' ]

proxies = random.choice(proxy_list)


for i in range(20):

    url = "https://www.amazon.com/s/ref=sr_pg_{}".format(i) + "?fst=p90x%3A1%2Cas%3Aoff&rh=n%3A172282%2Cn%3A541966%2Cn%3A193870011%2Cn%3A172500%2Ck%3Acorsair+ddr4%2Cp_89%3ACorsair&page={}".format(i) + "&keywords=corsair+ddr4&ie=UTF8&qid=1522049082"
    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0'}
    response = requests.get(url, verify=False, headers=headers, proxies=proxies)
    soup = BeautifulSoup(response.text.encode('utf-8'), 'html.parser')

but there is error keep poping out

AttributeError                            Traceback (most recent call last)
<ipython-input-7-82c14c70f937> in <module>()
     25         url = "https://www.amazon.com/s/ref=sr_pg_{}".format(i) + "?fst=p90x%3A1%2Cas%3Aoff&rh=n%3A172282%2Cn%3A541966%2Cn%3A193870011%2Cn%3A172500%2Ck%3Acorsair+ddr4%2Cp_89%3ACorsair&page={}".format(i) + "&keywords=corsair+ddr4&ie=UTF8&qid=1522049082"
     26         headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0'}
---> 27         response = requests.get(url, verify=False, headers=headers, proxies=proxies)
     28         soup = BeautifulSoup(response.text.encode('utf-8'), 'html.parser')
     29         containers = soup.select('li.s-result-item.celwidget')

c:\users\terry\appdata\local\programs\python\python36-32\lib\site-packages\requests\api.py in get(url, params, **kwargs)
     70 
     71     kwargs.setdefault('allow_redirects', True)
---> 72     return request('get', url, params=params, **kwargs)
     73 
     74 

c:\users\terry\appdata\local\programs\python\python36-32\lib\site-packages\requests\api.py in request(method, url, **kwargs)
     56     # cases, and look like a memory leak in others.
     57     with sessions.Session() as session:
---> 58         return session.request(method=method, url=url, **kwargs)
     59 
     60 

c:\users\terry\appdata\local\programs\python\python36-32\lib\site-packages\requests\sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    497 
    498         settings = self.merge_environment_settings(
--> 499             prep.url, proxies, stream, verify, cert
    500         )
    501 

c:\users\terry\appdata\local\programs\python\python36-32\lib\site-packages\requests\sessions.py in merge_environment_settings(self, url, proxies, stream, verify, cert)
    669         if self.trust_env:
    670             # Set environment's proxies.
--> 671             no_proxy = proxies.get('no_proxy') if proxies is not None else None
    672             env_proxies = get_environ_proxies(url, no_proxy=no_proxy)
    673             for (k, v) in env_proxies.items():

AttributeError: 'str' object has no attribute 'get'

what happens

Duncan · Accepted Answer · 2018-04-11 08:34:25Z

2

When you pass proxies to the requests library the object passed should be a mapping from protocol to proxy.

import requests

proxies = {
  'http': 'http://10.10.1.10:3128',
  'https': 'http://10.10.1.10:1080',
}

requests.get('http://example.org', proxies=proxies)

See http://docs.python-requests.org/en/master/user/advanced/#proxies

So in your case try:

proxy_list = [
'http://66.82.144.29:8080', 
'http://47.75.0.253:3129',    
'http://217.119.82.14:8080' ]

selected = random.choice(proxy_list)
proxies = { protocol: selected for protocol in ('http', 'https') }

Note that you also have to include the protocol as the proxy server itself could be using http, https, or indeed socks5.

answered Apr 11, 2018 at 8:34

Duncan

96.4k15 gold badges129 silver badges160 bronze badges

Sign up to request clarification or add additional context in comments.

Collectives™ on Stack Overflow

Using Proxies in Python 3

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related