pythonrequests代理ip_python使用requests模块使用ip代理池

import json

import telnetlib

import requests

import random

# 代理ip列表

proxy_url = “https://raw.githubusercontent.com/fate0/proxylist/master/proxy.list”

# 写入可用ip代理池文件路径

ip_pool_file = “verified_proxies.json”

# 用于测试代理ip是否可用的网站

test_url = “http://icanhazip.com/”

# user-agent头

USER_AGENTS = [

“Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)”,

“Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)”,

“Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6”,

“Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1”,

“Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0”,

“Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5”,

“Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6”,

“Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11”,

“Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52”,

“Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)”,

“Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)”,

“Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)”,

“Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)”,

“Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1”,

“Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1”,

“Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre”,

“Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0”,

“Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11”,

“Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10”,

]

# 从ip网站上爬取所有代理ip

def get_proxy(proxy_url):

response = requests.get(proxy_url)

proxy_list = response.text.split(“\n”)

for proxy_str in proxy_list:

proxy_json = json.loads(proxy_str)

host = proxy_json[“host”]

port = proxy_json[“port”]

ip_type = proxy_json[“type”]

check_and_save_ip(host, port, ip_type)

# 测试ip是否可用，可用的话存入文件

def check_and_save_ip(ip, port, ip_type):

proxies = {}

try:

# 测试是否能使用

telnet = telnetlib.Telnet(ip, port=port, timeout=3)

except Exception:

print(‘unconnected’)

else:

print(‘connected successfully’)

proxies[‘type’] = ip_type

proxies[‘host’] = ip

proxies[‘port’] = port

proxies_json = json.dumps(proxies)

with open(ip_pool_file, ‘a+’) as fp:

fp.write(proxies_json + ‘\n’)

print(“已写入：%s” % proxies)

# 随机获取一个UA头

def get_request_headers():

headers = {

‘User-Agent’: random.choice(USER_AGENTS),

‘Accept’: ‘text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8’,

‘Accept-Language’: ‘en-US,en;q=0.5’,

‘Connection’: ‘keep-alive’,

}

return headers

# 这里只是测试

def use_proxy():

proxies = []

with open(ip_pool_file, “r”) as fp:

while True:

item = fp.readline()

if item:

# print(type(json.loads(item)))

proxies.append(json.loads(item))

else:

break

# print(type(proxies[1]))

for item in proxies:

ip = item[‘host’]

port = item[‘port’]

# print(ip, port)

proxies_param = {

‘http’: ‘%s:%s’%(ip, port)

}

print(proxies_param)

try:

# 发送请求,获取响应数据

response = requests.get(test_url, headers=get_request_headers(), proxies=proxies_param, timeout=5)

if response.ok:

# 把响应的json字符串转换为字典

# dic = json.loads(response.text)

# print(dic[‘origin’])

print(response.content)

except Exception as ex:

print(ex)

if __name__ == “__main__”:

# get_proxy(proxy_url)

use_proxy()

# res = requests.get(‘http://icanhazip.com/’, proxies={‘http’: ‘167.99.145.189:3128’})

# print(res.content)