用python爬取网页内容_用python爬取网页文本内容，让你快速浏览网站内容

大家好，我是咿哑呀。今天我教大家学习网络爬虫，这个爬虫能够爬取网页文本内容，提高大家浏览网页的速度。下面我把代码贴出，大家仔细研究：

from urllib import request, parse

from urllib.parse import quote

import string

import chardet

from bs4 import BeautifulSoup

import re

import time

# 创建类

class spider:

# 构造函数 初始化

def __init__(self, my_root_url, title_tag, con_tag, OutputFile):

self.new_urls = set() # 待爬取的url

self.old_urls = set() # 已爬取的url

self.datas = [] # 存放搜集的数据

# 添加一个到url中

self.add_new_url(my_root_url)

count = 1

while self.has_new_url():

try:

new_url = self.get_new_url()

print(‘%d、爬取 %s’ % (count, new_url))

html_context = self.download(new_url)

new_urls, new_data = self.get_new_urls_data(new_url, html_context, title_tag, con_tag)

self.add_new_urls(new_urls)

self.output_html(new_data, OutputFile)

count += 1

time.sleep(1)

except():

print(“爬取失败”)

def download(self, new_url):

if new_url is None:

return None

headers = (“User-Agent”,

“Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063”)

opener = request.build_opener()

opener.addheaders = [headers]

request.install_opener(opener)

# 解决请求路径中含义中文或特殊字符

url_ = quote(new_url, safe=string.printable)

if request.urlopen(url_).getcode() != 200:

return None

else:

html = request.urlopen(url_).read()

charset = chardet.detect(html)[‘encoding’]

return html.decode(charset, ‘ignore’)

def output_html(self, new_data, OutputFile):

if new_data is None:

return

self.datas.append(new_data)

for data in self.datas:

if data[‘title’] and data[‘con’] and data[‘url’]:

fout = open(OutputFile+’\\’+data[‘title’].replace(‘ ‘, ”).replace(‘\n’,”)+’.txt’, ‘w’, encoding=’utf8′)

fout.write(‘%s’ % data[‘title’].replace(‘ ‘, ”).replace(‘\n’,”))

fout.write(‘%s’ % data[‘con’].replace(‘ ‘, ”))

fout.write(‘(来源:%s)’ % data[‘url’])

fout.close()

def get_new_urls_data(self, page_url, html_context, title_tag, con_tag):

if page_url is None or html_context is None:

return

new_urls = set()

red_data = {}

soup = BeautifulSoup(html_context, “html.parser”)

pat = re.compile(‘.htm|.asp’)

links = soup.find_all(href=pat)

for link in links:

if page_url not in link[“href”]:

new_url = link[“href”]

new_full_url = parse.urljoin(page_url, new_url)

new_urls.add(new_full_url)

red_data[‘url’] = page_url

if soup.find(class_=title_tag[0]):

title_node = soup.find(class_=title_tag[0])

if title_node.get_text():

red_data[‘title’] = title_node.get_text()

else:

red_data[‘title’] = “”

else:

red_data[‘title’] = “”

con_node = soup.find(class_=con_tag[0])

if con_node:

red_data[‘con’] = con_node.get_text()

else:

red_data[‘con’] = “”

return new_urls, red_data

# 向管理器中添加一个新的url

def add_new_url(self, my_root_url):

if my_root_url is None:

return

if my_root_url not in self.new_urls and my_root_url not in self.old_urls:

self.new_urls.add(my_root_url)

# 向管理器中添加批量新的url

def add_new_urls(self, urls):

if urls is None or len(urls) == 0:

return

for url in urls:

self.add_new_url(url)

# 判断是否有新的待爬取的url

def has_new_url(self):

return len(self.new_urls) != 0

# 获取一个待爬取的url

def get_new_url(self):

new_url = self.new_urls.pop()

self.old_urls.add(new_url)

return new_url

代码的主要意思就是先导入一些必须的库，然后创建一个爬虫类，下面的主函数就是运用这个类进行爬取网页文本的：

# 创建main方法

if __name__ == “__main__”:

root_url = “http://www..cn”

#