多线程爬取百思不得姐的图片和文字信息并保存到本地csv中

Author: xingag

README.md

@@ -1,13 +1,22 @@
 # spider_python
+## 普通的爬虫
+
 * [爬取电影天堂最新的电影数据 - xpath](./spiders/spider_dytt.py)
 
 * [爬取腾讯招聘的职位数据 - xpath](./spiders/spider_tencent_recruit.py)
 
-
-
 * [爬取中国天气网全国天气并生成饼状图 - bs4](./spiders/spider_china_weather.py)
+
 * [爬取古诗词网的数据 - re](./spiders/spider_gushiwen.py)
+
 * [爬取糗事百科上的段子数据 - re](./spiders/spider_qiu_shi_bai_ke.py)
+
+  ​
+
+
+## 多线程爬虫
+
 * [多线程爬取斗图吧的表情图并下载到本地 - xpath + threading](./spiders/spider_dou_tu_la.py)
+* [多线程爬取百思不得姐的文字和图片信息并写入到csv中](./spiders/spider_bai_si_bu_de_jie.py)
 
 

spiders/spider_bai_si_bu_de_jie.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python  
+# encoding: utf-8  
+
+""" 
+@version: v1.0 
+@author: xag 
+@license: Apache Licence  
+@contact: [email protected] 
+@site: http://www.xingag.top 
+@software: PyCharm 
+@file: spider_bai_si_bu_de_jie.py
+@time: 2018/9/25 19:58 
+@description：利用多线程爬取【百思不得姐】网站的文字和图片并下载到csv文件中
+"""
+
+import requests
+from lxml import etree
+import threading
+from queue import Queue
+import time
+import csv
+from urllib import request
+import fileutils
+
+HEADERS = {
+	'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
+	'Referer': 'http://www.budejie.com/hot/1'
+}
+
+
+class BSSpider(threading.Thread):
+	"""
+	爬取每一页的数据
+	"""
+
+	def __init__(self, page_queue, joke_queue, name, *args, **kwargs):
+		super(BSSpider, self).__init__(*args, **kwargs)
+
+		# 1.初始化数据
+		self.page_queue = page_queue
+		self.joke_queue = joke_queue
+		self.name = name
+
+	def run(self):
+		while True:
+			# 2.如果页面队列为空，就退出循环
+			if self.page_queue.empty():
+				print(self.name + '任务完成~')
+				# while not self.joke_queue.empty():
+				# 	print(self.joke_queue.get())
+				break
+
+			# 3.从队列中获取页面地址
+			page_url = self.page_queue.get()
+			self.spider_page(page_url)
+
+			# 6.休眠0.5秒
+			time.sleep(0.5)
+
+	def spider_page(self, page_url):
+		"""
+		爬取一页的数据
+		:param page_url:页面的url
+		:return:
+		"""
+		response = requests.get(page_url, headers=HEADERS)
+		text_raw = response.text
+		html_element = etree.HTML(text_raw)
+
+		# 4.利用xpath去解析数据
+		div_elements = html_element.xpath('//div[@class="j-r-list"]')
+
+		for div_element in div_elements:
+			duan_zi_elments = div_element.xpath('./ul/li')
+			for duan_zi_elment in duan_zi_elments:
+				# 【数据】用户名
+				username = duan_zi_elment.xpath('.//a[@class="u-user-name"]/text()')[0]
+
+				# 【数据】段子发布时间
+				pubtime = duan_zi_elment.xpath('.//span/text()')[0]
+
+				desc_element = duan_zi_elment.xpath('.//div[@class="j-r-list-c-desc"]')[0]
+				# 【数据】段子描述内容
+				content = desc_element.xpath('./a/text()')[0]
+
+				img_div_element = duan_zi_elment.xpath('.//div[@class="j-r-list-c-img"]')[0]
+				img = img_div_element.xpath('.//img/@data-original')[0]
+				alt = img_div_element.xpath('.//img/@alt')[0]
+
+				# 5.把解析后的数据以元组的方式放入到队列中去
+				self.joke_queue.put((username, content, img, alt, pubtime))
+
+
+class BSWriter(threading.Thread):
+	"""
+	下载图片、写入文字数据到csv文件中
+	"""
+
+	def __init__(self, page_queue, joke_queue, writer, gLock, name, *args, **kwargs):
+		super(BSWriter, self).__init__(*args, **kwargs)
+
+		# 1.初始化
+		self.page_queue = page_queue
+		self.joke_queue = joke_queue
+		self.writer = writer
+		self.gLock = gLock
+		self.name = name
+
+	def run(self):
+		while True:
+			if self.joke_queue.empty() and self.page_queue.empty():
+				print(self.name + '任务完成~')
+				break
+
+			# 2.从joke_queue队列中获取数据
+			joke_info = self.joke_queue.get(timeout=40)
+			username, content, img, alt, pubtime = joke_info
+
+			# 3.上锁
+			self.gLock.acquire()
+
+			# 4.写入数据到csv中
+			self.writer.writerow((username, content, img, alt, pubtime))
+
+			# 5.下载图片到本地
+			# file_name = alt + fileutils.get_file_suffix(img)
+			# request.urlretrieve(img, './imgs/%s' % file_name)
+
+			# 6.释放锁
+			self.gLock.release()
+
+			print('写入一条数据成功')
+
+
+class BSDownImg(threading.Thread):
+	"""
+	下载图片的消费者
+	"""
+
+	def __init__(self, page_queue, joke_queue, gLock, name, *args, **kwargs):
+		super(BSDownImg, self).__init__(*args, **kwargs)
+		self.page_queue = page_queue
+		self.joke_queue = joke_queue
+		self.gLock = gLock
+		self.name = name
+
+	def run(self):
+		while True:
+			if self.joke_queue.empty() and self.page_queue.empty():
+				print(self.name + '任务完成~')
+				break
+			username, content, img, alt, pubtime = self.joke_queue.get(timeout=40)
+
+			# 上锁并下载图片
+			self.gLock.acquire()
+			file_name = alt + fileutils.get_file_suffix(img)
+			request.urlretrieve(img, './imgs/%s' % file_name)
+			self.gLock.release()
+
+			print('下载一张图片成功')
+
+
+def spider():
+	"""
+	爬取百思不得姐的前20页数据
+	:return:
+	"""
+
+	# 1.构建队列【生产者、消费者需要上锁的对象】
+	page_queue = Queue(20)
+	joke_queue = Queue(200)
+
+	# 2.锁对象
+	gLock = threading.Lock()
+
+	# 3.写入
+	fp = open('jokes.csv', 'a', newline='', encoding='utf-8')
+	writer = csv.writer(fp)
+
+	# 4.写入csv表头信息
+	writer.writerow(['username', 'content', 'img', 'alt', 'pubtime'])
+
+	# 5.前10页待爬取的地址，放入到队列中
+	for page_num in range(1, 11):
+		page_url = 'http://www.budejie.com/hot/%d' % page_num
+		page_queue.put(page_url)
+
+	# 6.构建10个生成者来进行爬虫
+	for x in range(1, 6):
+		t = BSSpider(page_queue, joke_queue, name='生产者%d' % x)
+		t.start()
+
+	# 7.构建 20 个消费者来写入数据到csv文件中
+	for x in range(1, 21):
+		t = BSWriter(page_queue, joke_queue, writer, gLock, name='消费者-文字%d' % x)
+		t.start()
+
+	# 8.构建 50 个消费者来下载图片
+	for x in range(1, 51):
+		t = BSDownImg(page_queue, joke_queue, gLock, name='消费者-图片%d' % x)
+		t.start()
+
+
+if __name__ == '__main__':
+	spider()