爬虫学习3：爬虫的深度爬取

爬虫的深度爬取和爬取视频的方式

深度爬取豆瓣读书

import time
import fake_useragent
import requests
from lxml import etree
head = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0"
}
if __name__ == '__main__':# 1、urlurl = "https://book.douban.com/tag/%E4%BA%92%E8%81%94%E7%BD%91"#url地址# 2、发送请求resp = requests.get(url, headers=head)time.sleep(5)# 3、获取想要的数据res_text = resp.text#获得html形式的数据# 4、数据解析tree = etree.HTML(res_text)#将html转化为元素形式a_list = tree.xpath("//ul[@class='subject-list']/li/div[2]/h2/a")for a in a_list:time.sleep(3)# 1、urlbook_url = "".join(a.xpath("./@href"))#里面的网页的url# 2、发送请求book_res = requests.get(book_url, headers=head)# 3、获取想要的信息book_text = book_res.text# 4、数据解析book_tree = etree.HTML(book_text)book_name = "".join(book_tree.xpath("//span[@property='v:itemreviewed']/text()"))author = "".join(book_tree.xpath("//div[@class='subject clearfix']/div[2]/span[1]/a/text()"))publish = "".join(book_tree.xpath("//div[@class='subject clearfix']/div[2]/a[1]/text()"))y = "".join(book_tree.xpath("//span[@class='pl' and text()='出版年:']/following-sibling::text()[1]"))page = "".join(book_tree.xpath("//span[@class='pl' and text()='页数:']/following-sibling::text()[1]"))price = "".join(book_tree.xpath("//span[@class='pl' and text()='定价:']/following-sibling::text()[1]"))bind = "".join(book_tree.xpath("//span[@class='pl' and text()='装帧:']/following-sibling::text()[1]"))isbn = "".join(book_tree.xpath("//span[@class='pl' and text()='ISBN:']/following-sibling::text()[1]"))print(book_name, author, publish, y, page, price, bind, isbn)pass

代码思路

![在这里插入图片描述](https://i-blog.csdnimg.cn/direct/1769d142fe7f40c1b401416884ba93ec.jpeg#pic_center

B站爬取视频

# requests 请求b站视频
import json
import fake_useragent
import requests
from lxml import etree
if __name__ == '__main__':# UA伪装head = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0"# 防盗链, "Referer": "https://www.bilibili.com/"#必须有,"Cookie": "这里是登录要求，输入个人的登录信息"}# 1、指定urlurl = "https://www.bilibili.com/video/BV12z421z75d/?spm_id_from=333.1007.tianma.2-1-4.click&vd_source=4b1ef23e5d47e143cfb702705740719d"# 2、发送请求response = requests.get(url, headers=head)# 3、获取响应的数据res_text = response.text# 4、数据解析tree = etree.HTML(res_text)with open("b.html", "w", encoding="utf8") as f:f.write(res_text)base_info = "".join(tree.xpath("/html/head/script[4]/text()"))[20:]#b站固定的位置html/head/script[4]，前面的不需要，只有括号里的字符串有用所有取20以后的info_dict = json.loads(base_info)video_url = info_dict["data"]["dash"]['video'][0]["baseUrl"]#视频audio_url = info_dict["data"]["dash"]['audio'][0]["baseUrl"]#音频video_content = requests.get(video_url, head).content#和图片一样采用contentaudio_content = requests.get(audio_url, head).contentwith open("video.wmv", "wb") as f:#以二进制的形式f.write(video_content)with open("audio.mp4", "wb") as fp:fp.write(audio_content)pass