python爬虫入门教程

安装python

中文网 Python中文网官网

安装好后打开命令行执行（如果没有勾选添加到Path则注意配置环境变量）

python

出现如上界面则安装成功

设置环境变量

右键我的电脑->属性

设置下载依赖源

默认的是官网比较慢，可以设置为清华大学的地址

python -m pip install --upgrade pip
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

正则表达式

查看地址正则表达式-CSDN博客

实操

创建项目后安装requests

pip install requests

入门

搜狗网址搜索周杰伦爬虫

获取地址

发现浏览器地址很长，可以简化为 https://www.sogou.com/web?query=周杰伦

上代码

import requests
url = "https://www.sogou.com/web?query=周杰伦"
res = requests.get(url)
print(res.text)
res.close()

结果

上面显示非浏览器发送，我们模拟浏览器发送，然后操作，打开浏览器控制台，访问 https://www.sogou.com/web?query=周杰伦，

那么我们在header中添加如上代码

import requests
url = "https://www.sogou.com/web?query=周杰伦"
# 加header是为了伪装为浏览器
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0"
}
res = requests.get(url, headers=headers)
print(res.text)
res.close()

成功拿到结果

将结果存到文件

import requests
url = "https://www.sogou.com/web?query=周杰伦"
# 加header是为了伪装为浏览器
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0"
}
f = open("sogou.html", "w", encoding="utf-8")
res = requests.get(url, headers=headers)
f.write(res.text)
f.close()
res.close()
print('ok')

百度翻译爬虫

import requests
url = "https://fanyi.baidu.com/sug"
key = input('请输入要翻译的文字')
data = {"kw" : key
}
res = requests.post(url,data)
print(res.json())
res.close()

爬取豆瓣电影排名

import requests
url = "https://movie.douban.com/j/chart/top_list"
data = {"type":"24","interval_id":"100:90","action":"","start":"0","limit":"20"
}
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"
}
res = requests.get(url,params=data,headers=headers)
print(res.json())
res.close()

使用re解析数据

re介绍

import re# 返回所有匹配的值
list = re.findall(r'\d+', '我的电话号码10086,我女朋友电话号码10010')
print(list)
# 将匹配结果放到迭代器中
iter = re.finditer(r'\d+', '我的电话号码10086,我女朋友电话号码10010')
print(iter)
for i in iter:print(i)print(i.group())
# 查到一个就返回
search = re.search(r'\d+', '我的电话号码10086,我女朋友电话号码10010')
print(search)
print(search.group())
# 从头开始匹配，不会匹配到非头的部分
match = re.match(r'\d+', '10086,我女朋友电话号码10010')
print(match.group())# 预加载
obj = re.compile(r'\d+')
compile = obj.finditer('我的电话号码10086,我女朋友电话号码10010')
print(compile)# .* 贪婪匹配 尽可能多的匹配
# .*? 懒惰匹配 尽可能少的匹配
s = """
<div class='jay'><span id='1'>周杰伦</span></div>
<div class='jj'><span id='2'>李连杰</span></div>
<div class='aa'><span id='3'>刘亦菲</span></div>
<div class='weew'><span id='4'>刘涛</span></div>
"""
# (?P<wsss>.*?) 这个是把后面的.*?匹配上的字符串存到wsss这个组里
# (?P<分组名>正则) 可以从匹配结果中提取出对应分组的字符串
obj = re.compile(r"<div class='.*?'><span id='(?P<id>\d+)'>(?P<wsss>.*?)</span></div>", flags=re.S)  #re.S让.能匹配换行符
res = obj.finditer(s)
for i in res:print(i.group('id'))print(i.group('wsss'))

爬取豆瓣top250解析并存到csv文件

# 拿到源代码 requests
# 分析数据  re
import csv
import re
import requests
url = 'https://movie.douban.com/top250'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36'
}
res = requests.get(url, headers=headers)
# re.S是让.可以匹配空格
obj = re.compile(r'<li>.*?<div class="item">.*?<span class="title">(?P<name>.*?)</span>.*?'r'<div class="star">.*?<span class="rating_num" property="v:average">(?P<grade>.*?)</span>.*?'r'<span>(?P<num>.*?)人评价</span>', re.S)
list = obj.finditer(res.text)
f = open('douban.csv', 'w', encoding='utf-8')
writer = csv.writer(f)
for i in list:dic = i.groupdict()dic['name'] = dic['name'].strip()dic['grade'] = dic['grade'].strip()dic['num'] = dic['num'].strip()writer.writerow(dic.values())
f.close()
print('ok')

爬取电影天堂比看热片下载地址

其中res.encoding='gb2312'是设置响应编码要和返回的html中的charset一致

# 获取电影天堂首页  查看必看热片
# 获取子页面数据
import requests
import re
domain = "https://www.dytt89.com/"
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}
res = requests.get(domain, headers=headers,verify=False)
# 设置响应编码 要和返回的html中的charset一致
res.encoding='gb2312'
obj = re.compile(r'必看热片.*?<ul>.*?</ul>',re.S)
result = obj.search(res.text)
list = result.group()
obj2 = re.compile(r"<li><a href='(?P<url>.*?)'.*?>(?P<name>.*?)</a>",re.S)
res2 = obj2.finditer(list)
dic = []
f = open('movie.txt','w',encoding='utf-8')
f.write('片名     迅雷下载地址'+'\n')
for item in res2:dicItem = item.groupdict()dicItem['name'] = dicItem['name'].strip()dicItem['url'] = domain + dicItem['url']dic.append(dicItem)
for item in dic:res3 = requests.get(item['url'],headers=headers,verify=False)res3.encoding='gb2312'obj3 = re.compile(r'◎片　　名(?P<name>.*?)<.*?<li><a href="(?P<url>.*?)"',re.S)res4 = obj3.finditer(res3.text)for item in res4:di = item.groupdict()di['name']=di['name'].strip()di['url']=di['url'].strip()f.write(di['name']+'  '+di['url']+'\n')
f.close()
print('ok')

使用bs4分析数据

安装使用bs4

pip install bs4

测试1

# 安装bs4 pip install bs4
from bs4 import BeautifulSoup
s = """
<div class='jay' id='sss'><span id='1'>周杰伦</span></div>
<div class='jj'><span id='2'>李连杰</span></div>
<div class='jj'><span id='3'>刘亦菲</span></div>
<div class='weew'><span id='4'>刘涛</span></div>
"""
page = BeautifulSoup(s, 'html.parser')
# find(标签名,属性名) 只找第一个
res0 = page.find('div',{'id':'sss'})
res1 = page.find('div',{'class':'jj'})
# find_all(标签名,属性名) 找全部
res2 = page.find_all('div',{'class':'jj'})print(res0.text)
print(res1.text)
print(res2)

测试2

# 安装bs4 pip install bs4
from bs4 import BeautifulSoup
html= """
<table id='table1' class='aa'>
<tr><th>编号</th><th>姓名</th><th>职位</th><th>年龄</th><th>部门</th>
</tr>
<tr><td>003</td><td>王五</td><td>设计师</td><td>25</td><td>设计部</td>
</tr>
<tr><td>004</td><td>赵六</td><td>市场经理</td><td>35</td><td>市场部</td>
</tr>
</table>
<table id='table2' class='aa'><tr><th>编号</th><th>姓名</th><th>职位</th><th>年龄</th><th>部门</th></tr><tr><td>001</td><td>张三</td><td>软件工程师</td><td>28</td><td>技术部</td></tr><tr><td>002</td><td>李四</td><td>产品经理</td><td>32</td><td>产品部</td></tr>
</table>
"""
page = BeautifulSoup(html, 'html.parser')
# find(标签名,属性名) 只找第一个
res1 = page.find('table',{'class':'aa'})
# find_all(标签名,属性名) 找全部
res2 = page.find_all('table',{'class':'aa'})
# 下面切片从1开始，因为第一行是表头
trs = res1.find_all('tr')[1:]
for tr in trs:tds = tr.find_all('td')# 这边的.text是获取标签内的文本print([tds[0].text,tds[1].text,tds[2].text,tds[3].text,tds[4].text])

下载图片

import requests
from bs4 import BeautifulSoup
url = 'https://wap.umei.cc/gaoxiaotupian/'
# 获取网页源码
res = requests.get(url)
res.encoding = 'utf-8'
page = BeautifulSoup(res.text, 'html.parser')
# 获取所有ul标签
ul = page.find('ul')
# 获取所有img标签
imgs = ul.find_all('img')
for img in imgs:# 获取图片地址# 这边两种取值方法都可以 img.get('data-original') 和 img['data-original']imgUrl = img.get('data-original')# print(img['data-original'])# 下载图片# 文件名fileName = imgUrl.split('/')[-1]with open('img/'+fileName, 'wb') as f:f.write(requests.get(imgUrl).content)print('下载图片：', fileName+'成功')
res.close()
print('下载完成')

使用lxml解析数据

安装使用lxml

pip install lxml

lxml是根据路径来获取数据的，可以使用 * 作为通配符，如果获取的结果是多个则多个都返回到结果中，其中/text()是获取标签内的值，//可以表示多个文件夹，例如/book//author/text()表示获取book下面所有的author标签的值，//author/text()这个表示获取标签中所有的author的值

from lxml import etree
xml= """
<book><name>编程之美</name><price>79.99</price><author>张伟</author><author>丽丽</author><div><author>傻傻</author></div><span><author>傻傻span</author></span><div><div><author>哈哈</author></div></div>
</book>
"""
book = etree.XML(xml)
# 根据路径来获取元素   text()是获取元素内的文本
name = book.xpath('/book/name/text()')
print(name)
# 匹配多个的时候会把所有的元素都返回
author = book.xpath('/book/author/text()')
print(author)
# 获取div下面的author元素
div_author = book.xpath('/book/div/author/text()')
print(div_author)
# 获取div下面的所有的author元素
div_all_author = book.xpath('/book/div//author/text()')
print(div_all_author)
# 获取book下所有的author元素
allbook_author = book.xpath('/book//author/text()')
print(allbook_author)
# 获取所有的author元素
all_author = book.xpath('//author/text()')
# 获取 傻傻 和 傻傻span  *是通配符
author_span = book.xpath('/book/*/author/text()')
print(author_span)

技巧

可以打开页面右键复制xpath，如果是相邻的可以复制然后微调

加载html文件

html

<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"/><title>Title</title>
</head>
<body>
<ul><li><a href="111">text111</a></li><li><a href="222">text222</a></li><li><a href="333">text333</a></li>
</ul>
</body>
</html>

from lxml import etree
tree = etree.parse('a.html')
# 获取所有a标签的值
all_a = tree.xpath("/html/body/ul/li/a/text()")
print(all_a)
# 获取第一个a标签值 这边序号从1开始
first_a = tree.xpath("/html/body/ul/li[1]/a/text()")
print(first_a)
# 获取href属性值为333的a标签的
a_href_333 = tree.xpath("/html/body/ul/li/a[@href='333']/text()")
print(a_href_333)
# 获取所有a标签的href属性值
all_a_href = tree.xpath("/html/body/ul/li/a/@href")
print(all_a_href)
# 先获取所有的li标签,然后挨个获取a标签的值
all_li = tree.xpath("/html/body/ul/li")
for li in all_li:print(li.xpath("./a/text()"))

爬取图片

from lxml import etree
import requests
url = 'https://wap.umei.cc/gaoxiaotupian/'
# 获取网页源码
res = requests.get(url)
res.encoding = 'utf-8'
page = etree.HTML(res.text)
lis = page.xpath('/html/body/div[5]/ul/li')
for li in lis:img_url = li.xpath('./div/a/img/@data-original')[0]fileName = img_url.split('/')[-1]with open('img/'+fileName, 'wb') as f:f.write(requests.get(img_url).content)print('下载图片：', fileName+'成功')
res.close()
print('下载完成')