淘宝关键词页面爬取绘图进行数据分析

对爬虫、逆向感兴趣的同学可以查看文章，一对一小班V教学：https://blog.csdn.net/weixin_35770067/article/details/142514698

关键词页面爬取代码

from DrissionPage import WebPage, ChromiumOptions
from DataRecorder import Recorder
import time
import randompath = r'C:\Program Files\Google\Chrome\Application\chrome.exe'
ChromiumOptions().set_browser_path(path).save()def random_sleep():"""随机延时1-3秒"""time.sleep(random.uniform(3, 5))def main(url, save_name):r = Recorder('{}.xlsx'.format(save_name))r.add_data(('url', '标题', '店铺名', '销量数', '价格', '地区'))page = WebPage()#page.timeout = 5page = WebPage(timeout=5)  # 初始化时设置超时为5秒page.set.retry_times(3)page.set.retry_interval(1)scroll_step = 1000max_scroll_times = 2 #这个数字表示你要爬取多少页数据print(f"开始访问: {url}")page.get(url)captured_urls = set()current_scroll_times = 0while current_scroll_times < max_scroll_times:print("爬取第{}页数据".format(current_scroll_times + 1))page_ = input("请点击翻页按钮,点击完成请输入yes:")if page_ == "yes":try:random_sleep()# 滚动页面page.run_js(f'window.scrollTo(0, document.body.scrollHeight)')random_sleep()items = page.eles('css:.tbpc-col')print(f"本次找到 {len(items)} 个商品")for item in items:try:info = {'url': item('tag:a').link,'title': item('css:.title--qJ7Xg_90').text,'shop': item('css:.shopNameText--DmtlsDKm').text,'sales': item('css:.realSales--XZJiepmt').text,'price': item('css:.priceInt--yqqZMJ5a').text,'location': item('css:.procity--wlcT2xH9').text}if info['url'] not in captured_urls:captured_urls.add(info['url'])r.add_data((info['url'],info['title'],info.get('shop', '未知店铺'),info.get('sales', '0'),info.get('price', '价格未知'),info.get('location', '地区未知')))print(f"已添加商品: {info['title']}")except Exception as e:print(f"处理商品数据时出错: {e}")continuecurrent_scroll_times += 1print(f"已完成第 {current_scroll_times} 次滚动")# 每5次滚动保存一次if current_scroll_times % 5 == 0:r.record()print(f"已保存 {len(captured_urls)} 条数据")if len(items) == 0:print("未找到新商品，准备退出")breakexcept Exception as e:print(f"页面处理出错: {e}")continuer.record()print(f"总共抓取商品数量: {len(captured_urls)}")if __name__ == '__main__':# 淘宝关键词搜索页面url = 'https://s.taobao.com/search?q=%E6%89%8B%E6%9C%BA%E7%83%AD%E9%94%80%E6%A6%9C&tab=all'save_name = 'phone'try:main(url,save_name)except Exception as e:print(f"程序出错: {e}")finally:print("程序结束")

数据分析代码

import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
import warnings
warnings.filterwarnings('ignore')
# 设置中文字体，避免显示乱码
font = FontProperties(fname=r'C:\Windows\Fonts\SimHei.ttf')
plt.rcParams['axes.unicode_minus'] = False
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 10000)
pd.set_option('display.max_colwidth', 10000)
pd.set_option('display.max_rows', None)def extract_sales_number(text):"""从销量文本中提取数字，处理类似'3000+人付款'的格式"""if pd.isna(text):return 0# 移除'+'号和'人付款'字样，只保留数字number = str(text).replace('+', '').replace('人付款', '')try:return int(number)except:return 0def analyze_data():# 读取Excel文件，跳过前3行df = pd.read_excel('手机.xlsx', skiprows=3)# 设置列名df.columns = ['url', '标题', '店铺名', '销量数', '价格', '地区']# 数据清洗# 处理销量数据df['销量数'] = df['销量数'].apply(extract_sales_number)# 处理价格数据df['价格'] = pd.to_numeric(df['价格'], errors='coerce')# 移除无效数据df = df[df['价格'] > 0]df = df[df['销量数'] > 0]# 1. 价格与销量的关系（柱状图）plt.figure(figsize=(15, 8))price_bins = [0, 1000, 2000, 3000, 4000, 5000, float('inf')]price_labels = ['0-1000', '1000-2000', '2000-3000', '3000-4000', '4000-5000', '5000+']df['价格区间'] = pd.cut(df['价格'], bins=price_bins, labels=price_labels)price_sales = df.groupby('价格区间')['销量数'].mean()plt.bar(price_sales.index, price_sales.values, alpha=0.6, color='skyblue')plt.title('手机价格区间与平均销量关系图', fontproperties=font, fontsize=14)plt.xlabel('价格区间（元）', fontproperties=font)plt.ylabel('平均销量（件）', fontproperties=font)plt.xticks(rotation=45, fontproperties=font)plt.grid(True, linestyle='--', alpha=0.3)# 添加数值标签for i, v in enumerate(price_sales.values):plt.text(i, v, f'{float(v)}', ha='center', va='bottom', fontproperties=font)plt.tight_layout()plt.savefig('价格销量关系图.png')plt.close()# 2. 价格与地区的关系（折线图）region_price = df.groupby('地区')['价格'].mean().sort_values(ascending=False)plt.figure(figsize=(15, 8))plt.plot(region_price.index, region_price.values, marker='o', linewidth=2, markersize=8, color='orange')plt.title('各地区手机平均价格对比', fontproperties=font, fontsize=14)plt.xlabel('地区', fontproperties=font)plt.ylabel('平均价格（元）', fontproperties=font)plt.xticks(rotation=45, fontproperties=font)plt.grid(True, linestyle='--', alpha=0.7)# 添加数值标签for i, v in enumerate(region_price.values):plt.text(i, v, f'{int(v)}元', ha='center', va='bottom', fontproperties=font)plt.tight_layout()plt.savefig('地区价格关系图.png')plt.close()# 输出统计信息print("\n=== 数据统计信息 ===")print(f"\n总商品数量：{len(df)}个")print(f"\n价格统计：")price_stats = df['价格'].describe()print(f"平均价格：{price_stats['mean']:.2f}元")print(f"最高价格：{price_stats['max']:.2f}元")print(f"最低价格：{price_stats['min']:.2f}元")print(f"\n销量统计：")sales_stats = df['销量数'].describe()print(f"平均销量：{sales_stats['mean']:.2f}件")print(f"最高销量：{int(sales_stats['max'])}件")print(f"最低销量：{int(sales_stats['min'])}件")print(f"\n各地区平均价格：")for region, price in region_price.items():print(f"{region}: {price:.2f}元")print(f"\n各价格区间的平均销量：")for interval, sales in price_sales.items():print(f"{interval}: {float(sales)}件")if __name__ == '__main__':try:analyze_data()print("\n数据分析完成！图表已保存。")except Exception as e:print(f"数据分析过程中出错: {e}")import tracebackprint(traceback.format_exc())