文章目录
- 1 实战一：爬取豆瓣Top250图书信息
- 目标分析
- 完整代码
- 2 实战二：批量下载图片
- 关键技巧
- 多线程加速下载
- 3 实战三：爬取表格数据并存入Excel
- 安装依赖
- 爬取天气历史数据
- 4 完整项目：知乎热榜数据采集
下面通过几个经典案例把前面的知识串联起来。包含三类实战项目：静态页面爬取、图片批量下载、表格数据提取与存储。每个案例都有完整代码，可以直接运行。
1 实战一：爬取豆瓣Top250图书信息
豆瓣图书是学习爬虫的经典案例，页面结构清晰，适合练习CSS选择器和数据存储。
目标分析
目标：爬取豆瓣Top250图书的书名、作者、评分、出版信息、评价人数，保存为CSV文件。
打开F12，分析页面结构：
- 每本书在
div.item容器中 - 书名在
span.title中 - 评分在
span.rating_num中 - 作者和出版信息在
p标签中 - 翻页链接在
a标签，URL参数为?start=0,25,50...
完整代码
importrequestsfrombs4importBeautifulSoupimportcsvimporttimeimportrandomdefget_headers():"""返回伪装请求头"""return{'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36','Accept-Language':'zh-CN,zh;q=0.9'}defparse_book_list(html):"""解析单页图书列表"""soup=BeautifulSoup(html,'lxml')books=[]foriteminsoup.select('div.item'):# 书名（有时会有副标题）title_tags=item.select('span.title')title=title_tags[0].text.strip()iftitle_tagselse'未知'# 评分rating_tag=item.select_one('span.rating_num')rating=rating_tag.text.strip()ifrating_tagelse'0'# 评价人数inq_tag=item.select_one('span.inq')inq=inq_tag.text.strip()ifinq_tagelse''# 作者/出版信息info_tag=item.select_one('div.bd p')info=info_tag.text.strip().replace('\n',' ')ifinfo_tagelse''# 排名rank_tag=item.select_one('em')rank=rank_tag.textifrank_tagelse''books.append({'rank':rank,'title':title,'rating':rating,'inq':inq,'info':info})returnbooksdefcrawl_douban_books():"""爬取豆瓣图书Top250"""all_books=[]base_url='https://book.douban.com/top250'forstartinrange(0,250,25):url=f'{base_url}?start={start}'print(f'爬取第{start//25+1}页:{url}')try:response=requests.get(url,headers=get_headers(),timeout=10)response.raise_for_status()books=parse_book_list(response.text)all_books.extend(books)print(f'本页提取{len(books)}本书')exceptExceptionase:print(f'爬取失败:{e}')# 随机延时 1-3 秒，避免被封time.sleep(random.uniform(1,3))returnall_booksdefsave_to_csv(books,filename='douban_books.csv'):"""保存为CSV文件"""ifnotbooks:print('没有数据可保存')returnwithopen(filename,'w',encoding='utf-8-sig',newline='')asf:fieldnames=['rank','title','rating','inq','info']writer=csv.DictWriter(f,fieldnames=fieldnames)writer.writeheader()writer.writerows(books)print(f'已保存{len(books)}条数据至{filename}')if__name__=='__main__':books=crawl_douban_books()save_to_csv(books)# 打印前5条预览forbookinbooks[:5]:print(f'[{book["rank"]}]{book["title"]}-{book["rating"]}分')2 实战二：批量下载图片
批量下载图片是爬虫的常见需求。这个案例演示如何下载图片并保存到本地。
关键技巧
图片是二进制数据，需要用response.content获取，而不是response.text。
importrequestsimportosimporttimeimportrandomfrompathlibimportPathfromurllib.parseimporturlparsedefdownload_image(url,save_path,headers=None):"""下载单张图片"""try:response=requests.get(url,headers=headers,timeout=15,stream=True)response.raise_for_status()# 检查是否是图片类型content_type=response.headers.get('Content-Type','')if'image'notincontent_type:print(f'不是图片类型:{content_type}')returnFalse# 从URL中获取文件名parsed_url=urlparse(url)filename=os.path.basename(parsed_url.path)ifnotfilenameor'.'notinfilename:filename=f'image_{int(time.time())}.jpg'# 保存图片full_path=os.path.join(save_path,filename)withopen(full_path,'wb')asf:forchunkinresponse.iter_content(chunk_size=8192):f.write(chunk)file_size=os.path.getsize(full_path)print(f'下载成功:{filename}({file_size/1024:.1f}KB)')returnTrueexceptExceptionase:print(f'下载失败{url}:{e}')returnFalsedefbatch_download_images(image_urls,save_dir='images'):"""批量下载图片"""# 创建保存目录Path(save_dir).mkdir(parents=True,exist_ok=True)headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64)','Referer':'https://example.com'# 防盗链处理}success_count=0fail_count=0fori,urlinenumerate(image_urls,1):print(f'[{i}/{len(image_urls)}] 下载:{url[:60]}...')ifdownload_image(url,save_dir,headers):success_count+=1else:fail_count+=1# 下载间隔time.sleep(random.uniform(0.5,1.5))print(f'\n下载完成: 成功{success_count}张，失败{fail_count}张')# 使用示例：先爬取图片URL列表，再批量下载defget_image_urls_from_page(page_url):"""从页面中提取图片URL"""headers={'User-Agent':'Mozilla/5.0 ...'}response=requests.get(page_url,headers=headers)soup=BeautifulSoup(response.text,'lxml')image_urls=[]forimginsoup.select('div.gallery img'):src=img.get('src')orimg.get('data-src','')ifsrcandsrc.startswith('http'):image_urls.append(src)returnimage_urls# 主流程# page_url = 'https://example.com/gallery'# urls = get_image_urls_from_page(page_url)# batch_download_images(urls, save_dir='downloaded_images')多线程加速下载
当图片数量多时，单线程速度太慢。可以用多线程并发下载。
fromconcurrent.futuresimportThreadPoolExecutor,as_completeddefbatch_download_with_threads(image_urls,save_dir='images',max_workers=5):"""多线程批量下载"""Path(save_dir).mkdir(parents=True,exist_ok=True)headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}defdownload_task(args):idx,url=argsreturndownload_image(url,save_dir,headers)tasks=list(enumerate(image_urls,1))success=0withThreadPoolExecutor(max_workers=max_workers)asexecutor:future_to_url={executor.submit(download_task,task):taskfortaskintasks}forfutureinas_completed(future_to_url):idx,url=future_to_url[future]try:iffuture.result():success+=1exceptExceptionase:print(f'任务异常:{e}')print(f'多线程下载完成:{success}/{len(image_urls)}')3 实战三：爬取表格数据并存入Excel
很多网站有表格形式的数据，比如排行榜、统计数据等。这个案例演示如何完整爬取这类数据。
安装依赖
pipinstallopenpyxl pandas爬取天气历史数据
importrequestsfrombs4importBeautifulSoupimportpandasaspdfromdatetimeimportdatetimedefcrawl_table_data(url,table_selector='table'):""" 通用表格数据爬取函数 """headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}response=requests.get(url,headers=headers,timeout=10)response.encoding='utf-8'soup=BeautifulSoup(response.text,'lxml')table=soup.select_one(table_selector)ifnottable:print(f'未找到表格:{table_selector}')returnNone# 提取表头headers_row=table.select('tr:first-child th')ifnotheaders_row:headers_row=table.select('thead tr th')columns=[th.text.strip()forthinheaders_row]# 提取数据行rows=[]fortrintable.select('tbody tr'):cells=tr.select('td')ifcells:row_data=[cell.text.strip()forcellincells]rows.append(row_data)ifnotcolumnsornotrows:print('表格数据为空')returnNone# 转换为DataFramedf=pd.DataFrame(rows,columns=columns)returndfdefsave_to_excel(df,filename='table_data.xlsx',sheet_name='数据'):"""保存到Excel，支持格式美化"""withpd.ExcelWriter(filename,engine='openpyxl')aswriter:df.to_excel(writer,sheet_name=sheet_name,index=False)# 获取worksheet对象进行格式调整ws=writer.sheets[sheet_name]# 自动调整列宽forcolumninws.columns:max_length=0col_letter=column[0].column_letterforcellincolumn:ifcell.value:max_length=max(max_length,len(str(cell.value)))ws.column_dimensions[col_letter].width=min(max_length+4,30)print(f'已保存到{filename}')# 多页表格数据爬取defcrawl_multi_page_table(base_url,total_pages,page_param='page'):"""多页表格数据爬取"""all_data=[]forpageinrange(1,total_pages+1):url=f'{base_url}?{page_param}={page}'print(f'爬取第{page}页...')df=crawl_table_data(url)ifdfisnotNone:df['来源页码']=page all_data.append(df)importtime time.sleep(1)ifall_data:result=pd.concat(all_data,ignore_index=True)print(f'共爬取{len(result)}条数据')returnresultreturnNone4 完整项目：知乎热榜数据采集
importrequestsimportjsonimportcsvfromdatetimeimportdatetimedefcrawl_zhihu_hot():""" 爬取知乎热榜（接口版本，比解析HTML更稳定） """# 知乎热榜接口api_url='https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total'headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36','Referer':'https://www.zhihu.com/hot','Accept':'application/json'}params={'limit':50,'desktop':'true'}response=requests.get(api_url,headers=headers,params=params,timeout=10)ifresponse.status_code!=200:print(f'请求失败:{response.status_code}')return[]data=response.json()hot_list=data.get('data',[])results=[]forrank,iteminenumerate(hot_list,1):target=item.get('target',{})results.append({'排名':rank,'标题':target.get('title',''),'热度':item.get('detail_text',''),'回答数':target.get('answer_count',0),'关注数':target.get('follower_count',0),'链接':f"https://www.zhihu.com/question/{target.get('id','')}",'采集时间':datetime.now().strftime('%Y-%m-%d %H:%M')})returnresultsdefsave_hot_list(data,filename=None):ifnotfilename:filename=f'zhihu_hot_{datetime.now().strftime("%Y%m%d_%H%M")}.csv'withopen(filename,'w',encoding='utf-8-sig',newline='')asf:writer=csv.DictWriter(f,fieldnames=data[0].keys())writer.writeheader()writer.writerows(data)print(f'已保存{len(data)}条热榜数据至{filename}')if__name__=='__main__':hot_data=crawl_zhihu_hot()ifhot_data:save_hot_list(hot_data)print(f'\n当前热榜前10:')foriteminhot_data[:10]:print(f"[{item['排名']}]{item['标题']}-{item['热度']}")以上三个案例覆盖了静态爬虫的核心场景。注意在实际使用时，需要添加适当的请求延时，遵守网站robots.txt规则，不要爬取隐私数据。下一章进入爬虫的进阶领域：动态网页爬取。
