【测试】python抓取url不同线程下测试耗时，结果写入EXCEL

waletor

1、抓取https://www.mzitu.com/mm /page/2-33页面下图片链接，返回URL
2、统计串联、单线程、多线程同条件抓取耗时，返回数组
3、结果写入EXCEL
4、多次测试求平均，最大值，80%分位值，横向对比，了解多线程差异

1. import re
   
2. import time
   
3. from multiprocessing import Pool  #多线程
   
4. from bs4 import BeautifulSoup
   
5. import requests #获取HTTP相应
   
6. import xlwt  #Excel写入
   
7. 
   
8. def url_open(url):
   
9.     name=url
   
10.     headers={
    
11.         'Referer': 'https://www.mzitu.com/',
    
12.         'User-Agent':'Mozilla /5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'
    
13.         }
    
14.     response=requests.get(url,headers=headers)
    
15.     html=response.text
    
16.     soup=BeautifulSoup(html,'html.parser')
    
17.     div=soup.find('div',attrs={'id':'pins'})
    
18.     soup1=str(div)
    
19.     p=r'<a href="([^"]+\d)"'
    
20.     url=re.findall(p,soup1)
    
21.     #URL去重，排序不变
    
22.     url_list = list(set(url))
    
23.     url_list.sort(key=url.index)
    
24.     #print("页面链接%s：共有%d个有效果链接"%(name,len(url_list)))
    
25.     return url_list
    
26. 
    
27. #获取每次测试的时间，返回数组
    
28. def getTime(i):
    
29.     timeData=[]
    
30.     urls = ['https://www.mzitu.com/mm/page/{}/'.format(str(i)) for i in range(2, 32)]
    
31.     start_0 = time.time()
    
32.     for url in urls:
    
33.         url_open(url)
    
34.     end_0 = time.time()
    
35.     time_0=end_0 - start_0
    
36.     print('串行爬虫耗时:',time_0)
    
37. 
    
38.     start_1 = time.time()
    
39.     pool = Pool(processes=1)
    
40.     pool.map(url_open,urls)
    
41.     end_1 = time.time()
    
42.     time_1=end_1 - start_1
    
43.     print('1进程爬虫耗时:',time_1)
    
44.    
    
45.     start_2 = time.time()
    
46.     pool = Pool(processes=2)
    
47.     pool.map(url_open,urls)
    
48.     end_2 = time.time()
    
49.     time_2=end_2 - start_2
    
50.     print('2进程爬虫耗时:',time_2)
    
51.    
    
52.     start_3 = time.time()
    
53.     pool = Pool(processes=3)
    
54.     pool.map(url_open,urls)
    
55.     end_3 = time.time()
    
56.     time_3=end_3 - start_3
    
57.     print('3进程爬虫耗时:',time_3)
    
58.    
    
59.     start_4 = time.time()
    
60.     pool = Pool(processes=4)
    
61.     pool.map(url_open,urls)
    
62.     end_4 = time.time()
    
63.     time_4=end_4 - start_4
    
64.     print('4进程爬虫耗时:',time_4)
    
65.     timeData=[i,time_0,time_1,time_2,time_3,time_4]
    
66.     return timeData
    
67. 
    
68. if __name__ == "__main__":
    
69.     #新建Excel
    
70.     workbook = xlwt.Workbook(encoding = 'ascii')
    
71.     worksheet=workbook.add_sheet('My Worksheet')
    
72.     workbook.save('Excel_Workbook.xls')
    
73.     for x in range(1,2):
    
74.         timeData=getTime(x)
    
75.         #数组for循环写入单元格
    
76.         for i in range(0,6):
    
77.             worksheet.write(x, i, label = timeData[i])
    
78.         print("已完成第%d次测试，结果写入EXCEL成功"%x)
    
79.         workbook.save('Excel_Workbook.xls')
    
80.    
    

复制代码

EXCEL读写
https://www.cnblogs.com/xuxaut-558/p/10166642.html
多进程爬虫
https://www.cnblogs.com/MrLJC/p/3715783.html
实用才是硬道理，先模仿，再进步