昨天又去爬了下，我学习爬虫后，爬的第一个网站，知道是那个吗？

Stubborn

不要宣扬，放过头来看当初爬它的时候， 各种百度，各种难题，哈哈，回过头来，这种静态的已经是so easy 了，还能变着花样爬它。

1. from lxml import etree
   
2. from OpenSSL import SSL
   
3. import requests
   
4. import re
   
5. import time
   
6. import os
   
7. 
   
8. HEADERS = {
   
9.     'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
   
10.     'Referer': 'http://www.mzitu.com'
    
11. }
    
12. QUANTITY =220#图集页数  每页/24
    
13. 
    
14. FILE_PATH = None #下载路径，请设置绝对路径，默认以当前绝对路径做保存
    
15. 
    
16. 
    
17. 
    
18. def get_path(name,num=" "):
    
19.     '''
    
20.     当num为None时，用来查询目录是否存在
    
21.     :param name:
    
22.     :param num:
    
23.     :return:
    
24.     '''
    
25.     currrent_path = os.path.realpath(__file__)  # 文件绝对路径
    
26.     current_dir = os.path.split(currrent_path)[0]
    
27.     file_path = os.path.join(current_dir, name)
    
28.     if num==" ":
    
29.         if not os.path.exists(file_path):
    
30.             return False
    
31.     if FILE_PATH is None:
    
32.         if not os.path.exists(file_path):
    
33.             os.makedirs(file_path)
    
34.     else:
    
35.         file_path = FILE_PATH
    
36.     return os.path.join(file_path,num) + ".jpg"
    
37. 
    
38. def get_response(url):
    
39.     """返回URL响应"""
    
40.     time.sleep(2)
    
41.     return requests.get(url=url,headers=HEADERS)
    
42. 
    
43. def atlas():
    
44.     '''
    
45.     :yield:图集下载的地址
    
46.     '''
    
47.     for i in range(QUANTITY):
    
48.         url = "https://www.mzitu.com/page/{page}/".format(page=i)
    
49.         response = get_response(url=url).text
    
50.         imgurl_list = re.findall(r'<li><a href="(.*?)" target="_blank">',response)
    
51.         imgname_list = re.findall(r"alt='(.*?)' width=",response)
    
52.         for img_naem,img_url in zip(imgname_list,imgurl_list):
    
53.             item = {}
    
54.             item["name"] = img_naem
    
55.             print(img_naem)
    
56.             # if not get_path(name=img_naem):
    
57.             #     break
    
58.             item["img_url"] = img_url
    
59.             yield item
    
60. 
    
61. 
    
62. def get_download_url(item):
    
63.     """
    
64.     :param url:图片url
    
65.     :return:  图集下载地址
    
66.     """
    
67.     response = get_response(url=item["img_url"]).text
    
68.     etre = etree.HTML(response)
    
69.     num = etre.xpath("//div[@class='pagenavi']/a[5]/span/text()")
    
70.     for i in range(0,int(num[0])+1):
    
71.         item["download"] = item["img_url"]+ "/" + str(i)
    
72.         item["number"] = str(i)
    
73.         yield item
    
74. 
    
75. 
    
76. def download(item):
    
77.     headers = {
    
78.         'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
    
79.         'Referer': item["download"]
    
80.     }
    
81.     try:
    
82.         time.sleep(2)
    
83.         etre = etree.HTML(requests.get(url=item["download"],headers=headers).text)
    
84.         download_url = etre.xpath("//div[@class='main-image']/p/a/img/@src")[0]
    
85.         response = requests.get(url=download_url,headers=headers).content
    
86.         file_path = get_path(item["name"],item["number"])
    
87.         with open(file_path,"wb") as fp:
    
88.             fp.write(response)
    
89.     except SSL.SysCallError as e:
    
90.         print("当前出现错误%s"%e)
    
91.         print(item)
    
92. 
    
93. 
    
94. 
    
95. def product(c):
    
96.     c.send(None)
    
97.     for img in atlas():
    
98.         c.send(img)
    
99.     c.close()
    
100. 
     
101. 
     
102. def customer():
     
103.     data = ""
     
104.     while True:
     
105.         n = yield data
     
106.         if not n:
     
107.             return
     
108.         for each in get_download_url(item=n):
     
109.             download(each)
     
110. 
     
111. def main():
     
112.     func = customer()
     
113.     product(func)
     
114. 
     
115. 
     
116. if __name__ == '__main__':
     
117.     main()
     

复制代码

青梅

Traceback (most recent call last):
  File "C:/Users/Administrator/AppData/Local/Programs/Python/Python37-32/Save/1.py", line 1, in <module>
    from lxml import etree
ModuleNotFoundError: No module named 'lxml'
一脸蒙蔽

Stubborn

青梅 发表于 2019-6-1 23:54
Traceback (most recent call last):
  File "C:/Users/Administrator/AppData/Local/Programs/Python/Pyt ...

缺少lxml库，安装这个库，pip install lxml  

青梅

Stubborn 发表于 2019-6-2 00:58
缺少lxml库，安装这个库，pip install lxml

哦  晓得了

2164930278

看不懂，懵了

jermey1994

大佬进步神速

censing

进步神速，厉害

解技

Traceback (most recent call last):
  File "C:/Users/14028/Desktop/爱词霸.py", line 2, in <module>
    from OpenSSL import SSL
ModuleNotFoundError: No module named 'OpenSSL

Stubborn

解技 发表于 2019-11-6 19:05
Traceback (most recent call last):
  File "C:/Users/14028/Desktop/爱词霸.py", line 2, in
    from ...

pip install OpenSSl  少一个库

解技

Stubborn 发表于 2019-11-6 19:23
pip install OpenSSl  少一个库

ERROR: Could not find a version that satisfies the requirement OpenSSL (from versions: none)
ERROR: No matching distribution found for OpenSSL
不知道为什么安装不了

Stubborn

解技 发表于 2019-11-6 20:26
ERROR: Could not find a version that satisfies the requirement OpenSSL (from versions: none)
ERRO ...

那就不要了，在89行，异常捕捉，直接用except:   不指定具体错误。