改【爬百度图片】坂井泉水

Chysial

百度图片的是下拉出来后续的item，一般只能爬取前20个图片，昨天的写错了，改了下

1. from selenium import webdriver
   
2. import time
   
3. import json
   
4. import re
   
5. import random
   
6. from selenium.webdriver.common.by import By
   
7. from selenium.webdriver.common.keys import Keys
   
8. from selenium.webdriver.support import expected_conditions as EC
   
9. from selenium.webdriver.support.wait import WebDriverWait
   
10. from selenium.common.exceptions import TimeoutException, NoSuchElementException
    
11. from pyquery import PyQuery as pq
    
12. import os,os.path
    
13. import urllib.request
    
14. 
    
15. #配置模拟器
    
16. chrome_driver = r'C:\Users\Chysial\AppData\Local\Google\Chrome\Application\chromedriver.exe'
    
17. browser = webdriver.Chrome(chrome_driver)
    
18. wait = WebDriverWait(browser,10)
    
19. browser.get('https://www.baidu.com/')
    
20. 
    
21. #cookie模拟登陆，get一般运行一次就行，我因为调试后边把它注释了
    
22. def get_cookies(browser):
    
23.     #cookies没有时限方法，一种方法直接在里面把expiry改成int，另一种就是输出时候改写然后写入文件不然就需要修改了
    
24.     time.sleep(50)
    
25.     with open('C:\\Users\\Chysial\\Desktop\\cookies.txt','w') as cookief:
    
26.         cookief.write(json.dumps(browser.get_cookies()))
    
27.     browser.close()
    
28. 
    
29. def open_chrome(browser):
    
30.     browser.delete_all_cookies()
    
31.     with open('C:\\Users\\Chysial\\Desktop\\cookies.txt') as cookief:
    
32.         cookieslist = json.load(cookief)
    
33.         for cookies in cookieslist:
    
34.             browser.add_cookie(cookies)
    
35. #从主页面baidu一步一步爬到图片里去
    
36. def search():
    
37.     try:
    
38.         input = wait.until(
    
39.             EC.presence_of_element_located((By.CSS_SELECTOR ,'#kw'))
    
40.         )
    
41.         submit = wait.until(
    
42.             EC.presence_of_element_located((By.CSS_SELECTOR ,'#su'))
    
43.         )
    
44.         input.send_keys("百度图片")
    
45.         submit.click()
    
46.         wait.until(
    
47.             EC.presence_of_element_located((By.CSS_SELECTOR, '#page > a.n'))
    
48.         )
    
49.         html = browser.page_source
    
50.         doc = pq(html)
    
51.         items = doc('#1 .t a')
    
52.         links = []
    
53.         for each in items.items():
    
54.             links.append(each.attr('href'))
    
55.         links[0]
    
56.         browser.get(links[0])
    
57.         input = wait.until(
    
58.             EC.presence_of_element_located((By.CSS_SELECTOR ,'#kw'))
    
59.         )
    
60.         input.send_keys("坂井泉水")
    
61.         input.send_keys(Keys.ENTER)
    
62.         wait.until(
    
63.             EC.presence_of_element_located((By.CSS_SELECTOR, '.imgpage .imglist .imgitem'))
    
64.         )
    
65. 
    
66.         scorll_num(3)
    
67.         wait.until(
    
68.             EC.presence_of_element_located((By.CSS_SELECTOR, '.imgpage .imglist .imgitem'))
    
69.         )
    
70.         result = get_products()
    
71.         return result
    
72.     except TimeoutException:
    
73.         browser.refresh()
    
74.    
    
75. #获取姐姐图片的地址
    
76. def get_products():
    
77.         html = browser.page_source
    
78.         doc = pq(html)
    
79.         items = doc('.imgpage .imglist .imgitem').items()
    
80.         i = 0
    
81.         product = {}
    
82.         for each in items:
    
83.             i += 1
    
84.             product[ 'picture'+str(i)] = each.attr('data-objurl')
    
85.         return product
    
86.    
    
87. def scorll_num(num = 10):
    
88.     for i in range(num):
    
89.         target = browser.find_element_by_id("pageMoreWrap")
    
90.         browser.execute_script("arguments[0].scrollIntoView();", target) #拖动到可见的元素去
    
91.         wait.until(
    
92.             EC.presence_of_element_located((By.CSS_SELECTOR, '#pageMoreWrap'))
    
93.             )
    
94. #保存
    
95. def save_class(name,url):
    
96.     req = urllib.request.Request(url)
    
97.     response = urllib.request.urlopen(req)
    
98.     img = response.read()
    
99.     if '.jpg' in url:
    
100.         with open(name+'.jpg','wb') as f:
     
101.                 f.write(img)
     
102.     elif '.jpeg' in url:
     
103.         with open(name+'.jpeg','wb') as f:
     
104.                 f.write(img)
     
105.     elif '.png' in url:
     
106.         with open(name+'.png','wb') as f:
     
107.                 f.write(img)           
     
108. 
     
109. #下载
     
110. def download_picture(folder='坂井泉水'):
     
111.     os.chdir("C:\\Users\\Chysial\\Desktop")
     
112.     os.mkdir(folder)
     
113.     os.chdir(folder)
     
114.     each = search()
     
115.     for i in each:
     
116.         try:
     
117.             save_class(i,each[i])
     
118.         except:
     
119.             continue
     
120.    
     
121.    
     
122. def main():
     
123.     # 运行一次就行了，我这里就不重复了get_cookies(browser)
     
124.     open_chrome(browser)
     
125.     browser.refresh()
     
126.     download_picture(folder='我的泉水姐姐')
     
127. 
     
128.    
     
129. if __name__ == '__main__':
     
130.     main()
     

复制代码