爬取百度图片里边姐姐的照片——坂井泉水
这是直接使用的，还有只爬取地址存入数据库mongdb和PhantomJS版本，后边因为配置文件太多就不放出来了。先上代码，再上成果！
rom selenium import webdriver
import time
import json
import re
import random
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from pyquery import PyQuery as pq
import os,os.path
import urllib.request
#配置模拟器
chrome_driver = r'C:\Users\Chysial\AppData\Local\Google\Chrome\Application\chromedriver.exe'
browser = webdriver.Chrome(chrome_driver)
wait = WebDriverWait(browser,10)
browser.get('https://www.baidu.com/')
#cookie模拟登陆，get一般运行一次就行，我因为调试后边把它注释了
def get_cookies(browser):
#cookies没有时限方法，一种方法直接在里面把expiry改成int，另一种就是输出时候改写然后写入文件不然就需要修改了
time.sleep(50)
with open('C:\\Users\\Chysial\\Desktop\\cookies.txt','w') as cookief:
cookief.write(json.dumps(browser.get_cookies()))
browser.close()
def open_chrome(browser):
browser.delete_all_cookies()
with open('C:\\Users\\Chysial\\Desktop\\cookies.txt') as cookief:
cookieslist = json.load(cookief)
for cookies in cookieslist:
browser.add_cookie(cookies)
#从主页面baidu一步一步爬到图片里去
def search():
try:
input = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR ,'#kw'))
)
submit = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR ,'#su'))
)
input.send_keys("百度图片")
submit.click()
wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, '#page > a.n'))
)
html = browser.page_source
doc = pq(html)
items = doc('#1 .t a')
links = []
for each in items.items():
links.append(each.attr('href'))
links
browser.get(links)
input = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR ,'#kw'))
)
input.send_keys("坂井泉水")
input.send_keys(Keys.ENTER)
wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, '.imgpage .imglist .imgitem'))
)
except TimeoutException:
print("youcuom")
return get_products()
#获取姐姐图片的地址
def get_products():
wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, '.imgpage .imglist .imgitem'))
)
html = browser.page_source
doc = pq(html)
items = doc('.imgpage .imglist .imgitem').items()
i = 0
product = {}
for each in items:
i += 1
product[ 'picture'+str(i)] = each.attr('data-objurl')
if i==3:
break
return product
#下载与保存
def download_picture(folder='坂井泉水姐姐'):
os.chdir("C:\\Users\\Chysial\\Desktop")
os.mkdir(folder)
os.chdir(folder)
each = search()
for i in each:
req = urllib.request.Request(each)
response = urllib.request.urlopen(req)
img = response.read()
with open(i+'.jpg','wb') as f:
f.write(img)
def main():
# 运行一次就行了，我这里就不重复了get_cookies(browser)
open_chrome(browser)
browser.refresh()
each = search()
download_picture(folder='坂井泉水姐姐')
if __name__ == '__main__':
main()
内存小，电脑 就先爬几个意思一下 {:5_109:}
页:
[1]