爬豆瓣TOP250的所有图片，分享

xunyu

爬虫爬豆瓣250的图片，新手分享！小甲鱼yyds

1. import requests
   
2. import bs4
   
3. import re
   
4. def open_url(url='return a text type'):
   
5.     # 使用代理
   
6.     #proxies = {'http':"176.31.154.12:80","https": "176.31.154.12:80"}
   
7.     headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'}  
   
8.     #res = requests.get(url,headers= headers,proxies = proxies)
   
9.     res = requests.get(url,headers = headers)
   
10.     return res.text
    
11. def download_img(a,page = '第几页',number = '第几张'):
    
12.     headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'}  
    
13.     # res = request.get(url,headers= headers,proxies = proxies)
    
14.     res = requests.get(a,headers = headers)
    
15.     name = 'D:/1/' +'img' +str(page)+'&'+ str(number)+'.jpg'
    
16.     if res.status_code == 200:
    
17.         open(name,'wb').write(res.content)
    
18. def find_depth(res):
    
19.     soup = bs4.BeautifulSoup(res.text, 'html.parser')
    
20.     depth = soup.find('span',class_='next').previous_sibling.previous_sibling.text
    
21.     return int(depth)
    
22. url = 'https://movie.douban.com/top250'
    
23. a = open_url(url)
    
24. b = bs4.BeautifulSoup(a,features = 'lxml')
    
25. count_page = []
    
26. url_all = []
    
27. imgs_all = []
    
28. #找到所有页面的网址
    
29. for i in b('a'):
    
30.     aaa = i.attrs['href']
    
31.     print(i.attrs['href'])
    
32.     aa = re.search(r'start',aaa)
    
33.     if aa != None:
    
34.         count_page.append(aaa)
    
35. for j in count_page:
    
36.     url_all.append(url+j)
    
37. #找到网址的所有图片地址
    
38. c = b('img')
    
39. for l in c:
    
40.     imgs_all.append(l.attrs['src'])
    
41.     print(url_all)
    
42. #准备开爬
    
43. page = 0
    
44. number = 0
    
45. for i in url_all:
    
46.     page += 1
    
47.     a = open_url(i)
    
48.     b = bs4.BeautifulSoup(a,features = 'lxml')
    
49.     c = b('img')
    
50.     imgs_all = []
    
51.     for l in c:
    
52.         imgs_all.append(l.attrs['src'])  
    
53.     for j in imgs_all:
    
54.         number += 1
    
55.         download_img(j,page,number)
    

复制代码

xunyu

D:/1.png

学习编程中的Ben

来看看