在ooxx那一讲中的程序，

bingxue

import urllib.request
import os

def url_open(url):
    req=urllib.request.Request(url)
    req.add_header('User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0')
    response=urllib.request.urlopen(url)
    html=response.read()

    return html
def get_page(url):
    html=url_open(url).decode('utf-8')

    a=html.find('comment-report-c')+23
    b=html.find (']',a)

    return html[a:b]
def find_imgs(url):
    html=url_open(url).decode('utf-8')
    img_addrs=[]
    a=html.find('img src=')
   
    while a!=-1:
      b=html.find('.jpg',a,a+255)
      if b!=-1:
         img_addrs.append(html[a+9:b+4])
      else:
          b=a+9
      a=html.find('img src=',b)

    return img_addrs
      
def save_imgs(folder,img_addrs):
    for each in img_addrs:
        filename=each.split('/')[-1]
        with open(filename,'wb') as f:
            img=url_open()
            f.write(img)

def download_mm(folder='XXOO',pages=20):
    os.mkdir(folder)
    os.chdir(folder)

    url="http://jandan.net/ooxx "
    page_num=int(get_page(url))

    for i in range(pages):
        page_num -=i
        page_url=url+'page-'+str(page_num)+'#comments'
        img_addrs=find_imgs(page_url)
        save_imgs(img_addrs)

if __name__=='__main__':
    download_mm()

程序是这样的，但是一运行就
Traceback (most recent call last):
  File "C:\Users\Shizhe\Documents\download_mm.py", line 54, in <module>
    download_mm()
  File "C:\Users\Shizhe\Documents\download_mm.py", line 45, in download_mm
    page_num=int(get_page(url))
  File "C:\Users\Shizhe\Documents\download_mm.py", line 12, in get_page
    html=url_open(url).decode('utf-8')
  File "C:\Users\Shizhe\Documents\download_mm.py", line 6, in url_open
    req.add_header('User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0')
TypeError: add_header() missing 1 required positional argument: 'val'

我想请问一下，这是怎么回事

故梦L

这个网站的图片链接被加密了，爬不了

凌九霄

add_header有两个字符串参数。
req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0')

shangrilas

视屏教程不再更新了么？

awesometime

应该是这样吧，我的意见，不一定对
req.add_header('User-Agent‘， ’Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0')

秋木叶

selenium 模拟浏览器 了解一下

秋木叶

1. import os
   
2. import requests
   
3. import time
   
4. import re
   
5. from random import randint
   
6. import unittest
   
7. from selenium import webdriver
   
8. from selenium.webdriver.common.keys import Keys
   
9. 
   
10. 
    
11. 
    
12. def get_dir():
    
13.      '''判断文件夹是否存在，如果不存在就创建一个'''
    
14.      filename = "picture"
    
15.      if not os.path.isdir(filename):
    
16.          os.makedirs(filename)
    
17.      return filename
    
18. 
    
19. class ooxx_spider(unittest.TestCase):
    
20.     def setUp(self):
    
21.         chrome_options = webdriver.ChromeOptions()
    
22.         chrome_options.add_argument('--headless')
    
23.         chrome_options.add_argument('--disable-gpu')
    
24.         self.driver = webdriver.Chrome('C:\python3.6\chromedriver.exe', chrome_options=chrome_options)
    
25. 
    
26.     def test_spider(self):
    
27.         for i in range(1,63):
    
28.             #url = 'http://jandan.net/ooxx/' + 'page-' + str(i) + "#comments"
    
29.             url = 'http://jandan.net/pic/' + 'page-' + str(i) + "#comments"
    
30.             self.driver.get(url)
    
31.             print(url)
    
32.             elem = self.driver.find_elements_by_xpath('//*[@class="commentlist"]/li/div/div/div/p/a')#/li/div/div/div/p/img
    
33.             
    
34.             for j in elem:
    
35.                 print(j)
    
36.                 self.save_img(j.get_attribute('href'))
    
37.                
    
38.             print('第{}页爬取成功'.format(i))
    
39. 
    
40.     def save_img(self, res):
    
41.         suffix = res.split('.')[-1]
    
42.         if "gif" in suffix:
    
43.              destination = 'picture/' + str(randint(1, 1000)) + str(randint(1, 1000)) + '.'+ suffix
    
44.              r = requests.get(res)
    
45.              time.sleep(2)
    
46.              with open(destination, 'wb') as f:
    
47.                  f.write(r.content)
    
48. 
    
49.     def tearDown(self):
    
50.         self.driver.close()
    
51. 
    
52. if __name__ == '__main__':
    
53.     get_dir()
    
54.     unittest.main()
    

复制代码