[已解决]我这个代码为什么没反应啊

故里

import json
import requests
from requests.exceptions import RequestException
import re
import time

def get_one_page(url):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36'
        }
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.content.decode('ANSI')
        return None
    except RequestException:
        return None

url = 'http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/left.htm'

html = get_one_page(url)
from bs4 import BeautifulSoup

soup = BeautifulSoup(html,'lxml')

soup.prettify()
data_name_list = []
data_xls_list = []
pre_url = 'http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/'
for ul in soup.find_all('ul'):
    for li in ul.find_all(name='li'):
        a = li.a
        #数据格式为xls，去掉其他的
        if a != None and a.attrs['href'][-1] != 'm':
            data_name_list.append(li.get_text())
            data_xls_list.append(pre_url + a.attrs['href'])

import urllib
import os
path = 'C:\\Users\\'
i = 0
for url in data_xls_list:
    print(url)
    filename = os.path.join(path, data_name_list[i] + '.xls')
    urllib.request.urlretrieve(url, filename)
    i += 1

最佳答案

月排行榜 / 总排行榜

大马强

2021-10-21 18:54:09

1. def get_one_page(url):
   
2.     try:
   
3.         headers = {
   
4.             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36'
   
5.         }
   
6.         response = requests.get(url, headers=headers)
   
7.         if response.status_code == 200:
   
8.             return response.content.decode('ANSI')
   
9.         return None
   
10.     except RequestException:
    
11.         return None
    
12. 
    
13. 
    
14. url = 'http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/left.htm'
    
15. 
    
16. html = get_one_page(url)
    
17. 
    
18. soup = BeautifulSoup(html, 'lxml')
    
19. 
    
20. soup.prettify()
    
21. data_name_list = []
    
22. data_xls_list = []
    
23. pre_url = 'http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/'
    
24. count = 0
    
25. for ul in soup.find_all('ul'):
    
26.     # print(type(ul))
    
27.     for li in ul.find_all(name='li'):
    
28. 
    
29.         try:
    
30.             a = li.a['href']  # 获取href str
    
31.             # print(li.a['href'])
    
32.             # print(li.a['href'][-1])
    
33.             # print(a, li.get_text())
    
34.         except:
    
35.             pass
    
36.         if li.a != None and a[-1] == 'm':
    
37.             data_name_list.append(li.get_text())
    
38.             data_xls_list.append(pre_url + a)
    
39.     break
    
40. path = 'C:\\Users\\'
    
41. i = 0
    
42. for url in data_xls_list:
    
43.     print(url)
    
44.     filename = os.path.join(path, data_name_list[i] + '.xls')
    
45.     urllib.request.urlretrieve(url, filename)
    
46.     i += 1

复制代码

跳转到最佳答案楼层

大马强

1. if a != None and a.attrs['href'][-1] != 'm':

复制代码

你这是想爬啥？

大马强

大马强 发表于 2021-10-21 18:43
你这是想爬啥？

你这个判断导致后面的一个都拿不到

大马强

1. def get_one_page(url):
   
2.     try:
   
3.         headers = {
   
4.             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36'
   
5.         }
   
6.         response = requests.get(url, headers=headers)
   
7.         if response.status_code == 200:
   
8.             return response.content.decode('ANSI')
   
9.         return None
   
10.     except RequestException:
    
11.         return None
    
12. 
    
13. 
    
14. url = 'http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/left.htm'
    
15. 
    
16. html = get_one_page(url)
    
17. 
    
18. soup = BeautifulSoup(html, 'lxml')
    
19. 
    
20. soup.prettify()
    
21. data_name_list = []
    
22. data_xls_list = []
    
23. pre_url = 'http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/'
    
24. count = 0
    
25. for ul in soup.find_all('ul'):
    
26.     # print(type(ul))
    
27.     for li in ul.find_all(name='li'):
    
28. 
    
29.         try:
    
30.             a = li.a['href']  # 获取href str
    
31.             # print(li.a['href'])
    
32.             # print(li.a['href'][-1])
    
33.             # print(a, li.get_text())
    
34.         except:
    
35.             pass
    
36.         if li.a != None and a[-1] == 'm':
    
37.             data_name_list.append(li.get_text())
    
38.             data_xls_list.append(pre_url + a)
    
39.     break
    
40. path = 'C:\\Users\\'
    
41. i = 0
    
42. for url in data_xls_list:
    
43.     print(url)
    
44.     filename = os.path.join(path, data_name_list[i] + '.xls')
    
45.     urllib.request.urlretrieve(url, filename)
    
46.     i += 1

复制代码