求救，爬取这个网站新闻和标题时候爬不到内容，只能爬到导航栏

python0998

from urllib.request import urlopen
from bs4 import BeautifulSoup
import pandas as pd

def get_response(url,tag):
    response=urlopen(url)
    soup=BeautifulSoup(response.read(),"lxml")
    result=soup.find_all(tag)
    data=list()
    for a in result:
        if a:
            if a.text and len(a.text)>5: #过滤导航栏
                row=dict()
                row['title']=a.text
                row['href']=a.get('href')
                data.append(row)
    return data

url='http://jnxxq.jinan.gov.cn/col/col39742/index.html'
tag='a'

data=get_response(url,tag)
print(data)


解决加微信一盒烟钱红包

suchocolate

页面把新闻链接和标题信息放到了<datastore>，用re可以取出来。具体存放位置看一下生成的r.txt。

1. from urllib import request
   
2. import re
   
3. 
   
4. 
   
5. def main():
   
6.     headers = {'User-Agent': 'Firefox'}
   
7.     url = 'http://jnxxq.jinan.gov.cn/col/col39742/index.html'
   
8.     req = request.Request(url, headers=headers)
   
9.     r = request.urlopen(req)
   
10.     txt = r.read().decode('utf-8')
    
11.     with open('r.txt', 'w', encoding='utf-8') as f:
    
12.         f.write(txt)
    
13.     result = re.findall(r'</span><a (.*?)\'target', txt)
    
14.     for i in result:
    
15.         print(i)
    
16. 
    
17. 
    
18. if __name__ == "__main__":
    
19.     main()
    

复制代码