爬取网页问题

fytfytf

1. import requests
   
2. import bs4
   
3. 
   
4. def open_url(url):
   
5.     headers = {"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36"}
   
6.     r=requests.get(url,headers=headers)
   
7.     return r
   
8. 
   
9. def find_data(soup):
   
10.     div=soup.find_all('div',class_='spread-module')
    
11.     for i in div:
    
12.        print(i.a['href'])
    
13.    
    
14. 
    
15. def main():
    
16.     game_url='https://www.bilibili.com/v/game/'
    
17.     res=open_url(game_url)
    
18.     soup=bs4.BeautifulSoup(res.text,'lxml')
    
19.     finded_data=find_data(soup)
    
20.    
    
21.     '''with open('bilibiliweb.txt','w',encoding=res.encoding) as f:
    
22.         f.write(res.text)'''
    
23. if __name__=='__main__':
    
24.     main()
    

复制代码

想把下图里的a标签的地址打印出来,


结果什么都没有

求解

Marcus-Liu

'''with open('bilibiliweb.txt','w',encoding=res.encoding) as f:
        f.write(res.text)'''
为什么要加三引号？

fytfytf

Marcus-Liu 发表于 2020-7-24 16:08
'''with open('bilibiliweb.txt','w',encoding=res.encoding) as f:
        f.write(res.text)'''
为什 ...

注释起来，待会用

nahongyan1997

你爬取的网页是经过 .js 渲染后生成的数据，推荐使用 selenium 模块渲染后在进行爬取。

nahongyan1997

我这有selenium 模块使用的例子，如有需要请参照：

1. 
   
2. 
   
3. import urllib.request
   
4. import re
   
5. from selenium import webdriver
   
6. from selenium.webdriver import ChromeOptions
   
7. def search(music=None,singer=None):
   
8.     if not music and not singer:
   
9.         raise ValueError('参数错误！！')
   
10.    
    
11.     if not music:
    
12.         name = singer
    
13.     else:
    
14.         name = music
    
15.    
    
16.       # 创建参数设置对象.
    
17.     CLOUD_CHROME_OPTION = ChromeOptions()
    
18.     CLOUD_CHROME_OPTION.binary_location = '.\\plug-in\\Google\\Chrome\\Application\\chrome.exe'
    
19. #    chrome_opt.add_argument('--headless')   # 无界面化.
    
20. #    chrome_opt.add_argument('--disable-gpu')    # 配合上面的无界面化.
    
21. 
    
22.     driver = webdriver.Chrome(options = CLOUD_CHROME_OPTION,executable_path=r'chromedriver.exe')
    
23.     driver.get('https://music.163.com/#/search/m/?s=' + name)
    
24.     driver.switch_to.frame('g_iframe')
    
25.     music_list = re.findall(r'<a href="/song\?id=(\d*)"><b title="(.*?)">',driver.page_source)
    
26.     singer_list = re.findall(r'<a href="/artist\?id=\d*">(.*?)</a>',driver.page_source)
    
27. #     driver.quit()
    
28.     music_num = len(music_list)
    
29.     download_list = list()
    
30.     if singer:
    
31.         for i in range(music_num):
    
32.             if singer in singer_list[i]:
    
33.                 download_list.append(music_list[i]+(singer_list[i],))
    
34.     else:
    
35.         for i in range(music_num):
    
36.             download_list.append(music_list[i]+(singer_list[i],))
    
37.             
    
38.     print(download_list)
    
39.         
    
40.     down_load_sign = input(r'开始下载吗?[Y\N]:')
    
41.     if down_load_sign == 'Y':
    
42.         for each_music in download_list:
    
43.             print(f'{each_music[1]}-{each_music[2]}.mp3正在下载！')
    
44.             urllib.request.urlretrieve(f'http://music.163.com/song/media/outer/url?id={each_music[0]}.mp3',f'C:\\CLOUD Artification Intelligence\\clouddata\\Sound\\{each_music[1]}-{each_music[2]}.mp3')
    
45.             print(f'{each_music[1]}-{each_music[2]}.mp3下载完成！')
    
46.         
    
47.             
    
48. 
    
49. if __name__ == '__main__':
    
50.     while True:
    
51. #         music_name = input('请输入歌曲名:')
    
52. #         singer_name = input('请输入歌手名:')
    
53.         music_name = '大鱼'
    
54.         singer_name = ''
    
55.         search(music_name,singer_name)
    

复制代码

xiaosi4081

1. import requests
   
2. import re
   
3. headers = {
   
4.     "user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36",
   
5.     "referer":"https://www.bilibili.com/v/game/"
   
6. }
   
7. url = "https://api.bilibili.com/x/web-interface/dynamic/region?callback=jqueryCallback_bili_4729731045039791&jsonp=jsonp&ps=10&rid=17&_=1595579753063"
   
8. data = {
   
9.     "callback":"jqueryCallback_bili_4729731045039791",
   
10.     "jsonp":"jsonp",
    
11.     "ps":"10",
    
12.     "rid":"17",
    
13.     "&_":"1595579753063"
    
14. }
    
15. res = requests.get(url,headers=headers,data=data).text
    

复制代码

之后自己去看看json模块

fytfytf

xiaosi4081 发表于 2020-7-24 16:50
之后自己去看看json模块

这个有点难啊，能解释一下原理吗