[已解决]股票信息爬虫

JeremyCheung

请问个各位大神这个代码为什么爬不到股票的信息？求求帮助！请多多指教！谢谢！


import requests
from fake_user_agent.main import user_agent
import re
import csv


def getHtml(url):
    r = requests.get(url, headers={
        'User-Agent': user_agent("chrome"),
    })
    r.encoding = r.apparent_encoding
    return r.text


stockUrl = 'http://quote.eastmoney.com/stocklist.html'
PATTERN_STOCK = "<li><a.*>(\w*)\((\d{6})\)</a></li>"
if __name__ == '__main__':
    html = getHtml(stockUrl)
    reslist = re.findall(PATTERN_STOCK, html)
    # 数据清洗：去掉非个股,个股以6（沪市）,0（深市）,3（创业板）开头
    datalist = reslist[:]
    for res in reslist:
        print(res)
        if not (str(res[1]).startswith('6') or str(res[1]).startswith('3') or str(res[1]).startswith('0')):
            datalist.remove(res)
    f = open('C:/Users/MC/Desktop/stock csv/stock.csv', 'w+', encoding='utf-8', newline="")
    writer = csv.writer(f)
    writer.writerow(('名称', '代码'))
    for data in datalist:
        writer.writerow((data[0], data[1]))
    f.close()


非原创的~~

最佳答案

月排行榜 / 总排行榜

isdkz

2022-4-27 06:21:59

股票信息是 js 动态渲染的，换个模块，使用 requests-html 之前先执行以下命令安装：

1. pip install requests-html -i https://mirrors.aliyun.com/pypi/simple

复制代码

对你的代码修改如下：

1. from requests_html import HTMLSession
   
2. import csv
   
3. 
   
4. 
   
5. def getHtml(url):
   
6.     sess = HTMLSession()
   
7.     r = sess.get(url)
   
8.     r.html.render()
   
9.     return r.html
   
10. 
    
11. 
    
12. stockUrl = 'http://quote.eastmoney.com/stocklist.html'
    
13. 
    
14. if __name__ == '__main__':
    
15.     html = getHtml(stockUrl)
    
16.    
    
17.     # 数据清洗：去掉非个股,个股以6（沪市）,0（深市）,3（创业板）开头
    
18.     datalist = html.find('#table_wrapper-table > tbody > tr')
    
19.     reslist = datalist[:]
    
20.     print(datalist)
    
21.     for res in reslist:
    
22.         temp = res.find('td:nth-child(2) > a')[0].text
    
23.         print(temp)
    
24.         if (not ((temp.startswith('6') or
    
25.             temp.startswith('3') or temp.startswith('0')))):
    
26.             datalist.remove(res)
    
27.     f = open('stock.csv', 'w+', encoding='utf-8-sig', newline="")
    
28.     writer = csv.writer(f)
    
29.     writer.writerow(('名称', '代码'))
    
30.     for data in datalist:
    
31.         temp0 = data.find('td.mywidth > a')[0].text
    
32.         temp1 = data.find('td:nth-child(2) > a')[0].text
    
33.         writer.writerow((temp0, '\t' + temp1))
    
34.     f.close()

复制代码

跳转到最佳答案楼层

isdkz

股票信息是 js 动态渲染的，换个模块，使用 requests-html 之前先执行以下命令安装：

1. pip install requests-html -i https://mirrors.aliyun.com/pypi/simple

复制代码

对你的代码修改如下：

1. from requests_html import HTMLSession
   
2. import csv
   
3. 
   
4. 
   
5. def getHtml(url):
   
6.     sess = HTMLSession()
   
7.     r = sess.get(url)
   
8.     r.html.render()
   
9.     return r.html
   
10. 
    
11. 
    
12. stockUrl = 'http://quote.eastmoney.com/stocklist.html'
    
13. 
    
14. if __name__ == '__main__':
    
15.     html = getHtml(stockUrl)
    
16.    
    
17.     # 数据清洗：去掉非个股,个股以6（沪市）,0（深市）,3（创业板）开头
    
18.     datalist = html.find('#table_wrapper-table > tbody > tr')
    
19.     reslist = datalist[:]
    
20.     print(datalist)
    
21.     for res in reslist:
    
22.         temp = res.find('td:nth-child(2) > a')[0].text
    
23.         print(temp)
    
24.         if (not ((temp.startswith('6') or
    
25.             temp.startswith('3') or temp.startswith('0')))):
    
26.             datalist.remove(res)
    
27.     f = open('stock.csv', 'w+', encoding='utf-8-sig', newline="")
    
28.     writer = csv.writer(f)
    
29.     writer.writerow(('名称', '代码'))
    
30.     for data in datalist:
    
31.         temp0 = data.find('td.mywidth > a')[0].text
    
32.         temp1 = data.find('td:nth-child(2) > a')[0].text
    
33.         writer.writerow((temp0, '\t' + temp1))
    
34.     f.close()

复制代码

hornwong