比葫芦画瓢写的电影信息爬取的代码并保存到数据库

python小小白哟

需要安装以下库:
pymongo  requests  re  pyquery urllib logging

1. import pymongo
   
2. import requests
   
3. import re
   
4. from pyquery import PyQuery as pq
   
5. from urllib.parse import urljoin
   
6. import logging
   
7. 
   
8. logging.basicConfig(level=logging.INFO,format='%(asctime)s - %(levelname)s:%(message)s')
   
9. 
   
10. BASE_URL = 'https://ssr1.scrape.center'
    
11. TOTAL_PAGE = 10
    
12. 
    
13. def scrape_page(url):
    
14.     logging.info('scraping %s...', url)
    
15.     try:
    
16.         response = requests.get(url)
    
17.         if response.status_code == 200:
    
18.             return response.text
    
19.         logging.error('get invalid status code %s while scraping %s', response.status_code, url)
    
20.     except requests.RequestException:
    
21.         logging.error('error occurred while scraping %s', url, exc_info=True)
    
22. 
    
23. def scrape_index(page):
    
24.     index_url = f'{BASE_URL}/page/{page}'
    
25.     return scrape_page(index_url)
    
26. 
    
27. def parse_index(html):
    
28.     doc = pq(html)
    
29.     links = doc('.el-card .name')
    
30.     for link in links.items():
    
31.         href = link.attr('href')
    
32.         detail_url = urljoin(BASE_URL, href)
    
33.         logging.info('get detail url %s', detail_url)
    
34.         yield detail_url
    
35. 
    
36. 
    
37. 
    
38. def scrape_detail(url):
    
39.     return scrape_page(url)
    
40. 
    
41. def parse_detail(html):
    
42.     doc = pq(html)
    
43.     cover = doc('img.cover').attr('src')
    
44.     name = doc('a > h2').text()
    
45.     categories = [item.text() for item in doc('.categories button span').items()]
    
46.     published_at = doc('.info:contains(上映)').text()
    
47.     published_at = re.search('(\d{4}-\d{2}-\d{2})', published_at).group(1) \
    
48.         if published_at and re.search('\d{4}-\d{2}-\d{2}', published_at) else None
    
49.     drama = doc('.drama p').text()
    
50.     score = doc('p.score').text()
    
51.     score = float(score) if score else None
    
52.     return {
    
53.         'cover': cover,
    
54.         'name': name,
    
55.         'categories': categories,
    
56.         'published_at': published_at,
    
57.         'drama': drama,
    
58.         'score': score
    
59. }
    
60. 
    
61. 
    
62. 
    
63. client = pymongo.MongoClient(host='localhost', port=27017)
    
64. db = client.movies
    
65. collection = db.movies_mes
    
66. 
    
67. def save_data(data):
    
68.     collection.update_one({
    
69.     'name': data.get('name')
    
70.     }, {
    
71.     '$set': data
    
72.     }, upsert=True)
    
73. 
    
74. def main():
    
75.     for page in range(1, TOTAL_PAGE + 1):
    
76.         index_html = scrape_index(page)
    
77.         detail_urls = parse_index(index_html)
    
78.         for detail_url in detail_urls:
    
79.             detail_html = scrape_detail(detail_url)
    
80.             data = parse_detail(detail_html)
    
81.             logging.info('get detail data %s', data)
    
82.             logging.info('saving data to mongodb')
    
83.             save_data(data)
    
84.             logging.info('data saved successfully')
    
85. 
    
86. if __name__ == '__main__':
    
87.     main()

复制代码

以下是成功保存到数据库的界面:

sfqxx

涛4091