求助！！！！！！ 为啥mysql 里没有数据？？？？不知道哪错了～～～

shiey

# 导入支持库
from urllib.request import urlopen, Request
from bs4 import BeautifulSoup
import re
import pymysql
import time

# 设置Request对象
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36"
}

# 数据库连接
connect = pymysql.Connect(
    host='localhost',
    port=3306,
    user='root',
    password='',
    db='test_spider',
    charset='utf8'
)
# 获取游标
cursor = connect.cursor()
# 执行插入语句
sql_in = '''insert into doubanmovie (name,director,actor,style,country,release_time,time,score) value (%s,%s,%s,%s,%s,%s,%s,%s)'''


def get_movie_url(url):  # 获取每个电影的链接
    html = requests.get(url=url, headers=headers)
    selector = etree.HTML(html.text)
    movie_hrefs = selector.xpath('//div[@class="hd"]/a/@href')
    for movie_href in movie_hrefs:
        get_movie_info(movie_href)


def get_movie_info(url):
    html = requests.get(url=url, headers=headers)  # <Response [200]>
    selector = etree.HTML(html.text)  # <Element html at 0x20892e10108>
    try:
        name = selector.xpath('//*[@id="content"]/h1/span[1]/text()')[0]
    except IndexError:
        name = ''
    try:
        director = selector.xpath('//*[@id="info"]/span[1]/span[2]/a/text()')[0]
    except IndexError:
        director = ''
    try:
        actors = selector.xpath('//*[@class="actor"]/span[2]')[0]
        actor = actors.xpath('string(.)')
        # actors = re.findall('<a href="/.*?/" rel="v:starring">(.*?)</a>',html.text,re.S)
    except IndexError:
        actor = ''
    try:
        style = re.findall('<span property="v:genre">(.*?)</span>', html.text, re.S)[0]
    except IndexError:
        style = ''
    try:
        country = re.findall('<span class="pl">制片国家/地区:</span> (.*?)<br/>', html.text, re.S)[0]
    except IndexError:
        country = ''
    try:
        release_time = re.findall('<span property="v:initialReleaseDate" content=.*?>(.*?)</span>', html.text, re.S)[0]
    except IndexError:
        release_time = ''
    try:
        time = re.findall('<span property="v:runtime" content=.*?>(.*?)</span>', html.text, re.S)[0]
    except IndexError:
        time = ''
    try:
        score = selector.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/strong/text()')[0]
    except IndexError:
        score = ''
    cursor.execute(
        sql_in,
        [str(name), str(director), str(actor), str(style), str(country), str(release_time), str(time), str(score)]
    )

    urls = ['https://movie.douban.com/top250?start={}&filter='.format(i) for i in range(0, 250, 25)]

    for url in urls:
        get_movie_url(url)
        time.sleep(5)
        print('我好了！')
    connect.commit()

wp231957

在写入数据库之前先print一下你的数据
盲猜你根本就没取到数据

rsj0315

print大法看看拿没拿到啊，

shiey

有老铁帮忙实现一下吗  py小白

shiey

wp231957 发表于 2022-4-14 06:44
在写入数据库之前先print一下你的数据
盲猜你根本就没取到数据

py小白 难受

wp231957

shiey 发表于 2022-4-14 19:33
py小白  难受

不知道你的代码是打哪来的，要知道单单是数据库就够写一本书的了
你这爬虫也不知会否，数据库也不知会否
就弄出这么复杂的代码，狠是佩服你

isdkz

1. 
   
2. # 导入支持库
   
3. from urllib.request import urlopen, Request
   
4. from bs4 import BeautifulSoup
   
5. from lxml import etree                             # 需要从 lxml 模块 导入 etree
   
6. import re
   
7. import requests                           # 需要导入 requests 模块
   
8. import pymysql
   
9. import time
   
10. 
    
11. # 设置Request对象
    
12. headers = {
    
13.     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36"
    
14. }
    
15. 
    
16. # 数据库连接
    
17. connect = pymysql.Connect(
    
18.     host='localhost',
    
19.     port=3306,
    
20.     user='root',
    
21.     password='secureworks',
    
22.     db='test_spider',
    
23.     charset='utf8'
    
24. )
    
25. # 获取游标
    
26. cursor = connect.cursor()
    
27. # 执行插入语句
    
28. sql_in = '''insert into doubanmovie (name,director,actor,style,country,release_time,time,score) value (%s,%s,%s,%s,%s,%s,%s,%s)'''
    
29. 
    
30. 
    
31. def get_movie_url(url):  # 获取每个电影的链接
    
32.     html = requests.get(url=url, headers=headers)
    
33.     selector = etree.HTML(html.text)
    
34.     movie_hrefs = selector.xpath('//div[@class="hd"]/a/@href')
    
35.     for movie_href in movie_hrefs:
    
36.         get_movie_info(movie_href)
    
37. 
    
38. 
    
39. def get_movie_info(url):
    
40.     html = requests.get(url=url, headers=headers)  # <Response [200]>
    
41.     selector = etree.HTML(html.text)  # <Element html at 0x20892e10108>
    
42.     try:
    
43.         name = selector.xpath('//*[@id="content"]/h1/span[1]/text()')[0]
    
44.     except IndexError:
    
45.         name = ''
    
46.     try:
    
47.         director = selector.xpath('//*[@id="info"]/span[1]/span[2]/a/text()')[0]
    
48.     except IndexError:
    
49.         director = ''
    
50.     try:
    
51.         actors = selector.xpath('//*[@class="actor"]/span[2]')[0]
    
52.         actor = actors.xpath('string(.)')
    
53.         # actors = re.findall('<a href="/.*?/" rel="v:starring">(.*?)</a>',html.text,re.S)
    
54.     except IndexError:
    
55.         actor = ''
    
56.     try:
    
57.         style = re.findall('<span property="v:genre">(.*?)</span>', html.text, re.S)[0]
    
58.     except IndexError:
    
59.         style = ''
    
60.     try:
    
61.         country = re.findall('<span class="pl">制片国家/地区:</span> (.*?)<br/>', html.text, re.S)[0]
    
62.     except IndexError:
    
63.         country = ''
    
64.     try:
    
65.         release_time = re.findall('<span property="v:initialReleaseDate" content=.*?>(.*?)</span>', html.text, re.S)[0]
    
66.     except IndexError:
    
67.         release_time = ''
    
68.     try:
    
69.         time = re.findall('<span property="v:runtime" content=.*?>(.*?)</span>', html.text, re.S)[0]
    
70.     except IndexError:
    
71.         time = ''
    
72.     try:
    
73.         score = selector.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/strong/text()')[0]
    
74.     except IndexError:
    
75.         score = ''
    
76.     cursor.execute(
    
77.         sql_in,
    
78.         [str(name), str(director), str(actor), str(style), str(country), str(release_time), str(time), str(score)]
    
79.     )
    
80. 
    
81. '''这一部分代码缩进错误'''
    
82. urls = ['https://movie.douban.com/top250?start={}&filter='.format(i) for i in range(0, 250, 25)]
    
83. 
    
84. for url in urls:
    
85.     get_movie_url(url)
    
86.     time.sleep(5)
    
87.     print('我好了！')
    
88. connect.commit()
    
89. '''这一部分代码缩进错误'''

复制代码