爬虫问题
import refrom bs4 import BeautifulSoup
import urllib.request
url = 'https://movie.douban.com/top250?start='
findlink = re.compile(r"<a href='(.*?)'>")
def main():
Gteurl(url)
def Gteurl(url):
for i in range(0,10):
html = url + str(i*25)
response = askURL(html)
bs = BeautifulSoup(response,'html.parser')
for item in bs.find_all('div',class_ = 'item'):
data = []
item = str(item)
link = re.findall(findlink,item)#此处为什么会得到一堆的空列表？如果加上索引值则报错，IndexError: list index out of range
print(link)
def askURL(url):
head = {
'User-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36 Edg/100.0.1185.29'
}
req =urllib.request.Request(url = url,headers=head,)
response = urllib.request.urlopen(req)、
return response
if __name__ == '__main__':
main()
问题已在源码中注释 正则没匹配上，因为在 html 中 href 的值用的是双引号，你的正则中写单引号。
import re
from bs4 import BeautifulSoup
import urllib.request
url = 'https://movie.douban.com/top250?start='
findlink = re.compile(r'<a href="(.*?)">') # 改了这行
def main():
Gteurl(url)
def Gteurl(url):
for i in range(0,10):
html = url + str(i*25)
response = askURL(html)
bs = BeautifulSoup(response,'html.parser')
for item in bs.find_all('div',class_ = 'item'):
data = []
item = str(item)
link = re.findall(findlink,item)#此处为什么会得到一堆的空列表？如果加上索引值则报错，IndexError: list index out of range
print(link)
def askURL(url):
head = {
'User-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36 Edg/100.0.1185.29'
}
req =urllib.request.Request(url = url,headers=head,)
response = urllib.request.urlopen(req)
return response
if __name__ == '__main__':
main()
页:
[1]