[已解决]爬虫大神快来围观，菜鸟又遇到进阶问题了

L嘉

现在我只能爬到标题这些，但是我要爬取子页面的内容应该怎么写了呢？如下图，我要爬取子页面的栋数和户数，但是我这个写好的代码爬取没有反应怎么一回事啊


代码在最后

1. # -*- coding: utf-8 -*-
   
2. """
   
3. Created on Tue Aug  4 09:24:02 2020
   
4. 
   
5. @author: Administrator
   
6. """
   
7. 
   
8. from lxml import etree
   
9. import requests
   
10. import csv
    
11. from multiprocessing.dummy import Pool as pl    #导入线程池
    
12. 
    
13. def towrite(item):
    
14.     with open('balk.csv','a',encoding='utf-8') as csvfile:
    
15.         writer = csv.writer(csvfile)
    
16.         try:
    
17.             writer.writerow(item)
    
18.         except:
    
19.             print('write error!')
    
20.             
    
21.             
    
22. def spider(url):
    
23.     htm = requests.get(url, headers = headers)
    
24.     response=etree.HTML(htm.text)
    
25.    
    
26.     mingcheng = response.xpath('div[1]/div[1]/a/text()')[0]
    
27.    
    
28.     zaishou = response.xpath('div[2]/div[2]/a/span/text()')[0]
    
29.    
    
30.     junjia = response.xpath('div[2]/div[1]/div[1]/span/text()')[0]
    
31.    
    
32.     dongshu = response.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[5]/span[2]/text()')[0]
    
33.    
    
34.     hushu = response.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[6]/span[2]/text()')[0]
    
35.    
    
36.     xiaoqu_item = [mingcheng,zaishou,junjia,dongshu,hushu]
    
37.     towrite(xiaoqu_item)
    
38.     print('正在爬取小区：',mingcheng)
    
39.    
    
40.    
    
41.     if __name__ == '__main__':
    
42.        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.3'}
    
43.    
    
44.     start_url = 'https://cd.ke.com/xiaoqu/damian/pg'
    
45.     pool=pl(4)
    
46.     all_url = []
    
47.     for x in range(1,4):
    
48.         html = requests.get(start_url +str(x), headers=headers)
    
49.         slector = etree.HTML(html.text)
    
50.         xiaoqulist = slector.xpath('//*[@id="beike"]/div[1]/div[4]/div[1]/div[3]/ul/li')
    
51.         for xiaoqu in xiaoqulist:
    
52.             xiaoqu_url_houduan = xiaoqu.xpath('//*[@id="beike"]/div[1]/div[4]/div[1]/div[3]/ul/li[1]/div[1]/div[1]/a')[0]
    
53.             all_url.append(xiaoqu_url_houduan)
    
54.     pool.map(spider,all_url)
    
55.     pool.close()
    
56.     pool.join()

复制代码

最佳答案

月排行榜 / 总排行榜

Twilight6

2020-8-4 20:58:18

帮你改完了，应该能达到你的目的了：

1. # -*- coding: utf-8 -*-
   
2. """
   
3. Created on Tue Aug  4 09:24:02 2020
   
4. 
   
5. @author: Administrator
   
6. """
   
7. 
   
8. from lxml import etree
   
9. import requests
   
10. import csv
    
11. from multiprocessing.dummy import Pool as pl  # 导入线程池
    
12. 
    
13. 
    
14. def towrite(item):
    
15.     with open('balk.csv', 'a', encoding='utf-8') as csvfile:
    
16.         writer = csv.writer(csvfile)
    
17.         try:
    
18.             writer.writerow(item)
    
19.         except:
    
20.             print('write error!')
    
21. 
    
22. 
    
23. def spider(url):
    
24.     htm = requests.get(url, headers=headers)
    
25.     response = etree.HTML(htm.text)
    
26. 
    
27.     mingcheng = response.xpath('//div[@class="title"]/h1/text()')[0].strip()
    
28. 
    
29.     dongshu = response.xpath('//span[@class="xiaoquInfoContent"]/text()')[4]
    
30. 
    
31.     hushu = response.xpath('//span[@class="xiaoquInfoContent"]/text()')[5]
    
32. 
    
33.     for i in xiaoquname:
    
34.         if mingcheng in i[0]:
    
35.             idx = i[0].index(mingcheng)
    
36.             zaishou = i[1][idx][0]
    
37.             junjia = i[1][idx][1]
    
38.             break
    
39. 
    
40.     xiaoqu_item = [mingcheng, zaishou,junjia,dongshu, hushu]
    
41.     towrite(xiaoqu_item)
    
42.     print('正在爬取小区：', mingcheng)
    
43. 
    
44. 
    
45. if __name__ == '__main__':
    
46.     headers = {
    
47.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.3'}
    
48. 
    
49.     start_url = 'https://cd.ke.com/xiaoqu/damian/pg'
    
50.     pool = pl(4)
    
51.     all_url = []
    
52.     xiaoquname = []
    
53.     for x in range(1, 4):
    
54.         html = requests.get(start_url + str(x), headers=headers)
    
55.         slector = etree.HTML(html.text)
    
56.         xiaoqulist = slector.xpath('//div[@class="info"]/div[@class="title"]/a/@href')
    
57.         name = slector.xpath("//a[@class='maidian-detail']/text()")
    
58.         jiage = slector.xpath("//div[@class='totalPrice']/span/text()")
    
59.         zaishous = slector.xpath("//a[@class='totalSellCount']/span/text()")
    
60.         xiaoquname.append([name, list(zip(zaishous,jiage))])
    
61.         for xiaoqu in xiaoqulist:
    
62.             all_url.append(xiaoqu)
    
63.     pool.map(spider, all_url)
    
64.     pool.close()
    
65.     pool.join()

复制代码

跳转到最佳答案楼层

1q23w31

用代码编辑器传上来，复制的可能格式不对

liuzhengyuan

在 spider 函数里
先

1. global headers

复制代码

要把 headers 变成全局变量

L嘉

1q23w31 发表于 2020-8-4 10:53
用代码编辑器传上来，复制的可能格式不对

好的，你在帮忙看看

L嘉

liuzhengyuan 发表于 2020-8-4 11:05
在 spider 函数里
先
要把 headers 变成全局变量

还是没反应，空的

Twilight6

L嘉 发表于 2020-8-4 11:22
还是没反应，空的

你的 xpath 全部都是错误的呀，你要爬取哪些内容

zaishou 是什么

junjia 是什么

dongshu

hushu

L嘉

Twilight6 发表于 2020-8-4 12:20
你的 xpath 全部都是错误的呀，你要爬取哪些内容

zaishou 是什么

主要是子页面的爬取不会

Twilight6

L嘉 发表于 2020-8-4 13:03
主要是子页面的爬取不会

晚点帮你写吧，现在有点事哈~

L嘉

Twilight6 发表于 2020-8-4 14:10
晚点帮你写吧，现在有点事哈~

谢谢

johnnyb

用我写的规则.. 亲测 没有任何问题.

1. import requests
   
2. from lxml import etree
   
3. 
   
4. headers = {
   
5.     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'
   
6. }
   
7. url = 'https://cd.ke.com/xiaoqu/1611057941003/?fb_expo_id=343036232051900416'
   
8. r = requests.get(url=url, headers=headers, timeout=3)
   
9. 
   
10. tree = etree.HTML(r.text)
    
11. money = tree.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[2]/div/span[1]/text()')[0]
    
12. WY = tree.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[2]/span[2]/text()')[0].lstrip().rstrip()
    
13. WYGS = tree.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[3]/span[2]/text()')[0]
    
14. KFS = tree.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[4]/span[2]/text()')[0]
    
15. tag = tree.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[5]/span[2]/text()')[0]
    
16. tag2 = tree.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[6]/span[2]/text()')[0]
    
17. print('参考价格:{0}元/平'.format(money))
    
18. print('物业费:{0}'.format(WY))
    
19. print('物业公司:{0}'.format(WYGS))
    
20. print('开发商:{0}'.format(KFS))
    
21. print('楼栋总数:{0}'.format(tag))
    
22. print('房屋总数:{0}'.format(tag2))

复制代码

1. 
   
2. 参考价格:17235元/平
   
3. 物业费:1.29至5.2元/平米/月
   
4. 物业公司:成都安达祥和置业有限公司
   
5. 开发商:成都志达房地产开发有限公司
   
6. 楼栋总数:69栋
   
7. 房屋总数:5067户

复制代码

L嘉

johnnyb 发表于 2020-8-4 15:01
用我写的规则.. 亲测 没有任何问题.

那我要爬取每个小区的呢怎么写呢

1q23w31

L嘉 发表于 2020-8-4 17:16
那我要爬取每个小区的呢怎么写呢

1. # -*- coding: utf-8 -*-
   
2. """
   
3. Created on Tue Aug  4 09:24:02 2020
   
4. 
   
5. @author: Administrator
   
6. """
   
7. 
   
8. from lxml import etree
   
9. import requests
   
10. import csv
    
11. from multiprocessing.dummy import Pool as pl    #导入线程池
    
12. 
    
13. def towrite(item):
    
14.     with open('balk.csv','a') as csvfile:
    
15.         writer = csv.writer(csvfile)
    
16.         try:
    
17.             writer.writerow(item)
    
18.         except:
    
19.             print('write error!')
    
20.             
    
21.             
    
22. def spider(url):
    
23.     htm = requests.get(url, headers = headers)
    
24.     response=etree.HTML(htm.text)
    
25.    
    
26.     mingcheng = response.xpath('//*[@id="beike"]/div[1]/div[2]/div[2]/div/div/div[1]/h1/@title')
    
27.    
    
28.    
    
29.     junjia = response.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[2]/div/span[1]/text()')
    
30. 
    
31.     dongshu = response.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[5]/span[2]/text()')
    
32.    
    
33.     hushu = response.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[6]/span[2]/text()')
    
34.    
    
35.     xiaoqu_item = [mingcheng,junjia,dongshu,hushu]
    
36.     towrite(xiaoqu_item)
    
37.     print('正在爬取小区：',mingcheng)
    
38.    
    
39.    
    
40. if __name__ == '__main__':
    
41.    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.3'}
    
42. 
    
43. start_url = 'https://cd.ke.com/xiaoqu/damian/pg'
    
44. pool=pl(4)
    
45. all_url = []
    
46. zaishou = []
    
47. for x in range(1,4):
    
48.     html = requests.get(start_url +str(x), headers=headers)
    
49.     slector = etree.HTML(html.text)
    
50.     xiaoqulist = slector.xpath('//*[@id="beike"]/div[1]/div[4]/div[1]/div[3]/ul')
    
51. 
    
52. 
    
53.     for xiaoqu in xiaoqulist:
    
54.         xiaoqu_url_houduan = xiaoqu.xpath('//*[@id="beike"]/div[1]/div[4]/div[1]/div[3]/ul/li/a/@href')
    
55.         price = xiaoqu.xpath('//*[@id="beike"]/div[1]/div[4]/div[1]/div[3]/ul/li/div[2]/div[2]/a/span/text()')
    
56.         all_url.extend(xiaoqu_url_houduan)
    
57.         zaishou.extend(price)
    
58. 
    
59. pool.map(spider,all_url)
    
60. pool.close()
    
61. pool.join()
    

复制代码

不是很完美，除了在售，其他的都能写出

1q23w31

L嘉 发表于 2020-8-4 17:16
那我要爬取每个小区的呢怎么写呢

1. # -*- coding: utf-8 -*-
   
2. """
   
3. Created on Tue Aug  4 09:24:02 2020
   
4. 
   
5. @author: Administrator
   
6. """
   
7. 
   
8. from lxml import etree
   
9. import requests
   
10. import csv
    
11. from multiprocessing.dummy import Pool as pl #导入线程池
    
12. import re
    
13. 
    
14. def towrite(item):
    
15.     with open('balk.csv','a') as csvfile:
    
16.         writer = csv.writer(csvfile)
    
17.         try:
    
18.             writer.writerow(item)
    
19.         except:
    
20.             print('write error!')
    
21.             
    
22.             
    
23. def spider(url):
    
24.     htm = requests.get(url, headers = headers)
    
25.     response=etree.HTML(htm.text)
    
26.    
    
27.     mingcheng = response.xpath('//*[@id="beike"]/div[1]/div[2]/div[2]/div/div/div[1]/h1/@title')
    
28.    
    
29.     zaishou = response.xpath('/html/head/meta[9]/@content')
    
30.     zaishou = re.findall('在售二手房源.*?套',zaishou[0])
    
31. 
    
32.     junjia = response.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[2]/div/span[1]/text()')
    
33. 
    
34.     dongshu = response.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[5]/span[2]/text()')
    
35.    
    
36.     hushu = response.xpath('//*[@id="beike"]/div[1]/div[3]/div[1]/div[2]/div[3]/div[6]/span[2]/text()')
    
37.    
    
38.     xiaoqu_item = [mingcheng,zaishou,junjia,dongshu,hushu]
    
39.     towrite(xiaoqu_item)
    
40.     print('正在爬取小区：',mingcheng)
    
41.    
    
42.    
    
43. if __name__ == '__main__':
    
44.    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.3'}
    
45. 
    
46. start_url = 'https://cd.ke.com/xiaoqu/damian/pg'
    
47. pool=pl(4)
    
48. all_url = []
    
49. 
    
50. for x in range(1,4):
    
51.     html = requests.get(start_url +str(x), headers=headers)
    
52.     slector = etree.HTML(html.text)
    
53.     xiaoqulist = slector.xpath('//*[@id="beike"]/div[1]/div[4]/div[1]/div[3]/ul')
    
54. 
    
55. 
    
56.     for xiaoqu in xiaoqulist:
    
57.         xiaoqu_url_houduan = xiaoqu.xpath('//*[@id="beike"]/div[1]/div[4]/div[1]/div[3]/ul/li/a/@href')
    
58.         price = xiaoqu.xpath('//*[@id="beike"]/div[1]/div[4]/div[1]/div[3]/ul/li/div[2]/div[2]/a/span/text()')
    
59.         all_url.extend(xiaoqu_url_houduan)
    
60. 
    
61. 
    
62. pool.map(spider,all_url)
    
63. pool.close()
    
64. pool.join()

复制代码

                               
登录/注册后可看大图

完美解决

johnnyb

L嘉 发表于 2020-8-4 17:16
那我要爬取每个小区的呢怎么写呢

哥们你不是来做伸手党的. 要举一反三.  都给你写完了 你还能学到啥?   再说你都会用Pool了 xpath规则还不明白呢?  不会吧.

Twilight6

帮你改完了，应该能达到你的目的了：

1. # -*- coding: utf-8 -*-
   
2. """
   
3. Created on Tue Aug  4 09:24:02 2020
   
4. 
   
5. @author: Administrator
   
6. """
   
7. 
   
8. from lxml import etree
   
9. import requests
   
10. import csv
    
11. from multiprocessing.dummy import Pool as pl  # 导入线程池
    
12. 
    
13. 
    
14. def towrite(item):
    
15.     with open('balk.csv', 'a', encoding='utf-8') as csvfile:
    
16.         writer = csv.writer(csvfile)
    
17.         try:
    
18.             writer.writerow(item)
    
19.         except:
    
20.             print('write error!')
    
21. 
    
22. 
    
23. def spider(url):
    
24.     htm = requests.get(url, headers=headers)
    
25.     response = etree.HTML(htm.text)
    
26. 
    
27.     mingcheng = response.xpath('//div[@class="title"]/h1/text()')[0].strip()
    
28. 
    
29.     dongshu = response.xpath('//span[@class="xiaoquInfoContent"]/text()')[4]
    
30. 
    
31.     hushu = response.xpath('//span[@class="xiaoquInfoContent"]/text()')[5]
    
32. 
    
33.     for i in xiaoquname:
    
34.         if mingcheng in i[0]:
    
35.             idx = i[0].index(mingcheng)
    
36.             zaishou = i[1][idx][0]
    
37.             junjia = i[1][idx][1]
    
38.             break
    
39. 
    
40.     xiaoqu_item = [mingcheng, zaishou,junjia,dongshu, hushu]
    
41.     towrite(xiaoqu_item)
    
42.     print('正在爬取小区：', mingcheng)
    
43. 
    
44. 
    
45. if __name__ == '__main__':
    
46.     headers = {
    
47.         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.3'}
    
48. 
    
49.     start_url = 'https://cd.ke.com/xiaoqu/damian/pg'
    
50.     pool = pl(4)
    
51.     all_url = []
    
52.     xiaoquname = []
    
53.     for x in range(1, 4):
    
54.         html = requests.get(start_url + str(x), headers=headers)
    
55.         slector = etree.HTML(html.text)
    
56.         xiaoqulist = slector.xpath('//div[@class="info"]/div[@class="title"]/a/@href')
    
57.         name = slector.xpath("//a[@class='maidian-detail']/text()")
    
58.         jiage = slector.xpath("//div[@class='totalPrice']/span/text()")
    
59.         zaishous = slector.xpath("//a[@class='totalSellCount']/span/text()")
    
60.         xiaoquname.append([name, list(zip(zaishous,jiage))])
    
61.         for xiaoqu in xiaoqulist:
    
62.             all_url.append(xiaoqu)
    
63.     pool.map(spider, all_url)
    
64.     pool.close()
    
65.     pool.join()

复制代码