[已解决]多线程爬表情包爬不到内容

xmpython

大哥哥大姐姐能不能帮我看看我这程序哪里有问题啊，为啥运行几秒之后就自动结束，而且什么内容都没爬到
不用多线程的时候能爬图片，很正常，加上多线程之后就不行了，点运行之后啥反应都没有，过几秒自动结束程序，而且啥东西都没爬到。。。。要哭了

1. import requests
   
2. from lxml import etree
   
3. from fake_useragent import UserAgent
   
4. from urllib import request
   
5. import threading
   
6. from queue import Queue
   
7. 
   
8. 
   
9. class Producer(threading.Thread):
   
10.     headers = {
    
11.         "User-Agent": UserAgent().chrome
    
12.     }
    
13. 
    
14.     def __init__(self, page_queue, img_queue, *args, **kwargs):
    
15.         super(Producer, self).__init__(*args, **kwargs)
    
16.         self.page_queue = page_queue
    
17.         self.img_queue = img_queue
    
18. 
    
19.     def run(self):
    
20.         while True:
    
21.             if self.page_queue.empty():
    
22.                 break
    
23.             url = self.page_queue.get()
    
24.             self.get_url(url)
    
25. 
    
26.     def get_url(self, url):
    
27.         response = requests.get(url, headers=self.headers)
    
28.         e = etree.HTML(response.text)
    
29.         img_urls = e.xpath("//div[@class='page-content text-center']/div/a/img/@data-original")
    
30.         for img in img_urls:
    
31.             self.img_queue.put(img)
    
32. 
    
33. 
    
34. class Consumer(threading.Thread):
    
35.     def __init__(self, page_queue, img_queue, *args, **kwargs):
    
36.         super(Consumer, self).__init__(*args, **kwargs)
    
37.         self.page_queue = page_queue
    
38.         self.img_queue = img_queue
    
39. 
    
40.     def run(self):
    
41.         while True:
    
42.             if self.img_queue.empty() and self.page_queue.empty():
    
43.                 break
    
44.             img_url = self.img_queue.get()
    
45.             filename = img_url.split("/")[-1]
    
46.             request.urlretrieve(img_url, "images/" + filename, None)
    
47.             print(img_url)
    
48. 
    
49. 
    
50. def main():
    
51.     page_queue = Queue(3)
    
52.     img_queue = Queue(1000)
    
53.     for x in range(1, 4):
    
54.         page_url = "https://www.doutula.com/photo/list/?page=%d" % x
    
55.         page_queue.put(page_url)
    
56. 
    
57.     for x in range(5):
    
58.         t = Producer(page_queue, img_queue)
    
59.         t.start()
    
60. 
    
61.     for x in range(5):
    
62.         t = Consumer(page_queue, img_queue)
    
63.         t.start()
    
64. 
    
65. 
    
66. if __name__ == '__main__':
    
67.     main()
    

复制代码

新手还没有权限发图
图链接https://imgchr.com/i/YpPZss

                               
登录/注册后可看大图

最佳答案

月排行榜 / 总排行榜

suchocolate

2020-5-11 18:05:17

1. import requests
   
2. from lxml import etree
   
3. from urllib import request
   
4. import threading
   
5. from queue import Queue
   
6. 
   
7. 
   
8. class Producer(threading.Thread):
   
9.     def __init__(self, page_queue, img_queue, *args, **kwargs):
   
10.         super(Producer, self).__init__(*args, **kwargs)
    
11.         self.headers = {'user-agent': 'firefox'}      # 有改动
    
12.         self.page_queue = page_queue
    
13.         self.img_queue = img_queue
    
14. 
    
15.     def run(self):
    
16.         while not self.page_queue.empty():    # 有改动
    
17.             url = self.page_queue.get()
    
18.             self.get_url(url)
    
19. 
    
20.     def get_url(self, url):
    
21.         response = requests.get(url, headers=self.headers)
    
22.         e = etree.HTML(response.text)
    
23.         img_urls = e.xpath("//div[@class='page-content text-center']/div/a/img/@data-original")
    
24.         for img in img_urls:
    
25.             self.img_queue.put(img)
    
26. 
    
27. 
    
28. class Consumer(threading.Thread):
    
29.     def __init__(self, img_queue, *args, **kwargs):
    
30.         super(Consumer, self).__init__(*args, **kwargs)
    
31.         self.img_queue = img_queue     # 有改动
    
32. 
    
33.     def run(self):
    
34.         while not self.img_queue.empty():   # 有改动
    
35.             img_url = self.img_queue.get()
    
36.             filename = img_url.split("/")[-1]
    
37.             request.urlretrieve(img_url, "images/" + filename, None)
    
38.             print(img_url)
    
39. 
    
40. 
    
41. def main():
    
42.     page_queue = Queue(3)
    
43.     img_queue = Queue(1000)
    
44.     for x in range(1, 4):
    
45.         page_url = "https://www.doutula.com/photo/list/?page=%d" % x
    
46.         page_queue.put(page_url)
    
47.     t_l_p = []    # 新增
    
48.     t_l_c = []    # 新增
    
49.     for x in range(3):    # 线程数大于队列数，会有线程不干活的
    
50.         t = Producer(page_queue, img_queue)
    
51.         t.start()
    
52.         t_l_p.append(t)
    
53.     for t in t_l_p:
    
54.         t.join()    # 得等获取完url再继续
    
55.     for x in range(3):
    
56.         t = Consumer(img_queue)
    
57.         t.start()
    
58.         t_l_c.append(t)
    
59.     for t in t_l_c:
    
60.         t.join()
    
61.     print('It's over')
    
62. 
    
63. 
    
64. if __name__ == '__main__':
    
65.     main()
    

复制代码

跳转到最佳答案楼层

liuzhengyuan

用图床发
https://imgchr.com/

xmpython

liuzhengyuan 发表于 2020-5-3 18:09
用图床发
https://imgchr.com/

欧克欧克

赚小钱

不懂python

1. 你的 queue 容量为 3， 为什么有 5 个 Producer, Consumer
2. run 函数是会自动执行吗
3. 主线程需要等待 Producer Consumer 的退出吗
4. Queue 是线程安全的吗

suchocolate

1. import requests
   
2. from lxml import etree
   
3. from urllib import request
   
4. import threading
   
5. from queue import Queue
   
6. 
   
7. 
   
8. class Producer(threading.Thread):
   
9.     def __init__(self, page_queue, img_queue, *args, **kwargs):
   
10.         super(Producer, self).__init__(*args, **kwargs)
    
11.         self.headers = {'user-agent': 'firefox'}      # 有改动
    
12.         self.page_queue = page_queue
    
13.         self.img_queue = img_queue
    
14. 
    
15.     def run(self):
    
16.         while not self.page_queue.empty():    # 有改动
    
17.             url = self.page_queue.get()
    
18.             self.get_url(url)
    
19. 
    
20.     def get_url(self, url):
    
21.         response = requests.get(url, headers=self.headers)
    
22.         e = etree.HTML(response.text)
    
23.         img_urls = e.xpath("//div[@class='page-content text-center']/div/a/img/@data-original")
    
24.         for img in img_urls:
    
25.             self.img_queue.put(img)
    
26. 
    
27. 
    
28. class Consumer(threading.Thread):
    
29.     def __init__(self, img_queue, *args, **kwargs):
    
30.         super(Consumer, self).__init__(*args, **kwargs)
    
31.         self.img_queue = img_queue     # 有改动
    
32. 
    
33.     def run(self):
    
34.         while not self.img_queue.empty():   # 有改动
    
35.             img_url = self.img_queue.get()
    
36.             filename = img_url.split("/")[-1]
    
37.             request.urlretrieve(img_url, "images/" + filename, None)
    
38.             print(img_url)
    
39. 
    
40. 
    
41. def main():
    
42.     page_queue = Queue(3)
    
43.     img_queue = Queue(1000)
    
44.     for x in range(1, 4):
    
45.         page_url = "https://www.doutula.com/photo/list/?page=%d" % x
    
46.         page_queue.put(page_url)
    
47.     t_l_p = []    # 新增
    
48.     t_l_c = []    # 新增
    
49.     for x in range(3):    # 线程数大于队列数，会有线程不干活的
    
50.         t = Producer(page_queue, img_queue)
    
51.         t.start()
    
52.         t_l_p.append(t)
    
53.     for t in t_l_p:
    
54.         t.join()    # 得等获取完url再继续
    
55.     for x in range(3):
    
56.         t = Consumer(img_queue)
    
57.         t.start()
    
58.         t_l_c.append(t)
    
59.     for t in t_l_c:
    
60.         t.join()
    
61.     print('It's over')
    
62. 
    
63. 
    
64. if __name__ == '__main__':
    
65.     main()
    

复制代码

兢兢

斗图网有反爬，多线程会挂掉