[已解决]python线程

哈岁NB

请问这样写算多线程吗，还有想问一下这个时间怎么打印不出来

1. import threading
   
2. import multiprocessing
   
3. import time
   
4. 
   
5. import requests
   
6. import logging
   
7. import re
   
8. from urllib.parse import urljoin
   
9. from os.path import exists
   
10. from os import makedirs
    
11. 
    
12. result_dir = 'tu'
    
13. exists(result_dir) or makedirs(result_dir)
    
14. 
    
15. 
    
16. logging.basicConfig(level=logging.INFO,format = '%(asctime)s - %(levelname)s: %(message)s')
    
17. 
    
18. header = {
    
19.    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
    
20. }
    
21. url = 'http://md.itlun.cn/a/nhtp/list_2_{}.html'
    
22. pages = 3
    
23. 
    
24. 
    
25. def scrpae_page(url):
    
26.     logging.info('scraping %s...', url)
    
27.     try:
    
28.         response = requests.get(url=url,headers=header)
    
29.         if response.status_code == 200:
    
30.             response.encoding = 'gbk'
    
31.             return response
    
32.         logging.error('get invalid status code %s while scrape %s',response.status_code,url)
    
33.     except requests.RequestException:
    
34.         logging.error('error occurred while scraping %s',url,exc_info=True)
    
35. 
    
36. #拼接URl，并爬取主页
    
37. def scrape_index(page):
    
38.     index_url = url.format(page)
    
39.     return scrpae_page(index_url)
    
40. 
    
41. #解析详情页url
    
42. def parse_index(html):
    
43.     #logging.info('{}'.format(html))
    
44.     url_pattern = re.compile('<script.*?src = "(.*?)"; </script>',re.S)
    
45.     items = re.findall(url_pattern,html)
    
46.     title_pattern = re.compile('<LI>.*?><IMG id="img.*?><span>(.*?)</span></a></LI>',re.S)
    
47.     titles = re.findall(title_pattern,html)
    
48.     # logging.info('{}'.format(list(titles)))
    
49. 
    
50.     return {
    
51.         '名称':titles,
    
52.         'detail_url':items
    
53.     }
    
54. 
    
55. #拼接url，并调用scrpae_page
    
56. def parse_url(path):
    
57.     detail_url = urljoin(url,path)
    
58.     return scrpae_page(detail_url).content
    
59. 
    
60. #保存
    
61. def save(title,conect):
    
62.     invalid_chars = ['\\', '/', ':', '*', '?', '"', '<', '>', '|']  # Invalid Windows characters
    
63.     for char in invalid_chars:
    
64.         title = title.replace(char, '')
    
65.     img_path = result_dir + '/' + title + '.jpg'
    
66.     with open(img_path,'wb') as fp:
    
67.         fp.write(conect)
    
68. 
    
69. 
    
70. 
    
71. def main(page):
    
72.    
    
73.     index_html = scrape_index(page)
    
74.     titles = parse_index(index_html.text)['名称']
    
75.     details_url = parse_index(index_html.text)['detail_url']
    
76.     for title,url in zip(titles,details_url):
    
77.         conect = parse_url(url)
    
78.         save(title,conect)
    
79.         logging.info('保存成功')
    
80.    
    
81. 
    
82. 
    
83. 
    
84. if __name__ == '__main__':
    
85.     start = time.time()
    
86.     for page in range(1,pages):
    
87.         t = threading.Thread(target=main,args=(page,))
    
88.         t.start()
    
89.     end = time.time()
    
90.     print('time : {}'.format(end-start))

复制代码

最佳答案

月排行榜 / 总排行榜

isdkz

2023-5-30 18:48:18

这样写算多线程，时间不是没有打印，而是早就打印了，因为子线程在运行的同时主线程也会继续运行的

你可以使用 join 让主线程等待子线程运行结束

对你的代码修改如下（新增的内容在 86、92、93行）：

1. import threading
   
2. import multiprocessing
   
3. import time
   
4. 
   
5. import requests
   
6. import logging
   
7. import re
   
8. from urllib.parse import urljoin
   
9. from os.path import exists
   
10. from os import makedirs
    
11. 
    
12. result_dir = 'tu'
    
13. exists(result_dir) or makedirs(result_dir)
    
14. 
    
15. 
    
16. logging.basicConfig(level=logging.INFO,format = '%(asctime)s - %(levelname)s: %(message)s')
    
17. 
    
18. header = {
    
19.    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
    
20. }
    
21. url = 'http://md.itlun.cn/a/nhtp/list_2_{}.html'
    
22. pages = 3
    
23. 
    
24. 
    
25. def scrpae_page(url):
    
26.     logging.info('scraping %s...', url)
    
27.     try:
    
28.         response = requests.get(url=url,headers=header)
    
29.         if response.status_code == 200:
    
30.             response.encoding = 'gbk'
    
31.             return response
    
32.         logging.error('get invalid status code %s while scrape %s',response.status_code,url)
    
33.     except requests.RequestException:
    
34.         logging.error('error occurred while scraping %s',url,exc_info=True)
    
35. 
    
36. #拼接URl，并爬取主页
    
37. def scrape_index(page):
    
38.     index_url = url.format(page)
    
39.     return scrpae_page(index_url)
    
40. 
    
41. #解析详情页url
    
42. def parse_index(html):
    
43.     #logging.info('{}'.format(html))
    
44.     url_pattern = re.compile('<script.*?src = "(.*?)"; </script>',re.S)
    
45.     items = re.findall(url_pattern,html)
    
46.     title_pattern = re.compile('<LI>.*?><IMG id="img.*?><span>(.*?)</span></a></LI>',re.S)
    
47.     titles = re.findall(title_pattern,html)
    
48.     # logging.info('{}'.format(list(titles)))
    
49. 
    
50.     return {
    
51.         '名称':titles,
    
52.         'detail_url':items
    
53.     }
    
54. 
    
55. #拼接url，并调用scrpae_page
    
56. def parse_url(path):
    
57.     detail_url = urljoin(url,path)
    
58.     return scrpae_page(detail_url).content
    
59. 
    
60. #保存
    
61. def save(title,conect):
    
62.     invalid_chars = ['\\', '/', ':', '*', '?', '"', '<', '>', '|']  # Invalid Windows characters
    
63.     for char in invalid_chars:
    
64.         title = title.replace(char, '')
    
65.     img_path = result_dir + '/' + title + '.jpg'
    
66.     with open(img_path,'wb') as fp:
    
67.         fp.write(conect)
    
68. 
    
69. 
    
70. 
    
71. def main(page):
    
72.    
    
73.     index_html = scrape_index(page)
    
74.     titles = parse_index(index_html.text)['名称']
    
75.     details_url = parse_index(index_html.text)['detail_url']
    
76.     for title,url in zip(titles,details_url):
    
77.         conect = parse_url(url)
    
78.         save(title,conect)
    
79.         logging.info('保存成功')
    
80.    
    
81. 
    
82. 
    
83. 
    
84. if __name__ == '__main__':
    
85.     start = time.time()
    
86.     tasks = []
    
87.     for page in range(1,pages):
    
88.         t = threading.Thread(target=main,args=(page,))
    
89.         tasks.append(t)
    
90.         t.start()
    
91. 
    
92.     for t in tasks:
    
93.         t.join()
    
94.     end = time.time()
    
95.     print('time : {}'.format(end-start))
    

复制代码

跳转到最佳答案楼层

isdkz

这样写算多线程，时间不是没有打印，而是早就打印了，因为子线程在运行的同时主线程也会继续运行的

你可以使用 join 让主线程等待子线程运行结束

对你的代码修改如下（新增的内容在 86、92、93行）：

1. import threading
   
2. import multiprocessing
   
3. import time
   
4. 
   
5. import requests
   
6. import logging
   
7. import re
   
8. from urllib.parse import urljoin
   
9. from os.path import exists
   
10. from os import makedirs
    
11. 
    
12. result_dir = 'tu'
    
13. exists(result_dir) or makedirs(result_dir)
    
14. 
    
15. 
    
16. logging.basicConfig(level=logging.INFO,format = '%(asctime)s - %(levelname)s: %(message)s')
    
17. 
    
18. header = {
    
19.    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
    
20. }
    
21. url = 'http://md.itlun.cn/a/nhtp/list_2_{}.html'
    
22. pages = 3
    
23. 
    
24. 
    
25. def scrpae_page(url):
    
26.     logging.info('scraping %s...', url)
    
27.     try:
    
28.         response = requests.get(url=url,headers=header)
    
29.         if response.status_code == 200:
    
30.             response.encoding = 'gbk'
    
31.             return response
    
32.         logging.error('get invalid status code %s while scrape %s',response.status_code,url)
    
33.     except requests.RequestException:
    
34.         logging.error('error occurred while scraping %s',url,exc_info=True)
    
35. 
    
36. #拼接URl，并爬取主页
    
37. def scrape_index(page):
    
38.     index_url = url.format(page)
    
39.     return scrpae_page(index_url)
    
40. 
    
41. #解析详情页url
    
42. def parse_index(html):
    
43.     #logging.info('{}'.format(html))
    
44.     url_pattern = re.compile('<script.*?src = "(.*?)"; </script>',re.S)
    
45.     items = re.findall(url_pattern,html)
    
46.     title_pattern = re.compile('<LI>.*?><IMG id="img.*?><span>(.*?)</span></a></LI>',re.S)
    
47.     titles = re.findall(title_pattern,html)
    
48.     # logging.info('{}'.format(list(titles)))
    
49. 
    
50.     return {
    
51.         '名称':titles,
    
52.         'detail_url':items
    
53.     }
    
54. 
    
55. #拼接url，并调用scrpae_page
    
56. def parse_url(path):
    
57.     detail_url = urljoin(url,path)
    
58.     return scrpae_page(detail_url).content
    
59. 
    
60. #保存
    
61. def save(title,conect):
    
62.     invalid_chars = ['\\', '/', ':', '*', '?', '"', '<', '>', '|']  # Invalid Windows characters
    
63.     for char in invalid_chars:
    
64.         title = title.replace(char, '')
    
65.     img_path = result_dir + '/' + title + '.jpg'
    
66.     with open(img_path,'wb') as fp:
    
67.         fp.write(conect)
    
68. 
    
69. 
    
70. 
    
71. def main(page):
    
72.    
    
73.     index_html = scrape_index(page)
    
74.     titles = parse_index(index_html.text)['名称']
    
75.     details_url = parse_index(index_html.text)['detail_url']
    
76.     for title,url in zip(titles,details_url):
    
77.         conect = parse_url(url)
    
78.         save(title,conect)
    
79.         logging.info('保存成功')
    
80.    
    
81. 
    
82. 
    
83. 
    
84. if __name__ == '__main__':
    
85.     start = time.time()
    
86.     tasks = []
    
87.     for page in range(1,pages):
    
88.         t = threading.Thread(target=main,args=(page,))
    
89.         tasks.append(t)
    
90.         t.start()
    
91. 
    
92.     for t in tasks:
    
93.         t.join()
    
94.     end = time.time()
    
95.     print('time : {}'.format(end-start))
    

复制代码

哈岁NB

isdkz 发表于 2023-5-30 18:48
这样写算多线程，时间不是没有打印，而是早就打印了，因为子线程在运行的同时主线程也会继续运行的

你可 ...

好的，感谢感谢