爬虫前程无忧

金刚

1. from urllib.request import Request
   
2. from urllib.request import urlopen
   
3. from lxml import etree
   
4. import re, os, random, time
   
5. 
   
6. 
   
7. def get_html(url):
   
8.        
   
9.                 headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36'}
   
10.                 req = Request(url, headers=headers)
    
11.                 response = urlopen(req)
    
12.                 html = response.read().decode("gbk")
    
13.                 return html
    
14. def html_parse(html):
    
15.        
    
16.         tree = etree.HTML(html)
    
17.         job = tree.xpath('//div[@class="el"]/p//a/@title')
    
18.         name = tree.xpath('//div[@class="el"]/span//a/@title')
    
19.         address = tree.xpath('//div[@class="el"]/span[@class="t3"]/text()')
    
20.         salary = tree.xpath('//div[@class="el"]/span[@class="t4"]/text()')
    
21.         date = tree.xpath('//div[@class="el"]/span[@class="t5"]/text()')
    
22.         return [job, name, address, salary, date]
    
23. 
    
24. def optimize_result(all_msg_list, page):
    
25. 
    
26.         try:
    
27.                 all_l_optimized = []
    
28.                 for i in range(50):
    
29.                         line = [all_msg_list[0][i], all_msg_list[1][i], all_msg_list[2][i], all_msg_list[3][i], all_msg_list[4][i]]
    
30.                         line = [line+ "....." for line in line]
    
31.                         line_msg = str(i+1)+" : "+ "职位名："+line[0]+ "公司名："+line[1]+ "工作地点："+line[2]+ "薪资："+line[3]+ "发布时间："+line[4][0:-5]+"\n\n"
    
32.                         all_l_optimized.append(line_msg)
    
33.         except IndexError:
    
34.                 line = [line+ "....." for line in line]
    
35.                 line_msg = str(i+1)+" : "+ "职位名："+line[0]+ "公司名："+line[1]+ "工作地点："+line[2]+ "薪资："+line[3]+ "发布时间："+line[4][0:-5]
    
36.                 line_msg += "\n"
    
37.                 all_l_optimized.append(line_msg)
    
38.                 return all_l_optimized
    
39.         else:
    
40.                 return all_l_optimized
    
41.         finally:
    
42.                 print("第%d页获取开始！" % page)
    
43. 
    
44. 
    
45. def save_msg(all_l_optimized, page):
    
46.        
    
47.         with open("./"+files+"/"+str(page)+".txt", "w") as f:
    
48.                 for each_job_msg in all_l_optimized:
    
49.                         f.write(each_job_msg)
    
50.                         # timer = random.choice([3, 4, 5, 6])
    
51.                         # time.sleep(3)
    
52.                
    
53. 
    
54. 
    
55. def get_job_msg(page, url):
    
56.        
    
57.         html = get_html(url)
    
58.         all_msg_list = html_parse(html)
    
59.         all_l_optimized = optimize_result(all_msg_list, page)
    
60.         save_msg(all_l_optimized, page)
    
61. 
    
62. 
    
63. files = input("请输入保存的文件夹：")
    
64. os.mkdir(files)
    
65. start_page = int(input("请输入开始页数："))
    
66. end_page = int(input("请输入结束页数："))
    
67. for page in range(start_page, end_page+1):       
    
68.         url = "https://search.51job.com/list/000000,000000,0000,00,9,99,python,2,{}.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=".format(page)
    
69.         get_job_msg(page, url)
    

复制代码