|
马上注册,结交更多好友,享用更多功能^_^
您需要 登录 才可以下载或查看,没有账号?立即注册
x
本帖最后由 ontheway168 于 2020-5-16 00:13 编辑
- requests爬虫.py文件
- import re
- import requests
- from excel_write import excel_init
- workbook, worksheet = excel_init()
- #获取总页码数
- def get_pages():
- #模拟一个虚拟网站
- user_header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}
- #对应地区测试工程师搜索网址
- web_url = 'https://search.51job.com/list/030800,000000,0000,00,9,99,%25E6%25B5%258B%25E8%25AF%2595%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='
- res = requests.get(web_url, headers=user_header)
- res.encoding = 'gbk'
- pages = int(re.findall('<span class="td">共(.*?)页,到第</span>', res.text, re.S)[0].strip())
- return pages
- row = 1
- for page in range(1, get_pages()+1):
- #模拟一个虚拟网站
- user_header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}
- web_url = f'https://search.51job.com/list/030800,000000,0000,00,9,99,%25E6%25B5%258B%25E8%25AF%2595%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,2,{page}.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='
- res = requests.get(web_url, headers=user_header)
- res.encoding = 'gbk'
- # print(res.text)
- '''
- <div class="el">
- <p class="t1 tg1">
- <em class="check" name="delivery_em"></em>
- <input class="checkbox" type="checkbox" name="delivery_jobid" value="120608298" jt="0" style="display:none">
- <span>
- <a target="_blank" title="测试工程师" href="https://jobs.51job.com/dongguan/120608298.html?s=01&t=0">
- 测试工程师 </a>
- </span>
- <img width="30" height="20" src="//img01.51jobcdn.com/im/jobs/tag_xiaozhao.png?20190903" alt="校招">
- </p>
- <span class="t2"><a target="_blank" title="东莞新能德科技有限公司" href="https://jobs.51job.com/all/co5796557.html">东莞新能德科技有限公司</a></span>
- <span class="t3">东莞</span>
- <span class="t4"></span>
- <span class="t5">05-15</span>
- </div>
- '''
- lines = re.findall('<div class="el">(.*?)</div>', res.text, re.S)
- for line in lines:
- temp = re.findall('<a target="_blank" title="(.*?)" href', line, re.S)
- job = temp[0].strip()
- worksheet.write(row, 0, job)
- company = temp[1].strip()
- worksheet.write(row, 1, company)
- address = re.findall('<span class="t3">(.*?)</span>', line, re.S)[0].strip()
- worksheet.write(row, 2, address)
- salary = re.findall('<span class="t4">(.*?)</span>', line, re.S)[0].strip()
- worksheet.write(row, 3, salary)
- datetime = re.findall('<span class="t5">(.*?)</span>', line, re.S)[0].strip()
- worksheet.write(row, 4, datetime)
- row += 1
- # print(job,company,address,salary,datetime)
- workbook.save('./51job岗位信息.xls')
- excel_write.py文件
- import xlwt
- #创建一个excel文件,创建表单,写
- def excel_init():
- workbook = xlwt.Workbook(encoding='utf-8')
- worksheet = workbook.add_sheet('51job岗位信息')
- colNames = ['岗位名称', '公司名称', '地址', '薪资', '发布时间']
- for i in range(len(colNames)):
- worksheet.write(0, i, colNames[i])#(行号,列号,内容),行列号从0开始
- return workbook, worksheet
- [/i]
复制代码 |
|