利用python爬取51job工作岗位信息，并导入excel

ontheway168

requests爬虫.py文件
import re
import requests
from excel_write import excel_init
workbook, worksheet = excel_init()
#获取总页码数
def get_pages():
    #模拟一个虚拟网站
    user_header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}
    #对应地区测试工程师搜索网址
    web_url = 'https://search.51job.com/list/030800,000000,0000,00,9,99,%25E6%25B5%258B%25E8%25AF%2595%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='
    res = requests.get(web_url, headers=user_header)
    res.encoding = 'gbk'
    pages = int(re.findall('<span class="td">共(.*?)页，到第</span>', res.text, re.S)[0].strip())
    return pages

row = 1

for page in range(1, get_pages()+1):
    #模拟一个虚拟网站
    user_header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}

    web_url = f'https://search.51job.com/list/030800,000000,0000,00,9,99,%25E6%25B5%258B%25E8%25AF%2595%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,2,{page}.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='

    res = requests.get(web_url, headers=user_header)
    res.encoding = 'gbk'

    # print(res.text)
    '''
    <div class="el">
            <p class="t1 tg1">
                <em class="check" name="delivery_em"></em>
                <input class="checkbox" type="checkbox" name="delivery_jobid" value="120608298" jt="0" style="display:none">
                <span>
                    <a target="_blank" title="测试工程师" href="https://jobs.51job.com/dongguan/120608298.html?s=01&t=0">
                        测试工程师                </a>
                </span>
                                                                    <img width="30" height="20" src="//img01.51jobcdn.com/im/jobs/tag_xiaozhao.png?20190903" alt="校招">
                                    </p>
            <span class="t2"><a target="_blank" title="东莞新能德科技有限公司" href="https://jobs.51job.com/all/co5796557.html">东莞新能德科技有限公司</a></span>
            <span class="t3">东莞</span>
            <span class="t4"></span>
            <span class="t5">05-15</span>
        </div>
    '''
    lines = re.findall('<div class="el">(.*?)</div>', res.text, re.S)

    for line in lines:
        temp = re.findall('<a target="_blank" title="(.*?)" href', line, re.S)
        job = temp[0].strip()
        worksheet.write(row, 0, job)
        company = temp[1].strip()
        worksheet.write(row, 1, company)
        address = re.findall('<span class="t3">(.*?)</span>', line, re.S)[0].strip()
        worksheet.write(row, 2, address)
        salary = re.findall('<span class="t4">(.*?)</span>', line, re.S)[0].strip()
        worksheet.write(row, 3, salary)
        datetime = re.findall('<span class="t5">(.*?)</span>', line, re.S)[0].strip()
        worksheet.write(row, 4, datetime)
        row += 1
        # print(job,company,address,salary,datetime)

workbook.save('./51job岗位信息.xls')


excel_write.py文件
import xlwt
#创建一个excel文件，创建表单，写
def excel_init():
    workbook = xlwt.Workbook(encoding='utf-8')
    worksheet = workbook.add_sheet('51job岗位信息')

    colNames = ['岗位名称', '公司名称', '地址', '薪资', '发布时间']
    for i in range(len(colNames)):
        worksheet.write(0, i, colNames[i])#(行号，列号，内容)，行列号从0开始
    return workbook, worksheet

[/i]

Otaku007

可以嫖走学习吗