设为首页收藏本站
切换到窄版

鱼C论坛

 找回密码
 立即注册
鱼C论坛»论坛 › 编程语言专区 › Python交流 › 智联招聘爬虫问题求救！
返回列表 发新帖
查看: 2823|回复: 3

[已解决]智联招聘爬虫问题求救！

[复制链接]
熊猫竹子
发表于 2022-11-14 17:07:46 | 显示全部楼层 |阅读模式

马上注册，结交更多好友，享用更多功能^_^

您需要 登录 才可以下载或查看，没有账号？立即注册

x
目的：想爬取智联招聘网站的特定关键词的列表招聘信息和列表内招聘详情信息
问题：通过request模块和xpath定位到的内容和实际浏览器显示的内容不一致；下面是我的代码（纯小白，刚学的）有大佬解释一下是什么原因吗？


import requests
from lxml import etree
if __name__ == "__main__":
    #伪装浏览器
    headers = {'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.35'}
    #输入url
    #设置通用url
    #设置翻页url, 字符串中的{}.format()被替换，f'__'和.format()一样
    for pagenum in range(1,2):
        url = f'https://sou.zhaopin.com/?jl=763&kw=%E5%92%A8%E8%AF%A2&p={pagenum}'
    #获取参数
        params = {
            'accept': ' text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'accept-encoding': ' gzip, deflate, br',
            'accept-language': ' zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'cache-control': ' max-age=0',
            'cookie': ' x-zp-client-id=59fe1cae-f844-4a11-99bf-886b90370523; campusOperateJobUserInfo=a51e859b-1db1-4ced-b6df-4557593737b7; FSSBBIl1UgzbN7NO=5Oj3DFqFWYkxjSLGVtPQc72nArgWa90JU_JBoGEfhoz_Jlc7LsbF8tCJx2gicQuttOmssC0zdFyy7Ci6JmhK.oa; _uab_collina=166754798597534651997073; locationInfo_search={%22code%22:%22763%22%2C%22name%22:%22%E5%B9%BF%E5%B7%9E%22%2C%22message%22:%22%E5%8C%B9%E9%85%8D%E5%88%B0%E5%B8%82%E7%BA%A7%E7%BC%96%E7%A0%81%22}; LastCity=%E5%B9%BF%E5%B7%9E; LastCity%5Fid=763; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221044786621%22%2C%22first_id%22%3A%221843c1da41e95-017c0825a51e454-45647f52-1049088-1843c1da420381%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E8%87%AA%E7%84%B6%E6%90%9C%E7%B4%A2%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC%22%2C%22%24latest_referrer%22%3A%22https%3A%2F%2Fcn.bing.com%2F%22%2C%22%24latest_utm_source%22%3A%22360sem_b%22%2C%22%24latest_utm_medium%22%3A%22CPC%22%2C%22%24latest_utm_campaign%22%3A%22pp%22%2C%22%24latest_utm_content%22%3A%22bb%22%2C%22%24latest_utm_term%22%3A%2285622%22%7D%2C%22identities%22%3A%22eyIkaWRlbnRpdHlfY29va2llX2lkIjoiMTg0M2MxZGE0MWU5NS0wMTdjMDgyNWE1MWU0NTQtNDU2NDdmNTItMTA0OTA4OC0xODQzYzFkYTQyMDM4MSIsIiRpZGVudGl0eV9sb2dpbl9pZCI6IjEwNDQ3ODY2MjEifQ%3D%3D%22%2C%22history_login_id%22%3A%7B%22name%22%3A%22%24identity_login_id%22%2C%22value%22%3A%221044786621%22%7D%2C%22%24device_id%22%3A%221843c1da41e95-017c0825a51e454-45647f52-1049088-1843c1da420381%22%7D; selectCity_search=763; ssxmod_itna2=QqUxRieGqWqeqDKitDXDnBAtG=k+WO70+r3buxA6W5edD/+7YDFoR09ZPAPo8C5QBcSv6=q1iu+rz/D3hH6mtG6kq=Y9Y0uGXjKeUqQ61q7Uuh4RIOQYPG2Gl408Deq+D===; Hm_lvt_38ba284938d5eddca645bb5e02a02006=1667547986,1667873580,1667957310,1668056889; ssxmod_itna=QqGx9Dy7eYqwG7DzxAO4=KDtei=Y3f5GCYmUDBqfT4iNDnD8x7YDvm+EBm78SYnGDcXxvKmxp4=fKDOe3qsir7SooDU4i8DCkDw3bDeW=D5xGoDPxDeDADYE6DAqiOD7qDdfhTXtkDbxi3fxDbDim8mxGCDeKD0ZbFDQKDuEF5ZAFhOpBCrKyDP4xefxG1T40H1C3xfcYffbizP4fAD+4ODlKUDCF1uEyFr4Gd66v1DRqPA0DPb74q8Yr5QxhKwixx5cQqt=DP17iNP20D8QD4ZArnT74M4D; ZL_REPORT_GLOBAL={%22jobs%22:{%22funczoneShare%22:%22dtl_best_for_you%22%2C%22recommandActionidShare%22:%224b73965e-3682-41c6-9c01-5cb9fd099e43-job%22}}; zp_passport_deepknow_sessionId=25c482d3s20ec9461481c822f78edd7234e1; at=2e4a6506ec3a4e8095dbf71e06d82e37; rt=353844b5b82b4f1ab191f55fbab1827f; BEST_EMPLOYER_SHOW_TIME=[1667547773556%2C1667873519700%2C1668059329333]; acw_tc=2760828916680593302367664eb4a8e8082758d1bdab23beea2e0e19797a42; Hm_lpvt_38ba284938d5eddca645bb5e02a02006=1668059338; FSSBBIl1UgzbN7NP=53CnASCJ79WQqqqDl3rK9vA7o5w6B4qVFKttE5ESjI634Jn_yZqRthX3InsuOyvPZ6Hxj_KtNuXjM5ZFg5CViGkStIIlKoLuewH7YPEDYDDH1Twe1T_fyWbgEHCIRO27cJEInIEw_qQ_fFQP7Um4jL5DulQBzXBnWG9WSxvpAyASg3_0_KZzuSqRVK00QdWDXAoudH6ahVwfkUMOTAamHc2Fz6bFX1g0mmkdzG11xLfSyPVMSCW6LKdeUkU31P54.PZ0cWj36engC6Kc8g9Qe81AZckoi.adqXbxoZG6T1_uZUR2.ZmhrBobjmlGh5Tz.e4YgcD2ZctwbG7bWKwA3RK',
            'referer': ' https://www.zhaopin.com/',
            'sec-ch-ua': ' "Microsoft Edge";v="107", "Chromium";v="107", "Not=A?Brand";v="24"',
            'sec-ch-ua-mobile': ' ?0',
            'sec-ch-ua-platform': ' "Windows"',
            'sec-fetch-dest': ' document',
            'sec-fetch-mode': ' navigate',
            'sec-fetch-site': ' same-origin',
            'sec-fetch-user': ' ?1',
            'upgrade-insecure-requests': ' 1',
            'user-agent': ' Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.35',

        }
        #请求数据
        response = requests.get(url=url, params=params)
        # 对响应值进行存储赋值
        page_content = response.text

        with open('page_content.html','w', encoding='utf-8') as fp:
            fp.write(page_content)
        print('over!')
        #对数据进行解析
        #将数据导入etree
        parser = etree.HTMLParser(encoding='utf-8')
        data = etree.parse('page_content.html', parser=parser)
        #1.定位数据所在的标签,提取标签内的数据[去文本用text(),去属性下的值用/@XXX
        VR = data.xpath('/html/body/div/div[4]/div[2]/div[2]//span/@title')
        print(VR)
最佳答案
月排行榜 / 总排行榜
cflying
2022-11-14 18:27:12
本帖最后由 cflying 于 2022-11-14 18:31 编辑

requests是爬不到渲染后的页面的，得模拟浏览器
前半截是读取浏览器cookies的，后半截才是爬取和写入excel，自己看着改吧，格式乱了点，将就用
  1. import sqlite3
  2. import requests
  3. import os
  4. import json
  5. import random

  7. import sys
  8. import base64
  9. from cryptography.hazmat.backends import default_backend
  10. from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes


  13. def dpapi_decrypt(encrypted):
  14.     import ctypes
  15.     import ctypes.wintypes

  17.     class DATA_BLOB(ctypes.Structure):
  18.         _fields_ = [('cbData', ctypes.wintypes.DWORD),
  19.                     ('pbData', ctypes.POINTER(ctypes.c_char))]

  21.     p = ctypes.create_string_buffer(encrypted, len(encrypted))
  22.     blobin = DATA_BLOB(ctypes.sizeof(p), p)
  23.     blobout = DATA_BLOB()
  24.     retval = ctypes.windll.crypt32.CryptUnprotectData(
  25.         ctypes.byref(blobin), None, None, None, None, 0, ctypes.byref(blobout))
  26.     if not retval:
  27.         raise ctypes.WinError()
  28.     result = ctypes.string_at(blobout.pbData, blobout.cbData)
  29.     ctypes.windll.kernel32.LocalFree(blobout.pbData)
  30.     return result


  33. def aes_decrypt(encrypted_txt):
  34.     with open(os.path.join(os.environ['LOCALAPPDATA'],
  35.                            r"Microsoft\Edge\User Data\Local State"), encoding='utf-8', mode="r") as f:
  36.         jsn = json.loads(str(f.readline()))
  37.     encoded_key = jsn["os_crypt"]["encrypted_key"]
  38.     encrypted_key = base64.b64decode(encoded_key.encode())
  39.     encrypted_key = encrypted_key[5:]
  40.     key = dpapi_decrypt(encrypted_key)
  41.     nonce = encrypted_txt[3:15]
  42.     cipher = Cipher(algorithms.AES(key), None, backend=default_backend())
  43.     cipher.mode = modes.GCM(nonce)
  44.     decryptor = cipher.decryptor()
  45.     return decryptor.update(encrypted_txt[15:])


  48. def chrome_decrypt(encrypted_txt):
  49.     if sys.platform == 'win32':
  50.         try:
  51.             if encrypted_txt[:4] == b'x01x00x00x00':
  52.                 decrypted_txt = dpapi_decrypt(encrypted_txt)
  53.                 return decrypted_txt.decode()
  54.             elif encrypted_txt[:3] == b'v10':
  55.                 decrypted_txt = aes_decrypt(encrypted_txt)
  56.                 return decrypted_txt[:-16].decode()
  57.         except WindowsError:
  58.             return None
  59.     else:
  60.         raise WindowsError


  63. def get_cookies_from_chrome(domain):
  64.     sql = f'SELECT name, encrypted_value as value FROM cookies where host_key like "%{domain}%"'
  65.     if os.path.exists(os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Profile 3\Cookies'))==True:
  66.         filename = os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Profile 3\Cookies')
  67.     elif  os.path.exists(os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Default\Cookies'))==True:
  68.         filename = os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Default\Cookies')
  69.     elif  os.path.exists(os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Profile 3\Network\Cookies'))==True:
  70.         filename = os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Profile 3\Network\Cookies')
  71.     elif  os.path.exists(os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Default\Network\Cookies'))==True:
  72.         filename = os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Default\Network\Cookies')
  73.     else:
  74.         print('未找到cookies路径')
  75.     con = sqlite3.connect(filename)
  76.     con.row_factory = sqlite3.Row
  77.     cur = con.cursor()
  78.     cur.execute(sql)
  79.     cookie = ''
  80.     for row in cur:
  81.         if row['value'] is not None:
  82.             name = row['name']
  83.             value = chrome_decrypt(row['value'])
  84.             if value is not None:
  85.                 cookie += name + '=' + value + ';'
  86.     con.close()
  87.     return cookie

  89. #设置host，url
  90. host='.zhaopin.com'
  91. #获取url数据
  92. cookies_tmp=get_cookies_from_chrome(host)
  93. cookies=[]
  94. for i in cookies_tmp.split(";"):
  95.     if i:
  96.         if i.split("=")[-1]:
  97.             cookies.append({'name':i.split("=")[0],'value':i.split("=")[-1],'domain':host,'path': '/'})



  101. '''-----------------------------------------------------------------------------------------------------'''
  102. #使用前edeg浏览器要登录账号
  103. from playwright.sync_api import sync_playwright
  104. from bs4 import BeautifulSoup
  105. import openpyxl


  108. url0=r'https://passport.zhaopin.com/login?bkUrl=%2F%2Fi.zhaopin.com%2Fblank%3Fhttps%3A%2F%2Fwww.zhaopin.com%2F%3FvalidateCampus%253D'
  109. url1=r'https://sou.zhaopin.com/?jl=551&kw=%E4%BC%81%E7%AE%A1&p={}'




  114. def getinfo(uu):
  115.     p=1
  116.     url=uu.format(str(p))
  117.     tmp_data=[]
  118.     while True:
  119.         print(url)
  120.         page.goto(url)
  121.         bs = BeautifulSoup(page.content(),"html.parser")
  122.         for i in bs.find_all('div',attrs={'class':'joblist-box__item clearfix'}):
  123.             tmp_data.append([i.find('span',attrs={'class':'iteminfo__line1__jobname__name'}).text,i.find('p',attrs={'class':'iteminfo__line2__jobdesc__salary'}).text,'kong',i.find('span',attrs={'class':'iteminfo__line1__compname__name'}).text,i.find('a').attrs['href']])

  125.         if 'disabled' in bs.find('div',attrs={'class':'soupager'}).find_all('button')[1].attrs:
  126.             break
  127.         else:
  128.             p+=1
  129.             url=uu.format(str(p))

  131.     return tmp_data


  134. book = openpyxl.load_workbook('./infodata.xlsx')
  135. sheet = book['zhilian']
  136. with sync_playwright() as p:
  137.     browser = p.chromium.launch()
  138.     context = browser.new_context()
  139.     context.add_cookies(cookies=cookies)
  140.     page = context.new_page()
  141.     for u in [url1]:
  142.         data=getinfo(u)
  143.         for row in data:
  144.             sheet.append(row)
  145.         print('写入数据条数：',len(data))
  146.         book.save('./infodata.xlsx')
  147.     page.close()
  148.     browser.close()
  149. #input('结束')
复制代码
小甲鱼最新课程 -> https://ilovefishc.com
回复

使用道具 举报

cflying
发表于 2022-11-14 18:27:12 | 显示全部楼层    本楼为最佳答案   
本帖最后由 cflying 于 2022-11-14 18:31 编辑

requests是爬不到渲染后的页面的，得模拟浏览器
前半截是读取浏览器cookies的，后半截才是爬取和写入excel，自己看着改吧，格式乱了点，将就用
  1. import sqlite3
  2. import requests
  3. import os
  4. import json
  5. import random

  7. import sys
  8. import base64
  9. from cryptography.hazmat.backends import default_backend
  10. from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes


  13. def dpapi_decrypt(encrypted):
  14.     import ctypes
  15.     import ctypes.wintypes

  17.     class DATA_BLOB(ctypes.Structure):
  18.         _fields_ = [('cbData', ctypes.wintypes.DWORD),
  19.                     ('pbData', ctypes.POINTER(ctypes.c_char))]

  21.     p = ctypes.create_string_buffer(encrypted, len(encrypted))
  22.     blobin = DATA_BLOB(ctypes.sizeof(p), p)
  23.     blobout = DATA_BLOB()
  24.     retval = ctypes.windll.crypt32.CryptUnprotectData(
  25.         ctypes.byref(blobin), None, None, None, None, 0, ctypes.byref(blobout))
  26.     if not retval:
  27.         raise ctypes.WinError()
  28.     result = ctypes.string_at(blobout.pbData, blobout.cbData)
  29.     ctypes.windll.kernel32.LocalFree(blobout.pbData)
  30.     return result


  33. def aes_decrypt(encrypted_txt):
  34.     with open(os.path.join(os.environ['LOCALAPPDATA'],
  35.                            r"Microsoft\Edge\User Data\Local State"), encoding='utf-8', mode="r") as f:
  36.         jsn = json.loads(str(f.readline()))
  37.     encoded_key = jsn["os_crypt"]["encrypted_key"]
  38.     encrypted_key = base64.b64decode(encoded_key.encode())
  39.     encrypted_key = encrypted_key[5:]
  40.     key = dpapi_decrypt(encrypted_key)
  41.     nonce = encrypted_txt[3:15]
  42.     cipher = Cipher(algorithms.AES(key), None, backend=default_backend())
  43.     cipher.mode = modes.GCM(nonce)
  44.     decryptor = cipher.decryptor()
  45.     return decryptor.update(encrypted_txt[15:])


  48. def chrome_decrypt(encrypted_txt):
  49.     if sys.platform == 'win32':
  50.         try:
  51.             if encrypted_txt[:4] == b'x01x00x00x00':
  52.                 decrypted_txt = dpapi_decrypt(encrypted_txt)
  53.                 return decrypted_txt.decode()
  54.             elif encrypted_txt[:3] == b'v10':
  55.                 decrypted_txt = aes_decrypt(encrypted_txt)
  56.                 return decrypted_txt[:-16].decode()
  57.         except WindowsError:
  58.             return None
  59.     else:
  60.         raise WindowsError


  63. def get_cookies_from_chrome(domain):
  64.     sql = f'SELECT name, encrypted_value as value FROM cookies where host_key like "%{domain}%"'
  65.     if os.path.exists(os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Profile 3\Cookies'))==True:
  66.         filename = os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Profile 3\Cookies')
  67.     elif  os.path.exists(os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Default\Cookies'))==True:
  68.         filename = os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Default\Cookies')
  69.     elif  os.path.exists(os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Profile 3\Network\Cookies'))==True:
  70.         filename = os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Profile 3\Network\Cookies')
  71.     elif  os.path.exists(os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Default\Network\Cookies'))==True:
  72.         filename = os.path.join(os.environ['USERPROFILE'], r'AppData\Local\Microsoft\Edge\User Data\Default\Network\Cookies')
  73.     else:
  74.         print('未找到cookies路径')
  75.     con = sqlite3.connect(filename)
  76.     con.row_factory = sqlite3.Row
  77.     cur = con.cursor()
  78.     cur.execute(sql)
  79.     cookie = ''
  80.     for row in cur:
  81.         if row['value'] is not None:
  82.             name = row['name']
  83.             value = chrome_decrypt(row['value'])
  84.             if value is not None:
  85.                 cookie += name + '=' + value + ';'
  86.     con.close()
  87.     return cookie

  89. #设置host，url
  90. host='.zhaopin.com'
  91. #获取url数据
  92. cookies_tmp=get_cookies_from_chrome(host)
  93. cookies=[]
  94. for i in cookies_tmp.split(";"):
  95.     if i:
  96.         if i.split("=")[-1]:
  97.             cookies.append({'name':i.split("=")[0],'value':i.split("=")[-1],'domain':host,'path': '/'})



  101. '''-----------------------------------------------------------------------------------------------------'''
  102. #使用前edeg浏览器要登录账号
  103. from playwright.sync_api import sync_playwright
  104. from bs4 import BeautifulSoup
  105. import openpyxl


  108. url0=r'https://passport.zhaopin.com/login?bkUrl=%2F%2Fi.zhaopin.com%2Fblank%3Fhttps%3A%2F%2Fwww.zhaopin.com%2F%3FvalidateCampus%253D'
  109. url1=r'https://sou.zhaopin.com/?jl=551&kw=%E4%BC%81%E7%AE%A1&p={}'




  114. def getinfo(uu):
  115.     p=1
  116.     url=uu.format(str(p))
  117.     tmp_data=[]
  118.     while True:
  119.         print(url)
  120.         page.goto(url)
  121.         bs = BeautifulSoup(page.content(),"html.parser")
  122.         for i in bs.find_all('div',attrs={'class':'joblist-box__item clearfix'}):
  123.             tmp_data.append([i.find('span',attrs={'class':'iteminfo__line1__jobname__name'}).text,i.find('p',attrs={'class':'iteminfo__line2__jobdesc__salary'}).text,'kong',i.find('span',attrs={'class':'iteminfo__line1__compname__name'}).text,i.find('a').attrs['href']])

  125.         if 'disabled' in bs.find('div',attrs={'class':'soupager'}).find_all('button')[1].attrs:
  126.             break
  127.         else:
  128.             p+=1
  129.             url=uu.format(str(p))

  131.     return tmp_data


  134. book = openpyxl.load_workbook('./infodata.xlsx')
  135. sheet = book['zhilian']
  136. with sync_playwright() as p:
  137.     browser = p.chromium.launch()
  138.     context = browser.new_context()
  139.     context.add_cookies(cookies=cookies)
  140.     page = context.new_page()
  141.     for u in [url1]:
  142.         data=getinfo(u)
  143.         for row in data:
  144.             sheet.append(row)
  145.         print('写入数据条数：',len(data))
  146.         book.save('./infodata.xlsx')
  147.     page.close()
  148.     browser.close()
  149. #input('结束')
复制代码
小甲鱼最新课程 -> https://ilovefishc.com
回复 支持 反对

使用道具 举报

熊猫竹子
 楼主| 发表于 2022-11-14 20:37:47 | 显示全部楼层
谢谢大佬！我研究研究
小甲鱼最新课程 -> https://ilovefishc.com
回复 支持 反对

使用道具 举报

熊猫竹子
 楼主| 发表于 2022-11-14 20:40:27 | 显示全部楼层
cflying 发表于 2022-11-14 18:27
requests是爬不到渲染后的页面的，得模拟浏览器
前半截是读取浏览器cookies的，后半截才是爬取和写入excel ...

谢谢大佬！我研究研究
小甲鱼最新课程 -> https://ilovefishc.com
回复 支持 反对

使用道具 举报

返回列表 发新帖
高级模式
B Color Image Link Quote Code Smilies
您需要登录后才可以回帖 登录 | 立即注册

本版积分规则

小黑屋|手机版|Archiver|鱼C工作室 ( 粤ICP备18085999号-1 | 粤公网安备 44051102000585号)

GMT+8, 2025-4-25 22:13

Powered by Discuz! X3.4

© 2001-2023 Discuz! Team.

快速回复 返回顶部 返回列表