[已解决]爬取对应网页的各个简历模版下载，只能下载其中一个简历模版地址，而不是所有

15972156903

import requests
from lxml import etree

if __name__=="__main__":
    #Ua伪装
    headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"
        }
    #获取页面地址
    list_url = []
    url_1 = "https://sc.chinaz.com/jianli/free.html"
    list_url.append(url_1)
    for i in range(2,3):
        url_n = "https://sc.chinaz.com/jianli/free_{}.html".format(i)
        list_url.append(url_n)

    for url in list_url:
        page_text = requests.get(url=url,headers=headers).text
        tree = etree.HTML(page_text)
        jianli_div = tree.xpath("//div[@id='main']/div/div")
        for a in jianli_div:
            jianli_web = 'https:' + a.xpath("./a/@href")[0]
        
    #简历模版持久化存储
        jianli_text = requests.get(url=jianli_web,headers=headers).text
        tree = etree.HTML(jianli_text)

        download = tree.xpath("//ul[@class='clearfix']/li")
        for a in download:
            download_web = a.xpath("./a/@href")[0]
            print(download_web)

代码如上：运行结果，/Users/hudandan/PycharmProjects/爬数据/venv/bin/python /Users/hudandan/PycharmProjects/爬数据/简历模块爬取.py
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14213.rar
/jianli/201218292371.htm
/jianli/201217216600.htm
/jianli/201217576170.htm
/jianli/201216012640.htm
//sc.chinaz.com/zt/hanyi/dabao.html
//font.chinaz.com/171108141211.htm
//font.chinaz.com/171108123820.htm
//sc.chinaz.com/ppt/zongjie.html
//sc.chinaz.com/ppt/jihuashu.html
//sc.chinaz.com/jianli/hushi.html
//sc.chinaz.com/jianli/tongyong.html
//sc.chinaz.com/jiaoben/huandengpian.html
//sc.chinaz.com/jiaoben/caidanhaohang.html
//sc.chinaz.com/jiaoben/jiaodiantu.html
//sc.chinaz.com/psd/mingpianmoban.html
//sc.chinaz.com/tupian/siwameinvtupian.html
//sc.chinaz.com/tupian/rentiyishu.html
//sc.chinaz.com/ppt/shangwupptmoban.html
//font.chinaz.com/heitiziti.html
//font.chinaz.com/keaiziti.html
//sc.chinaz.com/yinxiao/RenWuPeiYin.html
//sc.chinaz.com/yinxiao/lingshengxiazai.html
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
https://downsc.chinaz.net/Files/ ... 012/jianli14193.rar
/jianli/201214415200.htm
/jianli/201213443620.htm
/jianli/201210283381.htm
/jianli/201209238730.htm
//sc.chinaz.com/zt/hanyi/dabao.html
//font.chinaz.com/171108141211.htm
//font.chinaz.com/171108123820.htm
//sc.chinaz.com/ppt/zongjie.html
//sc.chinaz.com/ppt/jihuashu.html
//sc.chinaz.com/jianli/hushi.html
//sc.chinaz.com/jianli/tongyong.html
//sc.chinaz.com/jiaoben/huandengpian.html
//sc.chinaz.com/jiaoben/caidanhaohang.html
//sc.chinaz.com/jiaoben/jiaodiantu.html
//sc.chinaz.com/psd/mingpianmoban.html
//sc.chinaz.com/tupian/siwameinvtupian.html
//sc.chinaz.com/tupian/rentiyishu.html
//sc.chinaz.com/ppt/shangwupptmoban.html
//font.chinaz.com/heitiziti.html
//font.chinaz.com/keaiziti.html
//sc.chinaz.com/yinxiao/RenWuPeiYin.html
//sc.chinaz.com/yinxiao/lingshengxiazai.html

只爬取了第一页和第二页的第一个简历模版的各个下载链接地址，其他的简历模板都没有爬取成功，不知道错在哪里。。

最佳答案

月排行榜 / 总排行榜

suchocolate

2020-12-22 13:19:11

import requests
from lxml import etree
import os


def main():
    folder = 'docs'
    if not os.path.exists(folder):
        os.mkdir(folder)
    os.chdir(folder)
    headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"}
    result = []
    url = "https://sc.chinaz.com/jianli/free.html"
    num = int(input('请输入查询的页数:'))
    for i in range(num):
        r = requests.get(url, headers=headers)
        html = etree.HTML(r.text)
        temp = html.xpath('//div[@id="main"]/div/div/a/@href')
        result.extend(temp)
        nx_url = html.xpath('//a[@class="nextpage"]/@href')[0]
        url = 'https://sc.chinaz.com/jianli/' + nx_url
    # print(result)
    file_counter = 1
    for i in result:
        url = 'https:' + i
        r = requests.get(url, headers=headers)
        html = etree.HTML(r.text)
        doc = html.xpath('//ul[@class="clearfix"]/li[1]/a/@href')[0]
        doc_name = doc.split('/')[-1]
        r = requests.get(doc, headers=headers)
        with open(doc_name, 'wb') as f:
            f.write(r.content)
            print(f'已下载{doc_name}, 共下载{file_counter}个简历。')
            file_counter += 1


if __name__ == "__main__":
    main()

跳转到最佳答案楼层

suchocolate

import requests
from lxml import etree
import os


def main():
    folder = 'docs'
    if not os.path.exists(folder):
        os.mkdir(folder)
    os.chdir(folder)
    headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"}
    result = []
    url = "https://sc.chinaz.com/jianli/free.html"
    num = int(input('请输入查询的页数:'))
    for i in range(num):
        r = requests.get(url, headers=headers)
        html = etree.HTML(r.text)
        temp = html.xpath('//div[@id="main"]/div/div/a/@href')
        result.extend(temp)
        nx_url = html.xpath('//a[@class="nextpage"]/@href')[0]
        url = 'https://sc.chinaz.com/jianli/' + nx_url
    # print(result)
    file_counter = 1
    for i in result:
        url = 'https:' + i
        r = requests.get(url, headers=headers)
        html = etree.HTML(r.text)
        doc = html.xpath('//ul[@class="clearfix"]/li[1]/a/@href')[0]
        doc_name = doc.split('/')[-1]
        r = requests.get(doc, headers=headers)
        with open(doc_name, 'wb') as f:
            f.write(r.content)
            print(f'已下载{doc_name}, 共下载{file_counter}个简历。')
            file_counter += 1


if __name__ == "__main__":
    main()

15972156903

suchocolate 发表于 2020-12-22 13:19

我现仔细看看大佬的代码。谢谢！

15972156903

suchocolate 发表于 2020-12-22 13:19

我看了哈我的代码，没有问题，就是jianli_text哪里开始都要缩进一下就可以了。。感谢大佬的代码