通过python爬取https://sofifa.com/所有的球员数据

zchb258

本人80后和一些球友常玩这个实况10游戏，因为球员和球员数据总是不断更新，单方面手动复制粘贴工程量很大，下面这篇文章很棒但我实在看不懂，看得有点晕头转向，想请各位大神结合文章出一下完整版，或给个教程，谢谢了

这是原链接https://mp.weixin.qq.com/s/1KX3dw3cfX1J7vk0andL8A
或许看图片文章

FishC

嗨！我了解你的问题。爬取网页数据是一个常见的需求，可以通过Python中的requests库来实现。下面是一个简单的示例代码，用于从https://sofifa.com/网站上爬取所有球员数据：

import requests
from bs4 import BeautifulSoup

url = "https://sofifa.com/"

# 发送HTTP请求
response = requests.get(url)

# 解析HTML
soup = BeautifulSoup(response.text, "html.parser")

# 获取球员数据所在的标签
player_divs = soup.find_all("div", class_="col-name")

# 遍历每个球员标签，并提取数据
for player_div in player_divs:
    # 提取球员名字和评分
    name = player_div.a.text.strip()
    rating = player_div.parent.find("div", class_="col col-oa").text.strip()

    # 打印结果
    print(name, rating)

在这个示例中，我们使用了requests库发送HTTP请求，然后使用BeautifulSoup库解析返回的HTML文档。通过查找特定的标签和类名，我们可以提取出球员的姓名和评分。

请注意，网站的结构可能会有所变化，所以代码可能需要根据实际情况进行调整。此外，爬取网站数据也需要尊重网站的规则和政策，不要进行过于频繁的请求，以免对网站造成负担。

希望这个简单的示例能帮助到你！如果还有其他问题，请随时提问。

以上回复来自 FishC AI 小助理，如未能正确解答您的问题，请继续追问。

小甲鱼的二师兄

帮你整合了一下代码：

import requests
from bs4 import BeautifulSoup
import re
import csv

def fetchData(id):
    url = f'https://sofifa.com/player/{str(id)}'
    headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
    'Referer': 'https://www.google.com/'
    }
    myRequest = requests.get(url)
    soup = BeautifulSoup(myRequest.text,'lxml') 
    print(myRequest)
    meta = soup.find(attrs={'name':'description'})
    if meta is None:
        return None
    
    meta = meta['content']
    years = soup.find(name='span',attrs={'class':'bp3-button-text'})
    if meta[:4] != 'FIFA' and (str(years.string)) != "FIFA 23" or meta[:4]=='FIFA':
        return None
    
    info = soup.find(name='div',attrs={'class':'info'})
    playerName = info.h1.string
    myList = [id, playerName]
    
    rawdata = soup.select("#body > div:nth-child(5) > div > div.col.col-12 > div.bp3-card.player > div > div")
    offset = rawdata[0].find_all("span")
    offset = (len(offset))-1
    temp = rawdata[0].text
    temp = re.split('\s+',temp)
    if offset > 0:
        for i in range(offset):
            temp.pop(i)
            
    month = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]
    mon = temp[3][1:]
    mon = month.index(mon)+1
    day = temp[4][:-1] 
    year = temp[5][:-1]
    birthday = [f"{str(year)}/{str(mon)}/{str(day)}"]
    birthday = eval(str(birthday)[1:-1])
    myList.append(birthday)
    
    height = int(temp[6][:-2])
    myList.append(height)
    weight = int(temp[7][:-2]) 
    myList.append(weight)
    
    rawdata = soup.select("#body > div:nth-child(5) > div > div.col.col-12 > div:nth-child(2) > div > ul")
    temp = rawdata[0].find_all('li',class_="ellipsis")
    preferred_foot = temp[0].contents[1]
    preferred_foot = 1 if (preferred_foot == 'Left') else 2
    myList.append(preferred_foot)
    
    skill_move_level = temp[2].contents[0]
    myList.append(int(skill_move_level))
    
    reputation = temp[3].contents[0]
    myList.append(int(reputation))
    
    todostr = temp[4].text
    workrateString = re.split('\s+',todostr)
    wr_att = workrateString[1][4:-1]
    wr_def = workrateString[2]
    wrList = ['Low',"Medium","High"]
    wr_att = wrList.index(wr_att)+1
    wr_def = wrList.index(wr_def)+1 
    myList.append(wr_att)
    myList.append(wr_def)
    
    rawdata = soup.select("#body > div:nth-child(5) > div > div.col.col-12 > div.bp3-card.player > img")
    img_url = rawdata[0].get("data-src")
    img_r = requests.get(img_url,stream=True)
    img_name = f"{id}_{playerName}.png"
    with open(f"./{img_name}","wb") as fi:
        for chunk in img_r.iter_content(chunk_size=120):
            fi.write(chunk)
            
    rawdata = soup.select("#body > div:nth-child(5) > div > div.col.col-12 > div.bp3-card.player > div > div > span") 
    allPos = ''.join(f"{p.text} " for p in rawdata)
    myList.append(allPos)
    
    rawdata = soup.select("#body > div:nth-child(6) > div > div.col.col-4 > ul > li:nth-child(1) > span")
    bestPos = rawdata[0].text
    myList.append(bestPos)
    
    rawdata = soup.select("#body > div:nth-child(5) > div > div.col.col-12 > div:nth-child(4) > div > h5> a")
    club = rawdata[0].text if len(rawdata)>0 else "没有俱乐部"
    myList.append(club)
    
    rawdata = soup.select("#body > div:nth-child(5) > div > div.col.col-12 > div.bp3-card.player > div > div > a")
    nation = rawdata[0].get("title") if len(rawdata)>0 else "其他国家"
    myList.append(nation)
    
    rawdata = soup.select('#body>div:nth-child(6)>div>div.col.col-12')
    data = rawdata[0].find_all(class_=re.compile('bp3-tag p'))
    myList.extend(allatt.text for allatt in data)
    
    return myList

def dealWithData(dataToWrite):
    header_list = ['id','name','birthday','height','weight','preferred_foot',"skill_move_level","reputation","wr_att","wr_def",'Positions','Best Position','Club',"nation",'Crossing','Finishing','Heading Accuracy', 'Short Passing','Volleys','Dribbling','Curve', 'FK Accuracy','Long Passing','Ball Control','Acceleration','Sprint Speed','Agility','Reactions','Balance','Shot Power','Jumping','Stamina','Strength','Long Shots','Aggression','Interceptions','Positioning','Vision','Penalties','Composure','Defensive Awareness','Standing Tackle','Sliding Tackle','GK Diving','GK Handling','GK Kicking','GK Positioning','GK Reflexes'] 
    with open('./output.csv', 'a+', encoding='utf-8-sig', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(header_list)
        writer.writerows(dataToWrite)

def getPlayerID(key):
    url = f"https://sofifa.com/players?keyword={str(key)}"
    myRequest = requests.get(url)
    soup = BeautifulSoup(myRequest.text,'lxml')
    playerTable = soup.select("#body>div.center>div>div.col.col-12>div>table>tbody")
    data = playerTable[0].contents
    playersCandicate = []
    
    if len(data) > 0:
        for p in data:
            id = p.find("img")["id"]
            name = p.find("a")["aria-label"]
            ovr = p.find(attrs={"data-col":"oa"}).get_text()
            playersCandicate.append([id,name,ovr])
    else:
        print("not found")
        playersCandicate.append(["not found","the name you're searching is >>", key])
        
    return playersCandicate

if __name__ == "__main__":
    # 通过递增ID搜索
    for start in range(20000, 40000, 1000):  # 每次爬取1000个球员
        soData = []
        for s in range(start, start + 1000):
            l = fetchData(s)
            if l != None:
                soData.append(l)
        dealWithData(soData)
        time.sleep(60)  # 爬取完一批次后,休眠60秒