使用BeautifulSoup爬取网页表格数据：常见问题与解决方案

import requests
from bs4 import BeautifulSoup

URL = "https://genshin-impact.fandom.com/wiki/Serenitea_Pot/Load"
page = requests.get(URL)

soup = BeautifulSoup(page.content, "html.parser")

results = soup.find(id="mw-content-text")
# 检查results是否为空，以确保主内容区域被正确找到
if results:
    # 尝试查找所有class包含"article-table"的表格
    teapot_loads = results.find_all("table", class_="article-table")

    for teapot_load in teapot_loads:
        # 打印当前表格的完整class列表，以检查其在原始HTML中的实际类名
        print(f"Table classes in raw HTML: {teapot_load.get_attribute_list('class')}")
        # 尝试查找表格头元素
        table_head_element = teapot_load.find("th", class_="headerSort")
        print(f"Found table head element: {table_head_element}")
        print()
else:
    print("Could not find element with id='mw-content-text'")

import pandas as pd

url = 'https://genshin-impact.fandom.com/wiki/Serenitea_Pot/Load'

try:
    # read_html返回一个DataFrame列表，因为一个页面可能包含多个表格
    dfs = pd.read_html(url)

    # 打印所有找到的表格数量
    print(f"Found {len(dfs)} tables on the page.")

    # 根据页面内容，通常所需的表格是列表中的某一个。
    # 在本例中，通过观察和尝试，发现目标表格是索引为1的DataFrame。
    if len(dfs) > 1:
        target_table_df = dfs[1]
        print("\nExtracted Table (first few rows):")
        print(target_table_df.head())
    else:
        print("Less than 2 tables found, check index.")

except Exception as e:
    print(f"An error occurred while reading HTML: {e}")

Found X tables on the page.

Extracted Table (first few rows):
   Unnamed: 0 Image                                      Name  Adeptal Energy  Load  ReducedLoad  Ratio
0         NaN   NaN       "A Bloatty Floatty's Dream of the Sky"              60    65           47   0.92
1         NaN   NaN        "A Guide in the Summer Woods"              60    35           24   1.71
2         NaN   NaN      "A Messenger in the Summer Woods"              60    35           24   1.71
3         NaN   NaN  "A Portrait of Paimon, the Greatest Companion"              90    35           24   2.57
4         NaN   NaN               "A Seat in the Wilderness"              20    50           50   0.40