解决Web抓取中HTML结构不一致问题的策略与实践

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0'
}

import httpx
import trio
import re
import pandas as pd
from bs4 import BeautifulSoup # 仅作为参考，实际解决方案中可能不直接使用BeautifulSoup解析表格

# 模拟浏览器User-Agent
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0'
}

async def fetch_table_data(identifier: str):
    """
    异步函数，用于根据标识符从IBM文档页面抓取表格数据。
    """
    async with httpx.AsyncClient(headers=headers, base_url='https://www.ibm.com/docs') as client:
        # 1. 构造初始页面URL参数
        initial_params = {
            'topic': f'tables-{identifier}' # 假设identifier用于构建topic参数
        }

        # 2. 发送初始请求，获取可能包含API路径的页面
        try:
            initial_response = await client.get('en/imdm/12.0', params=initial_params, follow_redirects=True)
            initial_response.raise_for_status() # 检查HTTP状态码
        except httpx.HTTPStatusError as e:
            print(f"Error fetching initial page for identifier {identifier}: {e}")
            return None
        except httpx.RequestError as e:
            print(f"Network error for identifier {identifier}: {e}")
            return None

        # 3. 从初始响应中提取内容API的oldUrl
        match = re.search(r'"oldUrl":"(.*?)"', initial_response.text)
        if not match:
            print(f"Could not find 'oldUrl' in initial response for identifier {identifier}. Trying direct parse.")
            # 如果没有找到oldUrl，尝试直接解析，以防有些页面直接返回内容
            try:
                # 尝试用pandas直接解析，如果页面直接包含表格
                df_list = pd.read_html(initial_response.content, attrs={'class': 'defaultstyle'})
                if df_list:
                    print(f"Successfully parsed table directly for {identifier}.")
                    return df_list[0]
                else:
                    print(f"No table found directly for {identifier}.")
                    return None
            except Exception as e:
                print(f"Direct parsing failed for {identifier}: {e}")
                return None

        old_url_path = match.group(1)

        # 4. 构造内容API的URL
        content_api_url = "api/v1/content/" + old_url_path
        content_api_params = {
            'parsebody': 'true',
            'lang': 'en'
        }

        # 5. 发送API请求，获取实际内容
        try:
            api_response = await client.get(content_api_url, params=content_api_params)
            api_response.raise_for_status()
        except httpx.HTTPStatusError as e:
            print(f"Error fetching content API for identifier {identifier} ({content_api_url}): {e}")
            return None
        except httpx.RequestError as e:
            print(f"Network error for content API {identifier} ({content_api_url}): {e}")
            return None

        # 6. 使用pandas直接解析表格
        try:
            # pandas.read_html返回一个DataFrame列表
            df_list = pd.read_html(api_response.content, attrs={'class': 'defaultstyle'})
            if df_list:
                print(f"Successfully extracted table for identifier: {identifier}")
                return df_list[0]
            else:
                print(f"No table with class 'defaultstyle' found in API response for {identifier}.")
                return None
        except Exception as e:
            print(f"Error parsing HTML table for identifier {identifier}: {e}")
            return None

async def main():
    # 示例用法：假设我们有一个identifiers.csv文件
    # df_identifiers = pd.read_csv('identifiers.csv')
    # identifiers = df_identifiers['Identifier'].tolist()

    # 为了演示，我们直接使用一个示例标识符
    identifiers = ['t-accessdateval', 'r_Tables', 'another_identifier'] # 替换为你的实际标识符列表

    all_results = []
    async with trio.open_nursery() as nursery:
        for identifier in identifiers:
            # 使用nursery.start_soon并行运行抓取任务
            nursery.start_soon(
                lambda id=identifier: trio.run(
                    lambda: fetch_table_data(id)
                ).add_done_callback(
                    lambda fut: all_results.append((id, fut.result()))
                )
            )
            # 注意：trio.run不能嵌套，这里是一个简化的演示。
            # 实际生产代码中，应将所有任务直接传递给一个trio.run的顶层async函数。
            # 更规范的trio并行：
            # async def run_all_fetches():
            #     async with trio.open_nursery() as nursery:
            #         for identifier in identifiers:
            #             nursery.start_soon(fetch_and_store, identifier, all_results)
            # await run_all_fetches()
            # 
            # def fetch_and_store(identifier, results_list):
            #     data = await fetch_table_data(identifier)
            #     results_list.append((identifier, data))

    # 由于trio.run不能嵌套，上述并行代码需要调整。
    # 这里为了演示，我们先简化为串行执行，但在实际应用中推荐使用trio的并行特性。
    print("\n--- Running tasks sequentially for demonstration ---")
    for identifier in identifiers:
        df = await fetch_table_data(identifier)
        if df is not None:
            print(f"\nData for {identifier}:\n{df.head()}")
            # 将每个DataFrame存储起来，或者进行合并
            all_results.append(df)

    # 合并所有DataFrame
    if all_results:
        combined_df = pd.concat(all_results, ignore_index=True)
        print("\n--- Combined DataFrame (first 5 rows) ---")
        print(combined_df.head())
        # combined_df.to_csv('combined_table_data.csv', index=False)
    else:
        print("No data was successfully extracted.")


if __name__ == "__main__":
    trio.run(main)