Python中高效处理大型XML文件的内存优化策略

import re
from html.parser import HTMLParser
import pandas as pd

class MyHTMLParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.data = {} # 用于存储按class分类的解析结果
        self.current_managed_object = None # 存储当前正在处理的managedObject数据
        self.current_list_name = None # 存储当前list标签的name属性
        self.current_p_name = None # 存储当前p标签的name属性

    def handle_starttag(self, tag, attrs):
        """处理开始标签事件"""
        attrs = dict(attrs) # 将属性列表转换为字典

        if tag == "managedobject":
            # 当遇到managedObject标签时，初始化一个新的数据字典
            # 从distName属性中提取MRBTS, NRBTS, NRCELL, NRREL等信息
            self.current_managed_object = dict(re.findall(r"([^/]+?)-([^/]+)", attrs["distname"])[1:])
            # 将id属性也添加到当前managedObject数据中
            self.current_managed_object["id"] = attrs["id"]
            # 根据class属性将数据添加到self.data字典中
            self.data.setdefault(attrs["class"], []).append(self.current_managed_object)
        elif tag == "list":
            # 记录当前list标签的name属性，用于构建嵌套p标签的字段名
            self.current_list_name = attrs["name"]
        elif tag == "p":
            # 记录当前p标签的name属性
            if self.current_list_name:
                # 如果p标签在list标签内部，则字段名为 Item-{list_name}-{p_name}
                self.current_p_name = f'Item-{self.current_list_name}-{attrs["name"]}'
            else:
                # 否则，字段名直接为p标签的name属性
                self.current_p_name = attrs["name"]

    def handle_endtag(self, tag):
        """处理结束标签事件"""
        if tag == "managedobject":
            # managedObject结束，清空当前对象，准备处理下一个
            self.current_managed_object = None
        elif tag == "list":
            # list标签结束，清空current_list_name
            self.current_list_name = None
        elif tag == "p":
            # p标签结束，清空current_p_name
            self.current_p_name = None

    def handle_data(self, data):
        """处理标签内部文本数据事件"""
        # 只有在正在处理managedObject且有对应的p标签名时才记录数据
        if self.current_managed_object is not None and self.current_p_name is not None:
            self.current_managed_object[self.current_p_name] = data.strip() # 移除空白符

# 实例化解析器
parser = MyHTMLParser()

# 逐行读取大文件并喂给解析器
file_path = "test.xml" # 替换为你的XML文件路径
with open(file_path, "r", encoding="utf-8") as f_in:
    for line in f_in:
        parser.feed(line)
parser.close() # 关闭解析器，释放资源

# 将解析结果转换为Pandas DataFrame并导出到Excel
output_excel_path = "output_streamed.xlsx"
with pd.ExcelWriter(output_excel_path) as writer:
    for class_name, records in parser.data.items():
        if records: # 确保有数据才创建DataFrame
            df = pd.DataFrame(records)
            # 尝试将数字字符串转换为数值类型，忽略错误
            df = df.apply(pd.to_numeric, errors="ignore")
            df.to_excel(writer, sheet_name=class_name, index=False)
            print(f"Sheet '{class_name}' created with {len(records)} records.")

print(f"数据已成功导出到 {output_excel_path}")