解决LabelEncoder在训练/测试数据中遇到未知标签的ValueError

import pandas as pd
from sklearn.preprocessing import LabelEncoder

# 假设 train_data 和 test_data 是 MultiIndex DataFrame
# 假设 categoricals 是需要编码的列名列表，例如 ['TRAITS', 'UNIT_1']

le = {col: LabelEncoder() for col in categoricals}

# 错误的编码方式示例
# train_data 和 test_data 结构类似原始问题中的 MultiIndex
# train_data[(col, sub_col)] 是一个 Series
# test_data[(col, sub_col)] 是一个 Series

for col_level0 in train_data.columns.levels[0]: # 例如 'TEAM ONE', 'TEAM TWO'
    for col_level1 in train_data.columns.levels[1]: # 例如 'TRAITS', 'UNIT_1'
        if col_level1 in le: # 如果当前二级列名是需要编码的分类列
            # 在训练数据上拟合并转换
            train_data[(col_level0, col_level1)] = le[col_level1].fit_transform(
                train_data[(col_level0, col_level1)].astype(str)
            )
            # 在测试数据上直接转换
            test_data[(col_level0, col_level1)] = le[col_level1].transform(
                test_data[(col_level0, col_level1)].astype(str)
            )

import pandas as pd
from sklearn.preprocessing import LabelEncoder
import numpy as np

# 1. 创建模拟的 MultiIndex DataFrame 数据
# 定义需要编码的分类列（二级列名）
categoricals = ['TRAITS', 'UNIT_1', 'AUGMENT_1']

# 模拟训练数据
train_data_raw = {
    ('TEAM ONE', 'TRAITS'): ['A', 'B', 'C', 'A', 'B'],
    ('TEAM ONE', 'UNIT_1'): ['X', 'Y', 'Z', 'X', 'Y'],
    ('TEAM ONE', 'AUGMENT_1'): ['P', 'Q', 'R', 'P', 'Q'],
    ('TEAM TWO', 'TRAITS'): ['A', 'C', 'D', 'A', 'C'],
    ('TEAM TWO', 'UNIT_1'): ['X', 'Z', 'W', 'X', 'Z'],
    ('TEAM TWO', 'AUGMENT_1'): ['P', 'R', 'S', 'P', 'R'],
}
train_df = pd.DataFrame(train_data_raw)
train_df.columns = pd.MultiIndex.from_tuples(train_df.columns)

# 模拟测试数据 (故意包含训练集未见的标签，如 'E', 'F', 'V', 'T', 'G', 'H', 'U', 'K')
test_data_raw = {
    ('TEAM ONE', 'TRAITS'): ['A', 'B', 'E', 'A', 'F'],
    ('TEAM ONE', 'UNIT_1'): ['X', 'Y', 'V', 'X', 'Y'],
    ('TEAM ONE', 'AUGMENT_1'): ['P', 'Q', 'T', 'P', 'Q'],
    ('TEAM TWO', 'TRAITS'): ['A', 'C', 'G', 'A', 'H'],
    ('TEAM TWO', 'UNIT_1'): ['X', 'Z', 'U', 'X', 'Z'],
    ('TEAM TWO', 'AUGMENT_1'): ['P', 'R', 'K', 'P', 'R'],
}
test_df = pd.DataFrame(test_data_raw)
test_df.columns = pd.MultiIndex.from_tuples(test_df.columns)

print("原始训练数据:\n", train_df)
print("\n原始测试数据:\n", test_df)

# 2. 初始化 LabelEncoder 字典
le = {col: LabelEncoder() for col in categoricals}

# 3. 正确的编码方法：在所有数据上拟合编码器
train_df_encoded = train_df.copy()
test_df_encoded = test_df.copy()

for sub_col_name in categoricals:
    # 收集当前二级列名在所有一级列（TEAM ONE, TEAM TWO）和所有数据集（train, test）中的所有唯一值
    all_values_for_sub_col = pd.Series(dtype='object')
    for team_col in train_df.columns.levels[0]: # 遍历一级列名，如 'TEAM ONE', 'TEAM TWO'
        # 检查 MultiIndex 列是否存在，以防数据不完整
        if (team_col, sub_col_name) in train_df.columns:
            all_values_for_sub_col = pd.concat([all_values_for_sub_col, train_df[(team_col, sub_col_name)].astype(str)])
        if (team_col, sub_col_name) in test_df.columns:
            all_values_for_sub_col = pd.concat([all_values_for_sub_col, test_df[(team_col, sub_col_name)].astype(str)])

    # 使用所有收集到的唯一值来拟合对应的 LabelEncoder
    # .unique() 确保只传递唯一值，提高效率
    le[sub_col_name].fit(all_values_for_sub_col.unique())

    # 4. 使用拟合好的编码器转换训练集和测试集
    for team_col in train_df_encoded.columns.levels[0]:
        if (team_col, sub_col_name) in train_df_encoded.columns:
            train_df_encoded[(team_col, sub_col_name)] = le[sub_col_name].transform(
                train_df_encoded[(team_col, sub_col_name)].astype(str)
            )
        if (team_col, sub_col_name) in test_df_encoded.columns:
            test_df_encoded[(team_col, sub_col_name)] = le[sub_col_name].transform(
                test_df_encoded[(team_col, sub_col_name)].astype(str)
            )

print("\n编码后的训练数据 (正确方法):\n", train_df_encoded)
print("\n编码后的测试数据 (正确方法):\n", test_df_encoded)

# 验证逆转换（可选）
# print("\n逆转换示例 (TEAM ONE, TRAITS 在测试数据中):")
# print(le['TRAITS'].inverse_transform(test_df_encoded[('TEAM ONE', 'TRAITS')]))