文本分类中不平衡数据处理：多类别场景下的有效策略

from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import classification_report
import numpy as np

# 假设 X 是文本数据列表，y 是对应的类别标签列表
# 模拟不平衡数据
X = ["tweet about A"] * 183 + ["tweet about B"] * 171 + \
    ["tweet about C"] * 120 + ["tweet about D"] * 110 + \
    ["tweet about E"] * 98 + ["no theme tweet"] * 964
y = ["theme A"] * 183 + ["theme B"] * 171 + \
    ["theme C"] * 120 + ["theme D"] * 110 + \
    ["theme E"] * 98 + ["no theme"] * 964

# 随机打乱数据
indices = np.arange(len(X))
np.random.shuffle(indices)
X = [X[i] for i in indices]
y = [y[i] for i in indices]

# 文本特征提取
vectorizer = TfidfVectorizer(max_features=1000)
X_vec = vectorizer.fit_transform(X)

# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X_vec, y, test_size=0.2, random_state=42, stratify=y)

# 使用 class_weight='balanced' 的逻辑回归模型
print("--- Logistic Regression with class_weight='balanced' ---")
lr_model = LogisticRegression(solver='liblinear', class_weight='balanced', random_state=42, max_iter=1000)
lr_model.fit(X_train, y_train)
y_pred_lr = lr_model.predict(X_test)
print(classification_report(y_test, y_pred_lr))

# 使用 class_weight='balanced' 的支持向量机模型
print("\n--- SVM with class_weight='balanced' ---")
svm_model = SVC(kernel='linear', class_weight='balanced', random_state=42)
svm_model.fit(X_train, y_train)
y_pred_svm = svm_model.predict(X_test)
print(classification_report(y_test, y_pred_svm))

# 示例：自定义权重（假设我们想特别强调 'theme A' 和 'theme B'）
# 首先计算各类别在训练集中的比例，然后根据需求调整
from collections import Counter
class_counts = Counter(y_train)
total_samples = sum(class_counts.values())
# 基础权重，例如可以以多数类为基准，给少数类更高的权重
custom_weights = {cls: total_samples / (len(class_counts) * count) for cls, count in class_counts.items()}
# 进一步调整，例如，将 'theme A' 和 'theme B' 的权重再提高一倍
if 'theme A' in custom_weights: custom_weights['theme A'] *= 2
if 'theme B' in custom_weights: custom_weights['theme B'] *= 2

print("\n--- Logistic Regression with custom class_weight ---")
lr_model_custom = LogisticRegression(solver='liblinear', class_weight=custom_weights, random_state=42, max_iter=1000)
lr_model_custom.fit(X_train, y_train)
y_pred_lr_custom = lr_model_custom.predict(X_test)
print(classification_report(y_test, y_pred_lr_custom))

from imblearn.over_sampling import SMOTE
from imblearn.pipeline import Pipeline
# from sklearn.ensemble import RandomForestClassifier # 或者其他分类器

# 假设 X_train_vec 和 y_train 是经过TF-IDF或其他向量化处理的训练数据

# 构建一个包含SMOTE和分类器的管道
# 注意：SMOTE应在交叉验证的训练折叠上应用
# 在实际应用中，如果使用词嵌入，X_train_vec 应该是嵌入向量
print("\n--- Logistic Regression with SMOTE ---")
pipeline = Pipeline([
    ('smote', SMOTE(random_state=42)),
    ('classifier', LogisticRegression(solver='liblinear', random_state=42, max_iter=1000))
])

pipeline.fit(X_train, y_train)
y_pred_smote = pipeline.predict(X_test)
print(classification_report(y_test, y_pred_smote))

# 对于文本数据，更推荐在稠密嵌入上使用SMOTE
# 例如：
# from sentence_transformers import SentenceTransformer
# model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
# X_embeddings = model.encode(X) # 假设X是原始文本
# ... 然后在X_embeddings上应用SMOTE