BERT主题模型中异常值（-1主题）的优化策略

from bertopic import BERTopic
from sentence_transformers import SentenceTransformer
import pandas as pd

# 假设您有一个名为 'skills_augmented' 的文档列表
# 这里我们创建一个模拟数据用于演示
docs = [
    "Python programming skills",
    "Java development experience",
    "Data analysis with Pandas",
    "Machine learning algorithms",
    "Cloud computing expertise",
    "Project management methodologies",
    "SQL database administration",
    "Frontend web development",
    "Backend API design",
    "Agile software development",
    "Excellent communication skills", # 潜在的异常值
    "Team collaboration abilities",   # 潜在的异常值
    "Strategic planning",             # 潜在的异常值
    "Problem-solving aptitude",       # 潜在的异常值
    "Customer service orientation"    # 潜在的异常值
] * 100 # 模拟40000个文档，这里简化为1500个

# 1. 加载或训练嵌入模型
llm_mod = "all-MiniLM-L6-v2"
model = SentenceTransformer(llm_mod)

# 2. 生成文档嵌入
embeddings = model.encode(docs, show_progress_bar=True)

# 3. 训练BERTopic模型
# verbose=True 会显示训练过程中的信息
bertopic_model = BERTopic(verbose=True)
topics, probs = bertopic_model.fit_transform(docs, embeddings)

print("--- 原始主题分布 ---")
# 统计主题分布
topic_counts = pd.Series(topics).value_counts()
print(topic_counts.head())

# 4. 减少异常值
# new_topics 将包含重新分配后的主题ID
new_topics = bertopic_model.reduce_outliers(docs, topics, embeddings=embeddings)

print("\n--- 优化后的主题分布 ---")
# 统计优化后的主题分布
new_topic_counts = pd.Series(new_topics).value_counts()
print(new_topic_counts.head())

# 检查-1主题的文档数量是否减少
if -1 in topic_counts.index:
    original_outliers = topic_counts[-1]
    print(f"\n原始-1主题文档数: {original_outliers}")
else:
    original_outliers = 0
    print("\n原始主题分布中没有-1主题。")

if -1 in new_topic_counts.index:
    new_outliers = new_topic_counts[-1]
    print(f"优化后-1主题文档数: {new_outliers}")
else:
    new_outliers = 0
    print("优化后主题分布中没有-1主题。")

if new_outliers < original_outliers:
    print(f"成功减少了 {original_outliers - new_outliers} 个异常值。")
elif original_outliers > 0 and new_outliers == original_outliers:
    print("未能减少异常值，可能需要尝试其他策略或调整参数。")
else:
    print("异常值处理完成。")