机器学习模型评估中指标结果一致性异常排查与最佳实践

import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from nltk.corpus import stopwords
from sklearn.metrics import accuracy_score, f1_score, classification_report
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
import warnings

warnings.filterwarnings('ignore')

# 加载数据
df = pd.read_csv("payload_mini.csv", encoding='utf-16')
df = df[(df['attack_type'] == 'sqli') | (df['attack_type'] == 'norm')]

X = df['payload']
y = df['label']

# 文本特征提取
vectorizer = CountVectorizer(min_df=2, max_df=0.8, stop_words=stopwords.words('english'))
X = vectorizer.fit_transform(X.values.astype('U')).toarray()

# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # 添加random_state以确保可复现性
print(f"X_train shape: {X_train.shape}, y_train shape: {y_train.shape}")
print(f"X_test shape: {X_test.shape}, y_test shape: {y_test.shape}")

# 高斯朴素贝叶斯分类器
nb_clf = GaussianNB()
nb_clf.fit(X_train, y_train)
y_pred_nb = nb_clf.predict(X_test) # 使用y_pred_nb作为预测结果变量

print("--- Naive Bayes Classifier ---")
print(f"Accuracy of Naive Bayes on test set : {accuracy_score(y_pred_nb, y_test)}")
print(f"F1 Score of Naive Bayes on test set : {f1_score(y_pred_nb, y_test, pos_label='anom')}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred_nb))

--- Naive Bayes Classifier ---
Accuracy of Naive Bayes on test set : 0.9806066633515664
F1 Score of Naive Bayes on test set : 0.9735234215885948

Classification Report:
              precision    recall  f1-score   support

        anom       0.97      0.98      0.97       732
        norm       0.99      0.98      0.98      1279

    accuracy                           0.98      2011
   macro avg       0.98      0.98      0.98      2011
weighted avg       0.98      0.98      0.98      2011

# 随机森林分类器 (错误示例)
rf_clf = RandomForestClassifier(random_state=42) # 添加random_state
rf_clf.fit(X_train, y_train)
y_pred_rf = rf_clf.predict(X_test) # 随机森林的预测结果

print("\n--- Random Forest Classifier (Problematic) ---")
# 错误地使用了y_pred_nb（或之前未定义的y_pred）来计算随机森林的指标
print(f"Accuracy of Random Forest on test set : {accuracy_score(y_pred_nb, y_test)}") # 错误：应为y_pred_rf
print(f"F1 Score of Random Forest on test set : {f1_score(y_pred_nb, y_test, pos_label='anom')}") # 错误：应为y_pred_rf
print("\nClassification Report:")
print(classification_report(y_test, y_pred_rf)) # 注意：这里Classification Report是正确的，因为它使用了y_pred_rf

# 随机森林分类器 (修正后)
rf_clf = RandomForestClassifier(random_state=42)
rf_clf.fit(X_train, y_train)
y_pred_rf = rf_clf.predict(X_test) # 随机森林的预测结果

print("\n--- Random Forest Classifier (Corrected) ---")
# 使用正确的y_pred_rf来计算随机森林的指标
print(f"Accuracy of Random Forest on test set : {accuracy_score(y_pred_rf, y_test)}") # 已修正
print(f"F1 Score of Random Forest on test set : {f1_score(y_pred_rf, y_test, pos_label='anom')}") # 已修正
print("\nClassification Report:")
print(classification_report(y_test, y_pred_rf))