机器学习模型评估中的常见陷阱：变量误用导致指标结果一致性问题解析

import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from nltk.corpus import stopwords
from sklearn.metrics import accuracy_score, f1_score, classification_report
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.preprocessing import LabelEncoder
import joblib
import tensorflow as tf
import numpy as np
from tensorflow.keras import models, layers
import warnings

warnings.filterwarnings('ignore') # 忽略警告信息，保持输出简洁

# 加载数据集，指定编码格式
df = pd.read_csv("payload_mini.csv", encoding='utf-16')

# 筛选出感兴趣的攻击类型和正常流量
df = df[(df['attack_type'] == 'sqli') | (df['attack_type'] == 'norm')]

# 划分特征X和目标Y
X = df['payload']
y = df['label']

# 使用CountVectorizer进行文本特征向量化，过滤低频词和高频词，并移除英文停用词
vectorizer = CountVectorizer(min_df=2, max_df=0.8, stop_words=stopwords.words('english'))
X = vectorizer.fit_transform(X.values.astype('U')).toarray()

# 划分训练集和测试集，测试集占总数据的20%
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

print(f"训练集特征形状: {X_train.shape}")
print(f"训练集标签形状: {y_train.shape}")
print(f"测试集特征形状: {X_test.shape}")
print(f"测试集标签形状: {y_test.shape}")

训练集特征形状: (8040, 1585)
训练集标签形状: (8040,)
测试集特征形状: (2011, 1585)
测试集标签形状: (2011,)

nb_clf = GaussianNB()
nb_clf.fit(X_train, y_train)
y_pred_nb = nb_clf.predict(X_test) # 将朴素贝叶斯的预测结果存储在y_pred_nb中

print(f"朴素贝叶斯分类器在测试集上的准确率: {accuracy_score(y_pred_nb, y_test)}")
print(f"朴素贝叶斯分类器在测试集上的F1分数 (pos_label='anom'): {f1_score(y_pred_nb, y_test, pos_label='anom')}")
print("\n朴素贝叶斯分类报告:")
print(classification_report(y_test, y_pred_nb))

朴素贝叶斯分类器在测试集上的准确率: 0.9806066633515664
朴素贝叶斯分类器在测试集上的F1分数 (pos_label='anom'): 0.9735234215885948

朴素贝叶斯分类报告:
              precision    recall  f1-score   support

        anom       0.97      0.98      0.97       732
        norm       0.99      0.98      0.98      1279

    accuracy                           0.98      2011
   macro avg       0.98      0.98      0.98      2011
weighted avg       0.98      0.98      0.98      2011

rf_clf = RandomForestClassifier(random_state=42) # 设置random_state以确保可复现性
rf_clf.fit(X_train, y_train)
y_pred_rf = rf_clf.predict(X_test) # 随机森林模型的预测结果存储在y_pred_rf中

# 错误的评估代码：在计算accuracy_score和f1_score时，不小心引用了前一个模型（朴素贝叶斯）的预测结果y_pred_nb
print(f"随机森林分类器在测试集上的准确率 (错误示例): {accuracy_score(y_pred_nb, y_test)}") # 错误地使用了y_pred_nb
print(f"随机森林分类器在测试集上的F1分数 (pos_label='anom') (错误示例): {f1_score(y_pred_nb, y_test, pos_label='anom')}") # 错误地使用了y_pred_nb
print("\n随机森林分类报告 (基于y_pred_rf):") # 注意：classification_report这里使用了正确的y_pred_rf
print(classification_report(y_test, y_pred_rf))

随机森林分类器在测试集上的准确率 (错误示例): 0.9806066633515664
随机森林分类器在测试集上的F1分数 (pos_label='anom') (错误示例): 0.9735234215885948

随机森林分类报告 (基于y_pred_rf):
              precision    recall  f1-score   support

        anom       1.00      0.96      0.98       73