Python中如何进行文本分类？

import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

nltk.download('punkt')
nltk.download('stopwords')

def preprocess_text(text):
    # 分词
    tokens = word_tokenize(text.lower())
    # 去除停用词和标点符号
    stop_words = set(stopwords.words('english'))
    tokens = [token for token in tokens if token.isalnum() and token not in stop_words]
    return ' '.join(tokens)

# 示例文本
text = "This is an example sentence for text classification."
processed_text = preprocess_text(text)
print(processed_text)

from sklearn.feature_extraction.text import TfidfVectorizer

# 假设我们有一组文本
texts = ["This is the first document.", "This document is the second document.", "And this is the third one.", "Is this the first document?"]

# 使用TF-IDF进行特征提取
vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(texts)
print(X.toarray())

from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

# 假设我们有对应的标签
labels = ['class1', 'class2', 'class1', 'class2']

# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.2, random_state=42)

# 训练模型
clf = MultinomialNB()
clf.fit(X_train, y_train)

# 预测并评估
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred))

from imblearn.over_sampling import RandomOverSampler

# 假设我们有一个不平衡的数据集
X, y = ...  # 你的数据

# 使用过采样来平衡数据集
ros = RandomOverSampler(random_state=42)
X_resampled, y_resampled = ros.fit_resample(X, y)

# 重新训练模型
X_train, X_test, y_train, y_test = train_test_split(X_resampled, y_resampled, test_size=0.2, random_state=42)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred))

from sklearn.model_selection import GridSearchCV

# 定义参数网格
param_grid = {
    'alpha': [0.1, 0.5, 1.0],
    'fit_prior': [True, False]
}

# 使用网格搜索进行调参
grid_search = GridSearchCV(MultinomialNB(), param_grid, cv=5)
grid_search.fit(X_train, y_train)

# 输出最佳参数和最佳得分
print("Best parameters:", grid_search.best_params_)
print("Best cross-validation score:", grid_search.best_score_)

# 使用最佳参数重新训练模型
best_clf = grid_search.best_estimator_
best_clf.fit(X_train, y_train)
y_pred = best_clf.predict(X_test)
print(classification_report(y_test, y_pred))