Python中如何预处理文本数据？

import re

def clean_text(text):
    # 去除HTML标签
    text = re.sub(r'<.*?>', '', text)
    # 去除特殊字符和数字
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    # 去除多余的空格
    text = re.sub(r'\s+', ' ', text).strip()
    return text.lower()

sample_text = "<p>Hello, World! 123</p>"
cleaned_text = clean_text(sample_text)
print(cleaned_text)  # 输出: hello world

import nltk
from nltk.tokenize import word_tokenize

nltk.download('punkt')

def tokenize_text(text):
    tokens = word_tokenize(text)
    return tokens

tokenized_text = tokenize_text(cleaned_text)
print(tokenized_text)  # 输出: ['hello', 'world']

from nltk.corpus import stopwords

nltk.download('stopwords')

def remove_stopwords(tokens):
    stop_words = set(stopwords.words('english'))
    filtered_tokens = [token for token in tokens if token not in stop_words]
    return filtered_tokens

filtered_text = remove_stopwords(tokenized_text)
print(filtered_text)  # 输出: ['hello', 'world']

from nltk.stem import PorterStemmer, WordNetLemmatizer

nltk.download('wordnet')

def stem_and_lemmatize(tokens):
    stemmer = PorterStemmer()
    lemmatizer = WordNetLemmatizer()
    stemmed_tokens = [stemmer.stem(token) for token in tokens]
    lemmatized_tokens = [lemmatizer.lemmatize(token) for token in tokens]
    return stemmed_tokens, lemmatized_tokens

stemmed, lemmatized = stem_and_lemmatize(filtered_text)
print("Stemmed:", stemmed)  # 输出: ['hello', 'world']
print("Lemmatized:", lemmatized)  # 输出: ['hello', 'world']

import pandas as pd

# 假设我们有一个包含文本数据的DataFrame
data = pd.DataFrame({
    'text': ['<p>Hello, World! 123</p>', '<p>Welcome to Python! 456</p>']
})

# 应用清理函数
data['cleaned_text'] = data['text'].apply(clean_text)

# 应用分词函数
data['tokenized_text'] = data['cleaned_text'].apply(tokenize_text)

# 应用去除停用词函数
data['filtered_text'] = data['tokenized_text'].apply(remove_stopwords)

print(data)