Pandas DataFrame文本预处理：数据类型与处理顺序深度解析

import pandas as pd
import nltk
from nltk.corpus import stopwords, wordnet
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from unidecode import unidecode
import contractions
import re
import string
# from textblob import TextBlob # TextBlob可能在处理单个单词时效率不高或行为不符预期，此处暂时注释

# 下载NLTK所需资源
try:
    nltk.data.find('corpora/stopwords')
except nltk.downloader.DownloadError:
    nltk.download('stopwords')
try:
    nltk.data.find('corpora/wordnet')
except nltk.downloader.DownloadError:
    nltk.download('wordnet')
try:
    nltk.data.find('taggers/averaged_perceptron_tagger')
except nltk.downloader.DownloadError:
    nltk.download('averaged_perceptron_tagger')
try:
    nltk.data.find('tokenizers/punkt')
except nltk.downloader.DownloadError:
    nltk.download('punkt')


# 初始化词形还原器和词性标注映射
lemmatizer = WordNetLemmatizer()
pos_tag_dict = {"J": wordnet.ADJ, "N": wordnet.NOUN, "V": wordnet.VERB, "R": wordnet.ADV}

# 定义停用词
local_stopwords = set(stopwords.words('english'))
additional_stopwords = ["http", "u", "get", "like", "let", "nan"]
words_to_keep = ["i'", " i ", "me", "my", "we", "our", "us"] # 注意：这里的"i'"可能需要进一步处理，例如 "i'm"
local_stopwords.update(additional_stopwords)
# 确保要保留的词不在停用词列表中
for word in words_to_keep:
    if word in local_stopwords:
        local_stopwords.remove(word)

def lemmatize_pos_tagged_text(text, lemmatizer, post_tag_dict):
    """
    对给定的文本进行分句、小写、分词、词性标注和词形还原。
    注意：此函数期望输入为单个字符串（通常是一个句子或一个单词），
    在DataFrame的apply中，如果处理的是单词列表，需要确保每次传入的是列表中的一个单词。
    """
    # 鉴于此函数在后续处理中被应用于单个单词，其内部的分句和分词逻辑需要调整
    # 如果text已经是单个单词，则直接处理
    if not isinstance(text, str): # 确保输入是字符串
        return text # 或者报错，取决于需求

    # 简化为处理单个单词的逻辑
    word = text.lower() # 确保单词小写
    pos_tuples = nltk.pos_tag([word]) # 对单个单词进行词性标注
    nltk_word_pos = pos_tuples[0][1] # 获取标注结果

    wordnet_word_pos = post_tag_dict.get(nltk_word_pos[0].upper(), None)
    if wordnet_word_pos is not None:
        new_word = lemmatizer.lemmatize(word, wordnet_word_pos)
    else:
        new_word = lemmatizer.lemmatize(word)
    return new_word

def processing_steps(df):
    """
    对DataFrame中的文本列进行一系列NLP预处理操作。

    参数:
    df (pd.DataFrame): 包含文本数据的DataFrame。

    返回:
    pd.DataFrame: 经过预处理的新DataFrame。
    """
    new_data = {} # 用于存储处理结果，避免SettingWithCopyWarning

    for column in df.columns:
        # 1. 分词 (Tokenization)
        # 将每个字符串单元格转换为单词列表
        results = df[column].apply(word_tokenize)

        # 2. 小写转换 (Lowercasing)
        # 遍历每个单词列表，将每个单词转换为小写
        results = results.apply(lambda x: [word.lower() for word in x])

        # 3. 移除停用词和非字母字符 (Stopword Removal & Non-alphabetic Filter)
        # 遍历每个单词列表，过滤掉停用词和非字母的单词
        results = results.apply(lambda tokens: [word for word in tokens if word.isalpha() and word not in local_stopwords])

        # 4. 去除变音符号 (Diacritic Replacement)
        # 遍历每个单词列表，对每个单词应用unidecode
        results = results.apply(lambda x: [unidecode(word, errors="preserve") for word in x])

        # 5. 扩展缩写 (Expand Contractions)
        # 遍历每个单词列表，对每个单词应用contractions.fix
        # 注意：contractions.fix期望字符串，如果word本身是复合词（如"don't"），word.split()仍会返回单元素列表，
        # 但contractions.fix会正确处理。
        results = results.apply(lambda x: [contractions.fix(word) for word in x]) # 简化了原有的word.split()

        # 6. 移除数字 (Remove Numbers)
        # 遍历每个单词列表，对每个单词移除数字
        results = results.apply(lambda x: [re.sub(r'\d+', '', word) for word in x])

        # 7. 移除标点符号 (Remove Punctuation except period, then remove period if it's the only char)
        # 遍历每个单词列表，对每个单词移除标点符号 (除了句号，但如果只剩句号也移除)
        # 这里需要注意，如果一个单词只包含标点，它可能在前面被isalpha()过滤掉。
        # 这里的处理是针对可能残留的标点符号。
        results = results.apply(lambda x: [re.sub(r'[%s]' % re.escape(string.punctuation), '', word) for word in x])
        # 进一步清理可能因为移除标点而产生的空字符串
        results = results.apply(lambda x: [word for word in x if word])


        # 8. 移除多余空格 (Remove Extra Spaces)
        # 遍历每个单词列表，对每个单词移除多余空格（尽管此时单词通常不含空格）
        results = results.apply(lambda x: [re.sub(r' +', ' ', word).strip() for word in x])
        # 再次清理可能产生的空字符串
        results = results.apply(lambda x: [word for word in x if word])


        # 9. 词形还原 (Lemmatization)
        # 遍历每个单词列表，对每个单词应用自定义的词形还原函数
        results = results.apply(lambda x: [lemmatize_pos_tagged_text(word, lemmatizer, pos_tag_dict) for word in x])

        # 将处理后的结果存入新字典
        new_data[column] = results

    # 从处理后的数据创建新的DataFrame
    new_df = pd.DataFrame(new_data)

    return new_df

# 创建一个示例文本DataFrame
data = {'title': ["Don't go to that HTTP site.", "It's a great day! 123.", "I'm happy to get this."],
        'body': ["The article was really good, I liked it. http://example.com", "Let's meet at 5 p.m. It's awesome.", "U should try it. Nan, it's not bad."]}
df = pd.DataFrame(data)

print("原始DataFrame:")
print(df)

# 执行预处理
processed_df = processing_steps(df.copy()) # 传入副本以防修改原始df

print("\n预处理后的DataFrame:")
print(processed_df)

原始DataFrame:
                         title                                               body
0  Don't go to that HTTP site.  The article was really good, I liked it. http:...
1          It's a great day! 123.                  Let's meet at 5 p.m. It's awesome.
2      I'm happy to get this.       U should try it. Nan, it's not bad.

预处理后的DataFrame:
                           title                         body
0                [do, not, go, site]  [article, really, good, like]
1                     [great, day]             [let, meet, awesome]
2  [i, am, happy, get, this]              [try, not, bad]