Python与NLTK：高效提取文本名词的实用指南

# 安装NLTK库
# pip install nltk

import nltk

# 下载必要的NLTK数据包
# 如果是首次运行，需要执行以下命令下载
try:
    nltk.data.find('tokenizers/punkt')
except nltk.downloader.DownloadError:
    nltk.download('punkt')
try:
    nltk.data.find('taggers/averaged_perceptron_tagger')
except nltk.downloader.DownloadError:
    nltk.download('averaged_perceptron_tagger')
# 可选：下载停用词列表，如果需要去除停用词
try:
    nltk.data.find('corpora/stopwords')
except nltk.downloader.DownloadError:
    nltk.download('stopwords')

from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords

text_response = "Marriage is a big step in one's life. Python is a popular programming language used for various tasks, including data analysis and web development. NLTK provides powerful tools for natural language processing."

# 1. 文本分句
sentences = sent_tokenize(text_response)
print("分句结果:", sentences)

# 2. 词语标记化（针对每个句子）
all_words = []
for sentence in sentences:
    words = word_tokenize(sentence)
    all_words.extend(words)
print("\n词语标记化结果（所有词语）:", all_words)

分句结果: ["Marriage is a big step in one's life.", 'Python is a popular programming language used for various tasks, including data analysis and web development.', 'NLTK provides powerful tools for natural language processing.']

词语标记化结果（所有词语）: ['Marriage', 'is', 'a', 'big', 'step', 'in', 'one', "'s", 'life', '.', 'Python', 'is', 'a', 'popular', 'programming', 'language', 'used', 'for', 'various', 'tasks', ',', 'including', 'data', 'analysis', 'and', 'web', 'development', '.', 'NLTK', 'provides', 'powerful', 'tools', 'for', 'natural', 'language', 'processing', '.']

# 3. 词性标注
tagged_words = nltk.pos_tag(all_words)
print("\n词性标注结果:", tagged_words)

词性标注结果: [('Marriage', 'NN'), ('is', 'VBZ'), ('a', 'DT'), ('big', 'JJ'), ('step', 'NN'), ('in', 'IN'), ('one', 'CD'), ("'s", 'POS'), ('life', 'NN'), ('.', '.'), ('Python', 'NNP'), ('is', 'VBZ'), ('a', 'DT'), ('popular', 'JJ'), ('programming', 'NN'), ('language', 'NN'), ('used', 'VBN'), ('for', 'IN'), ('various', 'JJ'), ('tasks', 'NNS'), (',', ','), ('including', 'VBG'), ('data', 'NNS'), ('analysis', 'NN'), ('and', 'CC'), ('web', 'NN'), ('development', 'NN'), ('.', '.'), ('NLTK', 'NNP'), ('provides', 'VBZ'), ('powerful', 'JJ'), ('tools', 'NNS'), ('for', 'IN'), ('natural', 'JJ'), ('language', 'NN'), ('processing', 'NN'), ('.', '.')]

# 4. 筛选名词
extracted_nouns = []
for word, tag in tagged_words:
    if tag.startswith('NN'):
        extracted_nouns.append(word)

print("\n提取出的名词:", extracted_nouns)

提取出的名词: ['Marriage', 'step', 'life', 'Python', 'programming', 'language', 'tasks', 'data', 'analysis', 'web', 'development', 'NLTK', 'tools', 'language', 'processing']

# 获取英文停用词列表
stop_words = set(stopwords.words('english'))

filtered_words = [w for w in all_words if w.lower() not in stop_words and w.isalnum()] # 过滤停用词和非字母数字字符
tagged_filtered_words = nltk.pos_tag(filtered_words)

extracted_nouns_filtered = []
for word, tag in tagged_filtered_words:
    if tag.startswith('NN'):
        extracted_nouns_filtered.append(word)

print("\n提取出的名词（过滤停用词后）:", extracted_nouns_filtered)

提取出的名词（过滤停用词后）: ['Marriage', 'step', 'life', 'Python', 'programming', 'language', 'tasks', 'data', 'analysis', 'web', 'development', 'NLTK', 'tools', 'language', 'processing']

import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords

# 确保NLTK数据已下载
try:
    nltk.data.find('tokenizers/punkt')
    nltk.data.find('taggers/averaged_perceptron_tagger')
    nltk.data.find('corpora/stopwords')
except nltk.downloader.DownloadError:
    nltk.download('punkt')
    nltk.download('averaged_perceptron_tagger')
    nltk.download('stopwords')

def extract_nouns_from_text(text: str, filter_stopwords: bool = True) -> list:
    """
    从给定文本中提取名词。

    Args:
        text (str): 输入的文本字符串。
        filter_stopwords (bool): 是否在提取前过滤停用词。默认为True。

    Returns:
        list: 包含所有提取出的名词的列表。
    """
    # 1. 文本分句
    sentences = sent_tokenize(text)

    # 2. 词语标记化
    all_words = []
    for sentence in sentences:
        words = word_tokenize(sentence)
        all_words.extend(words)

    # 3. 可选：过滤停用词和非字母数字字符
    if filter_stopwords:
        stop_words = set(stopwords.words('english'))
        processed_words = [
            word for word in all_words
            if word.lower() not in stop_words and word.isalnum()
        ]
    else:
        processed_words = [word for word in all_words if word.isalnum()] # 只保留字母数字字符

    if not processed_words:
        return []

    # 4. 词性标注
    tagged_words = nltk.pos_tag(processed_words)

    # 5. 筛选名词
    extracted_nouns = [word for word, tag in tagged_words if tag.startswith('NN')]

    return extracted_nouns

# 示例使用
response_message = "Marriage is a big step in one's life. Python is a popular programming language used for various tasks, including data analysis and web development. NLTK provides powerful tools for natural language processing. I want to extract nouns from this message."

# 提取名词（过滤停用词）
nouns_filtered = extract_nouns_from_text(response_message, filter_stopwords=True)
print("提取出的名词 (过滤停用词):", nouns_filtered)

# 提取名词（不过滤停用词）
nouns_unfiltered = extract_nouns_from_text(response_message, filter_stopwords=False)
print("提取出的名词 (不过滤停用词):", nouns_unfiltered)