Python中如何处理自然语言？

import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords

# 下载必要的资源
nltk.download('punkt')
nltk.download('stopwords')

# 示例文本
text = "Python is an excellent language for natural language processing."

# 词语切分
tokens = word_tokenize(text)

# 去除停用词
stop_words = set(stopwords.words('english'))
filtered_tokens = [word for word in tokens if word.lower() not in stop_words]

print("原始词语:", tokens)
print("去除停用词后的词语:", filtered_tokens)

import jieba

text = "Python是一种优秀的自然语言处理语言。"

# 中文分词
tokens = list(jieba.cut(text))

print("中文分词结果:", tokens)

from gensim.models import Word2Vec

# 示例句子
sentences = [
    ['Python', 'is', 'an', 'excellent', 'language', 'for', 'natural', 'language', 'processing'],
    ['NLP', 'is', 'a', 'field', 'of', 'AI', 'that', 'deals', 'with', 'human', 'language']
]

# 训练词向量模型
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)

# 查找与"language"最相似的词语
similar_words = model.wv.most_similar('language', topn=3)
print("与'language'最相似的词语:", similar_words)

from multiprocessing import Pool
import nltk
from nltk.tokenize import word_tokenize

# 下载必要的资源
nltk.download('punkt')

def tokenize_text(text):
    return word_tokenize(text)

texts = [
    "Python is an excellent language for natural language processing.",
    "NLP is a field of AI that deals with human language.",
    "Machine learning is used in many NLP tasks."
]

# 使用多进程进行词语切分
with Pool(processes=4) as pool:
    tokenized_texts = pool.map(tokenize_text, texts)

for text, tokens in zip(texts, tokenized_texts):
    print(f"文本: {text}")
    print(f"词语切分结果: {tokens}")