Pandas DataFrame：高效利用文件名作为行索引构建文档-词矩阵

import pandas as pd
from pathlib import Path
from sklearn.feature_extraction.text import CountVectorizer
import re

# 假设文件存储在 'files' 目录下，请替换为您的实际路径
# 例如：'/Users/MyPath/files'
FILE_DIRECTORY = Path("./files") 

# 创建一个用于存储文件路径的列表
corpus = []

# 使用 pathlib 遍历指定目录下的所有 .txt 文件
for fichier in FILE_DIRECTORY.rglob("*.txt"):
    corpus.append(fichier) # 将 Path 对象直接添加到列表中

# 对文件路径进行排序，确保与后续处理的顺序一致
corpus.sort()

all_documents = []
for fichier_txt_path in corpus:
    with open(fichier_txt_path, 'r', encoding='utf-8') as f: # 建议指定编码
        fichier_txt_chaine = f.read()
        # 清理文本，只保留字母
        fichier_txt_chaine = re.sub('[^A-Za-z]', ' ', fichier_txt_chaine) 
    all_documents.append(fichier_txt_chaine)

# 使用 sklearn 构建文档-词矩阵
coun_vect = CountVectorizer(stop_words="english")
count_matrix = coun_vect.fit_transform(all_documents)
count_array = count_matrix.toarray()

# 方案一：将完整的 Path 对象作为索引
# pd.DataFrame(data=count_array, columns=coun_vect.get_feature_features(), index=corpus)
# 此时索引将是 Path 对象，如 PosixPath('files/1905.txt')

# ... (承接上述代码) ...

# 方案二：将文件名（含扩展名）作为索引
file_names = [f.name for f in corpus]
allDataframe_with_names = pd.DataFrame(data=count_array, 
                                       columns=coun_vect.get_feature_names_out(), # 推荐使用 get_feature_names_out()
                                       index=file_names)
print("DataFrame with filenames as index:")
print(allDataframe_with_names.head())

# ... (承接上述代码) ...

# 方案三：将文件名（不含扩展名）作为索引
file_stems = [f.stem for f in corpus]
allDataframe_with_stems = pd.DataFrame(data=count_array, 
                                       columns=coun_vect.get_feature_names_out(),
                                       index=file_stems)
print("\nDataFrame with file stems as index:")
print(allDataframe_with_stems.head())

# 将结果保存到CSV文件
allDataframe_with_stems.to_csv("Matrice_doc_term_with_filenames.csv", index=True) # index=True 确保索引也被写入CSV

import pandas as pd
from pathlib import Path
from sklearn.feature_extraction.text import CountVectorizer
import re
import os

# --- 模拟文件和目录创建 (仅为示例，实际使用时请替换为您的文件路径) ---
# 创建一个临时目录
temp_dir = Path("./temp_text_files")
temp_dir.mkdir(exist_ok=True)

# 创建模拟的 .txt 文件
file_contents = {
    "1905.txt": "Hello world, this is the year 1905. A new beginning.",
    "1906.txt": "I am here in 1906. The world is changing.",
    "1907.txt": "Am I still here? Yes, it's 1907. Life goes on.",
    "1908.txt": "Another year, another story. This is 1908.",
}

for filename, content in file_contents.items():
    (temp_dir / filename).write_text(content, encoding='utf-8')
# -----------------------------------------------------------------

# 指定文件目录为模拟创建的目录
FILE_DIRECTORY = temp_dir 

# 创建一个用于存储文件路径的列表
corpus = []

# 使用 pathlib 遍历指定目录下的所有 .txt 文件
for fichier in FILE_DIRECTORY.rglob("*.txt"):
    corpus.append(fichier)

# 对文件路径进行排序，确保与后续处理的顺序一致
corpus.sort()

all_documents = []
for fichier_txt_path in corpus:
    with open(fichier_txt_path, 'r', encoding='utf-8') as f:
        fichier_txt_chaine = f.read()
        # 清理文本，只保留字母
        fichier_txt_chaine = re.sub('[^A-Za-z]', ' ', fichier_txt_chaine) 
    all_documents.append(fichier_txt_chaine)

# 使用 sklearn 构建文档-词矩阵
coun_vect = CountVectorizer(stop_words="english")
count_matrix = coun_vect.fit_transform(all_documents)
count_array = count_matrix.toarray()

# 获取特征名（词汇）
feature_names = coun_vect.get_feature_names_out()

# 方案：将文件名（不含扩展名）作为索引
file_stems = [f.stem for f in corpus]
allDataframe = pd.DataFrame(data=count_array, 
                            columns=feature_names,
                            index=file_stems)

print("Final DataFrame with file stems as index:")
print(allDataframe)

# 将结果保存到CSV文件，并确保索引也被写入
allDataframe.to_csv("Matrice_doc_term_with_filenames.csv", index=True)

# --- 清理模拟文件和目录 (仅为示例) ---
for filename in file_contents.keys():
    (temp_dir / filename).unlink() # 删除文件
temp_dir.rmdir() # 删除目录
# ------------------------------------