优化Langchain RAG检索：提升文档匹配准确性

from langchain.document_loaders import PyPDFLoader, DirectoryLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Chroma
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.chains import RetrievalQA
from langchain.llms import OpenAI, HuggingFaceHub # 导入OpenAI和HuggingFaceHub

# 1. 文档加载
# 假设您的PDF文档位于指定路径
loader = PyPDFLoader("doc.pdf") # 或者 DirectoryLoader('/tmp/', glob="./*.pdf")
documents = loader.load()

# 2. 文本分割
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
texts = text_splitter.split_documents(documents)

# 3. 嵌入与向量存储 - 使用HuggingFace Embeddings
# 可以选择不同的模型，例如：
# "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" (多语言，轻量级)
# "bert-base-multilingual-cased" (多语言，更通用)
# "sentence-transformers/all-MiniLM-L6-v2" (英文，高性能)
embeddings = HuggingFaceEmbeddings(
    model_name="bert-base-multilingual-cased" # 选择一个合适的模型
)

persist_directory = "./chromadb_store" # 定义向量数据库的持久化路径
vectordb = Chroma.from_documents(documents=texts, embedding=embeddings, persist_directory=persist_directory)
vectordb.persist() # 持久化向量数据库，以便后续复用
print(f"向量数据库已创建并持久化到: {persist_directory}")

# 4. 集成LLM并构建QA链
# 使用OpenAI LLM作为示例
# 请确保您已设置OPENAI_API_KEY环境变量
llm_openai = OpenAI(temperature=0, model_name="text-davinci-003") # 或 "gpt-3.5-turbo" 等

qa_chain_openai = RetrievalQA.from_chain_type(
    llm=llm_openai,
    retriever=vectordb.as_retriever(),
    chain_type="stuff", # 常见的链类型，将所有检索到的文档填充到提示中
    return_source_documents=True
)

question = "您的问题是什么？"
response_openai = qa_chain_openai({"query": question})
print("\n--- OpenAI LLM 响应 ---")
print(response_openai)

# 也可以尝试使用HuggingFaceHub上的开源LLM
# 请确保您已设置HUGGINGFACEHUB_API_TOKEN环境变量
# llm_hf = HuggingFaceHub(
#     repo_id="google/flan-t5-base",
#     model_kwargs={"temperature": 0.6, "max_length": 500, "max_new_tokens": 200}
# )
# # 或者
# # llm_hf = HuggingFaceHub(repo_id="EleutherAI/gpt-neo-2.7B")

# qa_chain_hf = RetrievalQA.from_chain_type(
#     llm=llm_hf,
#     retriever=vectordb.as_retriever(),
#     chain_type="stuff",
#     return_source_documents=True
# )

# response_hf = qa_chain_hf({"query": question})
# print("\n--- HuggingFaceHub LLM 响应 ---")
# print(response_hf)