你的 RAG 为什么检索不准?八成是文档切分和 Embedding 没做好

# PDF 解析示例
import fitz  # PyMuPDF

defparse_pdf(file_path: str) -> list[dict]:
"""解析 PDF，返回页面列表"""
    doc = fitz.open(file_path)
    pages = []
    
    for page_num in range(len(doc)):
        page = doc[page_num]
        text = page.get_text()
        pages.append({
            "page": page_num + 1,
            "content": text,
            "metadata": {"source": file_path, "page": page_num + 1}
        })
    
    return pages

# 固定长度切分
deffixed_length_split(text: str, chunk_size: int = 500, overlap: int = 50) -> list[str]:
"""按固定长度切分，带重叠"""
    chunks = []
    start = 0
    
    while start < len(text):
        end = start + chunk_size
        chunk = text[start:end]
        chunks.append(chunk)
        start = end - overlap  # 重叠部分
    
    return chunks

# 语义边界切分
defsemantic_split(text: str, max_chunk_size: int = 1000) -> list[str]:
"""按语义边界切分（段落、标题）"""
    import re
    
    # 先按标题切分
    sections = re.split(r'\n(?=#{1,3} )', text)
    
    chunks = []
    for section in sections:
        section = section.strip()
        if not section:
            continue
        
        # 如果 section 太大，继续按段落切分
        if len(section) > max_chunk_size:
            paragraphs = section.split('\n\n')
            for para in paragraphs:
                if len(para) > max_chunk_size:
                    # 仍然太大，用固定长度切分
                    chunks.extend(fixed_length_split(para, max_chunk_size))
                else:
                    chunks.append(para)
        else:
            chunks.append(section)
    
    return chunks

# 递归切分（LangChain 风格）
from langchain.text_splitter import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    separators=["\n\n", "\n", "。", "！", "？", "，", " "]
)

chunks = splitter.split_text(document)

# OpenAI Embedding 示例
from openai import OpenAI

client = OpenAI()

defget_embedding(text: str) -> list[float]:
"""获取文本的 Embedding 向量"""
    response = client.embeddings.create(
        model="text-embedding-3-small",
        input=text
    )
    return response.data[0].embedding

# Chroma 示例
import chromadb

# 创建客户端
client = chromadb.Client()

# 创建集合
collection = client.create_collection("documents")

# 添加文档
collection.add(
    documents=["文档1内容", "文档2内容"],
    ids=["doc1", "doc2"],
    metadatas=[{"source": "file1.pdf"}, {"source": "file2.pdf"}]
)

# 检索
results = collection.query(
    query_texts=["查询问题"],
    n_results=3
)

# RAG 索引流程
from openai import OpenAI
import chromadb

defbuild_rag_index(documents: list[dict], collection_name: str = "documents"):
"""构建 RAG 索引"""
    
    # 初始化
    openai_client = OpenAI()
    chroma_client = chromadb.Client()
    collection = chroma_client.create_collection(collection_name)
    
    # 处理每个文档
    for doc in documents:
        # 1. 解析文档（假设已经是纯文本）
        text = doc["content"]
        metadata = doc.get("metadata", {})
        
        # 2. 切分
        chunks = semantic_split(text, max_chunk_size=1000)
        
        # 3. 向量化并存储
        for i, chunk in enumerate(chunks):
            # 获取 Embedding
            response = openai_client.embeddings.create(
                model="text-embedding-3-small",
                input=chunk
            )
            embedding = response.data[0].embedding
            
            # 存入 Chroma
            collection.add(
                documents=[chunk],
                embeddings=[embedding],
                ids=[f"{metadata.get('source', 'unknown')}_{i}"],
                metadatas=[{**metadata, "chunk_index": i}]
            )
    
    return collection