【码途手记-AI】第13章:文档切分Chunk怎么做:标题、段落、PDF乱码、长文本截断处理

# ====================================== 1、Chunk-递归字符切分Demo ======================================from FileReadTools import read_text_from_filefrom CleanTextTool import clean_textfrom langchain_text_splitters  import RecursiveCharacterTextSplitterfrom langchain_core.documents import Documentfrom pathlib import Path #引入文件目录处理组件# 1. 读取本地文档（可替换pdf/docx/txt文件路径）desktop = Path.home() / "Desktop"file_path = desktop / "Python【基础】.docx"raw_content = read_text_from_file(file_path)# 2. 复用统一清洗工具，解决PDF乱码、特殊符号、冗余空行clean_content = clean_text(raw_content)# 3. 初始化递归切分器（中文适配分隔符）CHUNK_SIZE = 500 # 单个分片最大字符长度CHUNK_OVERLAP = 120 # 分片重叠字符，保留跨段上下文SEPARATORS = ["\n\n", "\n", "。", "，", " ", ""] # 语义分割优先级splitter = RecursiveCharacterTextSplitter(    separators=SEPARATORS,    chunk_size=CHUNK_SIZE,    chunk_overlap=CHUNK_OVERLAP,    length_function=len,    is_separator_regex=False)# 4. 封装文档对象，执行切分doc = Document(page_content=clean_content, metadata={"source": "Python【基础】.docx"})chunk_list = splitter.split_documents([doc])# 5. 分片结果校验for idx, chunk in enumerate(chunk_list):    print()    print(f"=====分片{idx+1} 字符长度：{len(chunk.page_content)}=====")    print(chunk.page_content)

# ====================================== 2、Chunk-Markdown标题层级切分Demo ======================================from FileReadTools import read_text_from_filefrom CleanTextTool import clean_textfrom langchain_text_splitters import MarkdownHeaderTextSplitter, RecursiveCharacterTextSplitterfrom pathlib import Path #引入文件目录处理组件# 1. 读取并清洗md文档desktop = Path.home() / "Desktop"file_path = desktop / "Python【基础】.md"raw_content = read_text_from_file(file_path)clean_content = clean_text(raw_content)# 2. 定义识别标题层级headers_config = [    ("#", "一级标题"),    ("##", "二级标题"),    ("###", "三级标题")]# 标题切分器header_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_config)# 先按标题做大粒度分割header_chunks = header_splitter.split_text(clean_content)# 3. 二次递归切分兜底，避免单个标题下内容超长溢出size_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)final_chunk_list = size_splitter.split_documents(header_chunks)# 4. 分片校验for idx, chunk in enumerate(final_chunk_list):    print()    print(f"分片{idx+1} 元数据标题：{chunk.metadata}")    print(chunk.page_content)

# ====================================== 3、Chunk-段落优先切分Demo ======================================from FileReadTools import read_text_from_filefrom CleanTextTool import clean_textfrom langchain_text_splitters import RecursiveCharacterTextSplitterfrom pathlib import Path #引入文件目录处理组件from langchain_core.documents import Document #引入文件对象处理组件# 1. 读取本地文档（可替换pdf/docx/txt文件路径）desktop = Path.home() / "Desktop"file_path = desktop / "Python【基础】.docx"raw_content = read_text_from_file(file_path)clean_txt = clean_text(raw_content)# 2. 段落优先：第一优先级只按完整段落分割para_splitter = RecursiveCharacterTextSplitter(    separators=["\n\n", "\n", "。"],    chunk_size=500, # 单个分片最大字符长度    chunk_overlap=150 # 分片重叠字符，保留跨段上下文)doc = Document(page_content=clean_txt, metadata={"source": "Python【基础】.docx"})chunks = para_splitter.split_documents([doc]) # 注意这里传入的是列表# 3. 分片校验print(f"共切分出 {len(chunks)} 个片段\n")for idx, chunk in enumerate(chunks):    print(f"=====分片{idx+1} 字符长度：{len(chunk.page_content)}=====")    print(f"来源: {chunk.metadata.get('source', '未知')}")    # 打印内容（去除首尾多余空格，更整洁）    print(chunk.page_content.strip())    print()

# ====================================== Chunk 语义切分调用 Demo ======================================# pip install langchain-experimental langchain-openaifrom pathlib import Pathimport osfrom CleanTextTool import clean_textfrom FileReadTools import read_text_from_filefrom langchain_core.documents import Documentfrom langchain_openai import OpenAIEmbeddingsfrom ChunkTools import create_semantic_splitter, semantic_chunk# 1. 读取文档desktop = Path.home() / "Desktop"file_path = desktop / "客服聊天记录.txt"raw_content = read_text_from_file(file_path)raw_content = clean_text(raw_content)# 2. 初始化 Embedding 模型embeddings = OpenAIEmbeddings(    model="text-embedding-3-large",    api_key=os.getenv("LLM__API_KEY"),    base_url=os.getenv("LLM__BASE_URL"))# 3. 创建语义切分器splitter = create_semantic_splitter(    embeddings=embeddings,    breakpoint_threshold_type="percentile",    breakpoint_threshold_amount=96,)# 4. 执行切分text_chunks = semantic_chunk(    text=raw_content,    splitter=splitter,    max_chars=900,    min_chunk_chars=80,    min_lines_before_split=4,)# 5. 将字符串分片封装为Document对象，方便入库doc_chunks = [    Document(        page_content=chunk,        metadata={            "source": file_path.name,            "chunk_index": idx + 1,            "chunk_length": len(chunk),        },    )    for idx, chunk in enumerate(text_chunks)]# 6. 校验输出print(f"最终共生成 {len(doc_chunks)} 个片段\n")for idx, doc in enumerate(doc_chunks):    print(f"===== 片段 {idx + 1}，长度 {len(doc.page_content)} 字符 =====")    print(doc.page_content.strip())    print()