PageIndex/├── run_pageindex.py          # CLI 入口（PDF/Markdown 双模式）├── requirements.txt          # 依赖声明├── pageindex/│   ├── __init__.py           # 包导出聚合│  ├──client.py # PageIndexClient 客户端封装│   ├── page_index.py         # PDF 索引引擎（核心，约 900 行）│   ├── page_index_md.py      # Markdown 索引引擎（约 300 行）│   ├── retrieve.py           # 检索接口（get_document / get_document_structure / get_page_content）│  ├──utils.py # 工具函数库（LLM 调用、树操作、JSON 提取等）│   └── config.yaml           # 默认配置├── examples/│   ├── agentic_vectorless_rag_demo.py  # Agentic RAG 完整示例│   └── documents/            # 示例 PDF 文档集├── cookbook/ # Jupyter Notebook 教程└── .github/ # CI/CD 与 Issue 管理

run_pageindex.py  ├── pageindex.page_index (PDF)  ├── pageindex.page_index_md (Markdown)  └── pageindex.utils (ConfigLoader)pageindex.client (PageIndexClient)  ├── pageindex.page_index  ├── pageindex.page_index_md  ├── pageindex.retrieve  └── pageindex.utilspageindex.retrieve  ├── PyPDF2 (PDF 回退读取)  └── pageindex.utils (remove_fields)

from .page_index import *from .page_index_md import md_to_treefrom .retrieve import get_document, get_document_structure, get_page_contentfrom .client import PageIndexClient

if __name__ == "__main__":    parser = argparse.ArgumentParser(description='Process PDF or Markdown document')    parser.add_argument('--pdf_path', type=str)    parser.add_argument('--md_path', type=str)    parser.add_argument('--model', type=str, default=None)# ... 其他参数定义 ...    args = parser.parse_args()# PDF 模式if args.pdf_path:        user_opt = {'model': args.model,'toc_check_page_num': args.toc_check_pages,'max_page_num_each_node': args.max_pages_per_node,'max_token_num_each_node': args.max_tokens_per_node,'if_add_node_id': args.if_add_node_id,'if_add_node_summary': args.if_add_node_summary,'if_add_doc_description': args.if_add_doc_description,'if_add_node_text': args.if_add_node_text,        }   opt=ConfigLoader().load({k:vfork,vinuser_opt.items()ifvisnotNone})        toc_with_page_number = page_index_main(args.pdf_path, opt)# 保存到 ./results/{pdf_name}_structure.json# Markdown 模式elif args.md_path:        toc_with_page_number = asyncio.run(md_to_tree(            md_path=args.md_path,            if_thinning=args.if_thinning.lower() == 'yes',# ... 其他参数 ...        ))

classPageIndexClient:def__init__(self, api_key=None, model=None, retrieve_model=None, workspace=None):if api_key:            os.environ["OPENAI_API_KEY"] = api_keyelifnot os.getenv("OPENAI_API_KEY") and os.getenv("CHATGPT_API_KEY"):            os.environ["OPENAI_API_KEY"] = os.getenv("CHATGPT_API_KEY")        self.workspace = Path(workspace).expanduser() if workspace elseNone        overrides = {}if model:            overrides["model"] = modelif retrieve_model:            overrides["retrieve_model"] = retrieve_model        opt = ConfigLoader().load(overrides orNone)        self.model = opt.model        self.retrieve_model = _normalize_retrieve_model(opt.retrieve_model or self.model)

defindex(self, file_path: str, mode: str = "auto") -> str:    file_path = os.path.abspath(os.path.expanduser(file_path))    doc_id = str(uuid.uuid4())if mode == "pdf"or (mode == "auto"and ext == '.pdf'):        result = page_index(doc=file_path, model=self.model, ...)# 缓存每页文本，避免检索时重读 PDF        pages = []withopen(file_path, 'rb') as f:            pdf_reader = PyPDF2.PdfReader(f)fori,pageinenumerate(pdf_reader.pages,1):   pages.append({'page':i,'content':page.extract_text()or''})        self.documents[doc_id] = {..., 'structure': result['structure'], 'pages': pages}elif mode == "md":        result = asyncio.run(coro)  # 或在线程池中运行        self.documents[doc_id] = {..., 'structure': result['structure']}if self.workspace:        self._save_doc(doc_id)return doc_id

workspace/├── _meta.json            # 轻量注册表，仅存元数据（名称、路径、页数）└── {doc_id}.json# 完整文档 JSON（结构 + 页面内容）

defpage_index_main(pdf_path, opt):    pdf_name = get_pdf_name(pdf_path)    page_list = get_page_tokens(pdf_path, model=opt.model)# 1. 提取目录    toc_of_pdf = extract_toc(pdf_path)# 2. 目录条目验证（LLM 模糊匹配章节起始页）    toc_in_list = check_title_and_convert(toc_of_pdf, page_list, start_index=1, model=opt.model)# 3. 添加前言节点（如果第一章不在第 1 页）    toc_in_list = add_preface_if_needed(toc_in_list)# 4. 构建层级树    tree = post_processing(toc_in_list, end_physical_index=len(page_list))# 5. 添加节点文本内容    add_node_text_with_labels(tree, page_list)# 6. 生成节点摘要（异步并行）    tree = asyncio.run(generate_summaries_for_structure(tree, model=opt.model))# 7. 生成文档描述    doc_description = generate_doc_description(clean_structure, model=opt.model)return {'doc_name': pdf_name, 'doc_description': doc_description, 'structure': tree}

defget_page_tokens(pdf_path, model=None, pdf_parser="PyPDF2"):if pdf_parser == "PyMuPDF":        doc = pymupdf.open(pdf_path)for page in doc:            page_text = page.get_text()            token_length = litellm.token_counter(model=model, text=page_text)            page_list.append((page_text, token_length))elif pdf_parser == "PyPDF2":# ...

asyncdefcheck_title_appearance(item, page_list, start_index=1, model=None):    title = item['title']    page_number = item['physical_index']    page_text = page_list[page_number-start_index][0]    prompt = f"""   Yourjobistocheckifthegivensectionappearsorstartsinthegiven page_text.    Note: do fuzzy matching, ignore any space inconsistency in the page_text.    The given section title is {title}.    Reply format: {{"thinking": "...", "answer": "yes or no"}}    """    response = await llm_acompletion(model=model, prompt=prompt)

defpost_processing(structure, end_physical_index):for i, item inenumerate(structure):        item['start_index'] = item.get('physical_index')if i < len(structure) - 1:ifstructure[i+1].get('appear_start')=='yes':   item['end_index']=structure[i+1]['physical_index']- 1else:   item['end_index']=structure[i+1]['physical_index']else:            item['end_index'] = end_physical_index    tree = list_to_tree(structure)return tree

asyncdefgenerate_summaries_for_structure(structure, model=None):    nodes = structure_to_list(structure)  # 树展平为列表    tasks = [generate_node_summary(node, model=model) for node in nodes]    summaries = await asyncio.gather(*tasks)  # 并发执行所有LLM调用for node, summary inzip(nodes, summaries):        node['summary'] = summaryreturn structure

asyncdefmd_to_tree(md_path, if_thinning=False, ...):withopen(md_path, 'r', encoding='utf-8') as f:        markdown_content = f.read()# 1. 正则提取标题节点    node_list, markdown_lines = extract_nodes_from_markdown(markdown_content)# 2. 提取每个节点的文本内容    nodes_with_content = extract_node_text_content(node_list, markdown_lines)# 3. 可选：树瘦身（合并过小节点）if if_thinning:        nodes_with_content = tree_thinning_for_index(nodes_with_content, ...)# 4. 构建层级树    tree_structure = build_tree_from_nodes(nodes_with_content)# 5. 可选：生成摘要if if_add_node_summary == 'yes':        tree_structure = await generate_summaries_for_structure_md(tree_structure, ...)return {'doc_name': ..., 'line_count': ..., 'structure': tree_structure}

defextract_nodes_from_markdown(markdown_content):    header_pattern = r'^(#{1,6})\s+(.+)$'    code_block_pattern = r'^```'    in_code_block = Falsefor line_num, line inenumerate(lines, 1):if re.match(code_block_pattern, stripped_line):            in_code_block = not in_code_blockcontinueifnot in_code_block:match = re.match(header_pattern, stripped_line)ifmatch:   node_list.append({'node_title':...,'line_num': line_num})

deftree_thinning_for_index(node_list, min_node_token=None, model=None):for i inrange(len(result_list) - 1, -1, -1):        total_tokens = current_node.get('text_token_count', 0)if total_tokens < min_node_token:   children_indices=find_all_children(i,current_level, result_list)# 合并子节点文本到父节点   merged_text=parent_text+'\n\n'+ child_text            result_list[i]['text'] = merged_text# 标记子节点为待删除            nodes_to_remove.update(children_indices)

defget_document(documents: dict, doc_id: str) -> str:    doc_info = documents.get(doc_id)    result = {'doc_id': doc_id,'doc_name': doc_info.get('doc_name', ''),'doc_description': doc_info.get('doc_description', ''),'type': doc_info.get('type', ''),'status': 'completed',    }if doc_info.get('type') == 'pdf':        result['page_count'] = _count_pages(doc_info)else:        result['line_count'] = doc_info.get('line_count', 0)return json.dumps(result)

defget_document_structure(documents: dict, doc_id: str) -> str:    structure = doc_info.get('structure', [])    structure_no_text = remove_fields(structure, fields=['text'])return json.dumps(structure_no_text, ensure_ascii=False)

def_parse_pages(pages: str) -> list[int]:for part in pages.split(','):if'-'in part:            start, end = int(...)            result.extend(range(start, end + 1))else:            result.append(int(part))returnsorted(set(result))

defllm_completion(model, prompt, chat_history=None, return_finish_reason=False):    model = model.removeprefix("litellm/")    max_retries = 10for i inrange(max_retries):try:   response=litellm.completion(model=model,messages=messages,temperature=0)return response.choices[0].message.contentexcept Exception as e:ifi<max_retries-1:                time.sleep(1)else:return""

defextract_json(content):# 提取 ```json ... ``` 代码块    start_idx = content.find("```json")if start_idx != -1:        json_content = content[start_idx+7:content.rfind("```")].strip()# 兼容 Python None → JSON null    json_content = json_content.replace('None', 'null')# 移除尾随逗号    json_content = json_content.replace(',]', ']').replace(',}', '}')return json.loads(json_content)

model:"gpt-4o-2024-11-20"retrieve_model:"gpt-5.4"# 当前仓库默认值（非公开模型代号），实际使用时若未设置则回退为 model 的值toc_check_page_num:20max_page_num_each_node:10max_token_num_each_node:20000if_add_node_id:"yes"if_add_node_summary:"yes"if_add_doc_description:"no"if_add_node_text:"no"

classConfigLoader:defload(self, user_opt=None) -> config:if user_opt isNone:            user_dict = {}        self._validate_keys(user_dict)  # 拒绝未知配置键        merged = {**self._default_dict, **user_dict}return config(**merged)

from agents import function_toolfrom pageindex.retrieve import get_document, get_document_structure, get_page_content@function_tooldeftool_get_document(doc_id: str) -> str:"""获取文档元数据：名称、描述、页数"""return get_document(client.documents, doc_id)@function_tooldeftool_get_structure(doc_id: str) -> str:"""获取文档层级树结构（摘要，不含全文）"""return get_document_structure(client.documents, doc_id)@function_tooldeftool_get_content(doc_id: str, pages: str) -> str:"""获取指定页面内容，pages支持"5-7"或"3,8"格式"""return get_page_content(client.documents, doc_id, pages)