ARTICLE · 1154338
7python之pdf解析报错处理及本地test用例操作
7python之pdf解析报错处理及本地test用例操作
写在前面的话
存在即合理。今天在外面抽烟，一哥们来借火，唉声叹气的，我说怎么啦，这么郁闷。他说：节前和客户开会，说我项目开发的慢，说我技术不行，刚才领导打电话过来，同样规模的系统，客户自己的研发团队开发，两周就完事了，我这边开发两个月了，还一堆问题。哥们说当时他就顶了回去，说：让他们自己研发团队开发呀，还外包出去干啥。结果领导直接挂了电话，哥们想着怎么给领导说呢。我说：你这样做的确实欠考虑，你多大了。哥们说：85年的，今年41周岁，做开发一二十年了，首次被人怀疑技术。我说：应该稳重点呀，别那么大火气，首先人家客户有自己的开发团队，把项目外包出去，肯定有自己的考量，腾讯、阿里里面牛B程序员大把，也有些项目是外包出去的。不要怀疑自己的水平，多花点时间去做，或者自己有更挣钱的项目做，干脆把这个项目推掉，如果没有，就老老实实的做就完了，存在即合理，别想太多。哥们说：想想吧，一会给领导打电话好好说说，还是要生活的。
[81+200]-------->底部有张生活照片和昨日花销
【关键词】：python、ragflow、pdf解析、test用例
一、pdf解析报错处理（三级）
描述：现在pdf解析报错还是没处理完成，没找到原因，需要再研究一下。
开工：
第一步：手动研究(四级)
20250817周日时间段：19:30-21:00现在没启动任务，为啥能解析，看下为啥。
找到了原因：
background_thread = threading.Thread(target=DocumentService.update_progress)
background_thread.daemon = True
background_thread.start()
注：截图如下：
图7a-1
注：发现就是这三行是个死循环，一直在监控任务。
从表里拉数据，程序如下：
@classmethod
@DB.connection_context()
def get_unfinished_docs(cls):
fields = [cls.model.id,
cls.model.name,
cls.model.process_begin_at,
cls.model.parser_config,
cls.model.progress_msg,
cls.model.run,
cls.model.parser_id]
docs = cls.model.select(*fields) \
.where(
cls.model.status == StatusEnum.VALID.value,
~(cls.model.type == FileType.VIRTUAL.value),
cls.model.progress < 1,
cls.model.progress > 0)
return list(docs.dicts())
注：逐行解释下这个程序。
第二步：逐行解释（四级）
20250818周一时间段：11:10-13:00逐行解释如下：
图7a-2
注：接下来，看一下为啥报错。
找到几条数据：
INSERT INTO `rag_flow`.`document` (`id`, `create_time`, `create_date`, `update_time`, `update_date`, `thumbnail`, `kb_id`, `parser_id`, `parser_config`, `source_type`, `type`, `created_by`, `name`, `location`, `size`, `token_num`, `chunk_num`, `progress`, `progress_msg`, `process_begin_at`, `process_duation`, `run`, `status`, `meta_fields`) VALUES ('d6e442280a0b11f08889b2ca38881710', 1742970641483, '2025-03-26 14:30:41', 1755487039614, '2025-08-18 11:17:19', 'thumbnail_d6e442280a0b11f08889b2ca38881710.png', 'bf5b0a6eff1211ef94e1acde48001122', 'picture', '{\"auto_keywords\": 0, \"auto_questions\": 0, \"raptor\": {\"use_raptor\": false}, \"graphrag\": {\"use_graphrag\": true, \"entity_types\": [\"PET-SPECIES(\\u5ba0\\u7269\\u79cd\\u7c7b)\", \"BREED(\\u54c1\\u79cd)\", \"AGE(\\u5e74\\u9f84)\", \"GENDER(\\u6027\\u522b)\", \"WEIGHT(\\u4f53\\u91cd)\", \"TEMPERATURE(\\u4f53\\u6e29)\", \"DISEASE(\\u75be\\u75c5)\", \"SYMPTOM(\\u75c7\\u72b6)\", \"MEDICATION(\\u836f\\u7269)\", \"TREATMENT-METHOD(\\u6cbb\\u7597\\u65b9\\u6cd5)\", \"DIAGNOSTIC-TEST(\\u8bca\\u65ad\\u6d4b\\u8bd5)\", \"SIGN(\\u4f53\\u5f81)\", \"ORGAN-OR-SYSTEM(\\u5668\\u5b98\\u6216\\u7cfb\\u7edf)\", \"VACCINE(\\u75ab\\u82d7)\", \"ANIMAL-BEHAVIOR(\\u52a8\\u7269\\u884c\\u4e3a)\", \"ALLERGEN(\\u8fc7\\u654f\\u6e90)\", \"PROGNOSIS(\\u9884\\u540e)\", \"ENVIRONMENTAL-FACTORS(\\u73af\\u5883\\u56e0\\u7d20)\", \"NUTRITION(\\u8425\\u517b)\", \"FOOD(\\u98df\\u7269)\", \"WATER-INTAKE(\\u996e\\u6c34\\u60c5\\u51b5)\", \"LIFESTYLE(\\u751f\\u6d3b\\u4e60\\u60ef)\", \"ALLERGIC-REACTION(\\u8fc7\\u654f\\u53cd\\u5e94)\", \"LIVING-ENVIRONMENT(\\u5c45\\u4f4f\\u73af\\u5883)\", \"PARASITE(\\u5bc4\\u751f\\u866b)\", \"RESTRAINT-METHOD(\\u4fdd\\u5b9a\\u6cd5)\", \"EXAMINATION-METHOD(\\u68c0\\u67e5\\u65b9\\u6cd5)\", \"EPIDEMIOLOGY(\\u6d41\\u884c\\u75c5\\u5b66)\", \"LESION(\\u75c5\\u53d8)\", \"PREVENTION(\\u9884\\u9632\\u65b9\\u6cd5)\"], \"method\": \"general\", \"resolution\": true, \"community\": true}, \"chunk_token_num\": 1500, \"delimiter\": \"\\\\n!?;\\u3002\\uff1b\\uff01\\uff1f\", \"layout_recognize\": \"DeepDOC\", \"html4excel\": true}', 'local', 'visual', '79949d6a965d11efb03b0242ac120006', '4(5).jpg', '4(5).jpg', 31618, 0, 0, 0.00403236, 'Task is queued...', '2025-05-09 09:05:40', 8734300, '1', '1', '{}');INSERT INTO `rag_flow`.`document` (`id`, `create_time`, `create_date`, `update_time`, `update_date`, `thumbnail`, `kb_id`, `parser_id`, `parser_config`, `source_type`, `type`, `created_by`, `name`, `location`, `size`, `token_num`, `chunk_num`, `progress`, `progress_msg`, `process_begin_at`, `process_duation`, `run`, `status`, `meta_fields`) VALUES ('7668f0d82bbc11f09949528f0337fcc4', 1746674889067, '2025-05-08 11:28:09', 1755487039670, '2025-08-18 11:17:19', 'thumbnail_7668f0d82bbc11f09949528f0337fcc4.png', 'bf5b0a6eff1211ef94e1acde48001122', 'picture', '{\"auto_keywords\": 0, \"auto_questions\": 0, \"raptor\": {\"use_raptor\": false}, \"graphrag\": {\"use_graphrag\": true, \"entity_types\": [\"PET-SPECIES(\\u5ba0\\u7269\\u79cd\\u7c7b)\", \"BREED(\\u54c1\\u79cd)\", \"AGE(\\u5e74\\u9f84)\", \"GENDER(\\u6027\\u522b)\", \"WEIGHT(\\u4f53\\u91cd)\", \"TEMPERATURE(\\u4f53\\u6e29)\", \"DISEASE(\\u75be\\u75c5)\", \"SYMPTOM(\\u75c7\\u72b6)\", \"MEDICATION(\\u836f\\u7269)\", \"TREATMENT-METHOD(\\u6cbb\\u7597\\u65b9\\u6cd5)\", \"DIAGNOSTIC-TEST(\\u8bca\\u65ad\\u6d4b\\u8bd5)\", \"SIGN(\\u4f53\\u5f81)\", \"ORGAN-OR-SYSTEM(\\u5668\\u5b98\\u6216\\u7cfb\\u7edf)\", \"VACCINE(\\u75ab\\u82d7)\", \"ANIMAL-BEHAVIOR(\\u52a8\\u7269\\u884c\\u4e3a)\", \"ALLERGEN(\\u8fc7\\u654f\\u6e90)\", \"PROGNOSIS(\\u9884\\u540e)\", \"ENVIRONMENTAL-FACTORS(\\u73af\\u5883\\u56e0\\u7d20)\", \"NUTRITION(\\u8425\\u517b)\", \"FOOD(\\u98df\\u7269)\", \"WATER-INTAKE(\\u996e\\u6c34\\u60c5\\u51b5)\", \"LIFESTYLE(\\u751f\\u6d3b\\u4e60\\u60ef)\", \"ALLERGIC-REACTION(\\u8fc7\\u654f\\u53cd\\u5e94)\", \"LIVING-ENVIRONMENT(\\u5c45\\u4f4f\\u73af\\u5883)\", \"PARASITE(\\u5bc4\\u751f\\u866b)\", \"RESTRAINT-METHOD(\\u4fdd\\u5b9a\\u6cd5)\", \"EXAMINATION-METHOD(\\u68c0\\u67e5\\u65b9\\u6cd5)\", \"EPIDEMIOLOGY(\\u6d41\\u884c\\u75c5\\u5b66)\", \"LESION(\\u75c5\\u53d8)\", \"PREVENTION(\\u9884\\u9632\\u65b9\\u6cd5)\"], \"method\": \"general\", \"resolution\": true, \"community\": true}, \"chunk_token_num\": 1500, \"delimiter\": \"\\\\n!?;\\u3002\\uff1b\\uff01\\uff1f\", \"layout_recognize\": \"DeepDOC\", \"html4excel\": true}', 'local', 'visual', '2b2731bef8e711efb5c60242ac120006', '皮肤-螨虫-疥螨.png', '皮肤-螨虫-疥螨.png', 346255, 0, 0, 0.00456027, 'Task is queued...', '2025-05-09 09:21:31', 8733350, '1', '1', '{}');INSERT INTO `rag_flow`.`document` (`id`, `create_time`, `create_date`, `update_time`, `update_date`, `thumbnail`, `kb_id`, `parser_id`, `parser_config`, `source_type`, `type`, `created_by`, `name`, `location`, `size`, `token_num`, `chunk_num`, `progress`, `progress_msg`, `process_begin_at`, `process_duation`, `run`, `status`, `meta_fields`) VALUES ('11dd65620a0c11f0b769b2ca38881710', 1742970740435, '2025-03-26 14:32:20', 1755487039729, '2025-08-18 11:17:19', 'thumbnail_11dd65620a0c11f0b769b2ca38881710.png', 'bf5b0a6eff1211ef94e1acde48001122', 'picture', '{\"auto_keywords\": 0, \"auto_questions\": 0, \"raptor\": {\"use_raptor\": false}, \"graphrag\": {\"use_graphrag\": true, \"entity_types\": [\"PET-SPECIES(\\u5ba0\\u7269\\u79cd\\u7c7b)\", \"BREED(\\u54c1\\u79cd)\", \"AGE(\\u5e74\\u9f84)\", \"GENDER(\\u6027\\u522b)\", \"WEIGHT(\\u4f53\\u91cd)\", \"TEMPERATURE(\\u4f53\\u6e29)\", \"DISEASE(\\u75be\\u75c5)\", \"SYMPTOM(\\u75c7\\u72b6)\", \"MEDICATION(\\u836f\\u7269)\", \"TREATMENT-METHOD(\\u6cbb\\u7597\\u65b9\\u6cd5)\", \"DIAGNOSTIC-TEST(\\u8bca\\u65ad\\u6d4b\\u8bd5)\", \"SIGN(\\u4f53\\u5f81)\", \"ORGAN-OR-SYSTEM(\\u5668\\u5b98\\u6216\\u7cfb\\u7edf)\", \"VACCINE(\\u75ab\\u82d7)\", \"ANIMAL-BEHAVIOR(\\u52a8\\u7269\\u884c\\u4e3a)\", \"ALLERGEN(\\u8fc7\\u654f\\u6e90)\", \"PROGNOSIS(\\u9884\\u540e)\", \"ENVIRONMENTAL-FACTORS(\\u73af\\u5883\\u56e0\\u7d20)\", \"NUTRITION(\\u8425\\u517b)\", \"FOOD(\\u98df\\u7269)\", \"WATER-INTAKE(\\u996e\\u6c34\\u60c5\\u51b5)\", \"LIFESTYLE(\\u751f\\u6d3b\\u4e60\\u60ef)\", \"ALLERGIC-REACTION(\\u8fc7\\u654f\\u53cd\\u5e94)\", \"LIVING-ENVIRONMENT(\\u5c45\\u4f4f\\u73af\\u5883)\", \"PARASITE(\\u5bc4\\u751f\\u866b)\", \"RESTRAINT-METHOD(\\u4fdd\\u5b9a\\u6cd5)\", \"EXAMINATION-METHOD(\\u68c0\\u67e5\\u65b9\\u6cd5)\", \"EPIDEMIOLOGY(\\u6d41\\u884c\\u75c5\\u5b66)\", \"LESION(\\u75c5\\u53d8)\", \"PREVENTION(\\u9884\\u9632\\u65b9\\u6cd5)\"], \"method\": \"general\", \"resolution\": true, \"community\": true}, \"chunk_token_num\": 1500, \"delimiter\": \"\\\\n!?;\\u3002\\uff1b\\uff01\\uff1f\", \"layout_recognize\": \"DeepDOC\", \"html4excel\": true}', 'local', 'visual', '7d19a176807611efb0f80242ac120006', '角膜溃疡(1).jpg', '角膜溃疡(1).jpg_', 234443, 128, 0, 0.00878557, 'Task is queued...', '2025-05-09 09:05:33', 8734310, '1', '1', '{}');INSERT INTO `rag_flow`.`document` (`id`, `create_time`, `create_date`, `update_time`, `update_date`, `thumbnail`, `kb_id`, `parser_id`, `parser_config`, `source_type`, `type`, `created_by`, `name`, `location`, `size`, `token_num`, `chunk_num`, `progress`, `progress_msg`, `process_begin_at`, `process_duation`, `run`, `status`, `meta_fields`) VALUES ('9f8efb55ff1311efa42cce2d064a03c7', 1739269514592, '2025-02-11 18:25:14', 1741764521050, '2025-03-12 15:28:41', 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAB4AAAAXCAIAAACTXWg9AAAGdUlEQVR4nC2QSY8cZxmA3/dbaul9Ns/m8TJ2JpMEB1mGOJEjBW6IHDiALIRy4swNwYk7V/4DJyQQh4CUGyhghOzYJI5jZcb2TM/S3dPd1d1VXVVd9dW3vBySP/A8eh5cXb/hrLZaOauAXGXM7esbf/ndL1eWWs+6vS8G0elJ/7VLm3curTVCcZGW/xuMPz04O4mmmVJRnDpy5Mg5Z50jIuvIkquFwa8++gnuvf5upYqqyJTOrNFNT/zx1z+/987rz86mvTgOK3M5rG+ttiVHoyqtyyJVaUGHUfL3L19+/PRAa2OssdYBULPeqIWNrHDzTL9353tiTZ+XxmhuK2eWO83f/PSHH7z/nXmlna3ebNW2Vzpew3fOqUlSzNN0mjqidli7e6l9/e4bSTo/ynLrcDzLUfDv710bZvzp4ZhLevDMCk+yG1e2PEGc8P67t3/w3ltKWZrl+/UgWGnRUscu8rI/nEXpyWDyZfes0axx5xrEbm9s39+//ofHz50gjvrmle2396/++dOec2Vt6x4Xe8IFzV/c//Hdt6+m3Yt6YbK4AqgCn+Ra27WXIMtUf5hO036U/PPl0Xmur7Vbni+LZLE4Oru5uvLWcvuTg1eVtuNB/yFj0SjhsrVIcq8zF7//7Ue7WyvxUV9O0vrGEg85hgKC0EpBZ2fxIOqN4oPB+CBObODl42xRFtIXcwd/7Q12J5MWt1WlgRgZHU8m6zVzPlNl+dKWsbi9t9N7+sJP87XVmmgIW6vZTpsZTd3T0cngcDjtGXZiWePyzjxJ+lHSbHvA6aQ/eTmYLFbqeWE2VzuLspoXqtXUrYaEOGJgNlaXxfnj5x0Gy5tNkGgFI1+gKuFiFJ1PXs2K5q3v3rq0Xnz2RT5NqKJ23ev2pnGuvj6JhtO5J2GWVZ4nDFkDhEjcD7SjK5v1+x/eEZcbvlfzQYKTHnmSaQ1pHJ+NjydpZ3/vjQ/e/+rpVzebjVpYK3Y2t1aXPnnw5F+fd0OfbyyFUjBjrNZVrgwHaNX9wXyBjEeTcRJfCFnzSHBqeOQIlMFKqSjvxQWurK7f3NWLwpvNblinS8W2Nj3uT+P4IpkaY+uNIEqKrNSBQCIMfL623IwWKSJYR//472MBzRA4EkcoFKsMVW4yy0fGXH9zr7m8NIvGJk9NKGWr4/EgoJIHcv/q2jRVaWGyohKc1QJZ6Jwzpp1zgIiMMehdRALCgIhAlVAZJFfm1XyxuLS7c+XWDSHDeZ6f5mUg/RArH0Exq1QlGfMln6UVIat5slHzZunCk3I2LydxyYABAiII8HynNVqHQOCIwC2vdIrVDm+FwPyw01KefzaMJBeSCZ9jVhQEaKwttQYCX7J64AnBfSkn8ypXijEkAiAQZAkAERGMcY68sNb2UQQBeiEyT/gBIXZPT/zAj5PsYpgsykUt4L4vPcmE4PXQrwxxLoLAm+UlICEiOCBAAUDIkIQAT4IG5IGsdB25mWQs8JJoJv3QQnB4eFwBPzgaMoBGXexud9ZaIQEvlNHWWII0r/JSEyACAnwzhCECARJ6ApCBQeISo7j770eLRkP6Iokmu5sbxXD0qj9uEL220gp9v8VDb0kEvHx0PLTOOnLTvNCOIeMElgCBUIAjIAIpARmgRQ8AuWdpK86Go/RCdPa5u3xz/Ucb954cHr/o9vavXeaM11mrFXjP4+FRFJ+P546Ac+YIGDkiAkAiEAT0bYFggOAsIQIEvFFfmo7U0Yv5z/avhL7UZbm13e73tuZFMV8oqf0kTh8eHFdKt+v+NFMEgAjOfYsGAgEESBaqisiCRXIWABCRBLY31t/Z2cT8wKFFcJKc54lknJ0Np49eHv7n6PQomXMuhRRAoCqDjBGBc44jAwABRQWkUSlSBrUm55zTgOgQoip90O8ub+TNgJVFVZbmtD969HX346enA6V7ccaI1QQuSmWcIwLOBSIiohCCAQqeZ2CtU5UtlTNWV1pbXVS6sjax7tXngwdbtevLjVlRxFnx2eHZn550L/Ky4fFQiFLbUmttLCAQoXUVAjLGrLEAKFQ0LStX6CpblOfR7HicTDNVOOsYGs4nZfG3w0ldiotxMorTwVxNla573JOiLJS1zpBljBORA0IABCQi4wwAimeveuOi6o6jhy+Gx7P5tDCe7+9sr3Ekhhj4whfeKF00lloF0skk5YBhICfpojIOERwwIiAiAHAAgA6/UQD+H5+sLDsONvGcAAAAAElFTkSuQmCC', 'bf5b0a6eff1211ef94e1acde48001122', 'naive', '{\"auto_keywords\": 0, \"auto_questions\": 0, \"raptor\": {\"use_raptor\": false}, \"graphrag\": {\"use_graphrag\": true, \"entity_types\": [\"PET-SPECIES(宠物种类)\", \"BREED(品种)\", \"AGE(年龄)\", \"GENDER(性别)\", \"WEIGHT(体重)\", \"TEMPERATURE(体温)\", \"DISEASE(疾病)\", \"SYMPTOM(症状)\", \"MEDICATION(药物)\", \"TREATMENT-METHOD(治疗方法)\", \"DIAGNOSTIC-TEST(诊断测试)\", \"SIGN(体征)\", \"ORGAN-OR-SYSTEM(器官或系统)\", \"VACCINE(疫苗)\", \"ANIMAL-BEHAVIOR(动物行为)\", \"ALLERGEN(过敏源)\", \"PROGNOSIS(预后)\", \"ENVIRONMENTAL-FACTORS(环境因素)\", \"NUTRITION(营养)\", \"FOOD(食物)\", \"WATER-INTAKE(饮水情况)\", \"LIFESTYLE(生活习惯)\", \"ALLERGIC-REACTION(过敏反应)\", \"LIVING-ENVIRONMENT(居住环境)\", \"PARASITE(寄生虫)\", \"RESTRAINT-METHOD(保定法)\", \"EXAMINATION-METHOD(检查方法)\", \"EPIDEMIOLOGY(流行病学)\", \"LESION(病变)\", \"PREVENTION(预防方法)\"], \"method\": \"general\", \"resolution\": true, \"community\": true}, \"chunk_token_num\": 128, \"delimiter\": \"\\n!?;。；！？\", \"layout_recognize\": \"DeepDOC\", \"html4excel\": true}', 'local', 'visual', '7d19a176807611efb0f80242ac120006', '皮肤-真菌-猫癣1.png', '皮肤-真菌-猫癣1.png', 141625, 0, 0, 0.00972158, 'Task dispatched...', '2025-02-28 19:11:54', 1023410, '1', '1', NULL);
注：截图如下：
图7a-3
注：sql语句如下：
SELECT * from document where `status` =1 and type != 'virtual' and progress <1 and progress>0;注：现在把ragflow_server.py中的死循环注释掉了，应该没有更新task的地方了，截图如下：
图7a-4
注：也就是说，没有再操作任务的程序了，服务器层面。只剩个本地test用例操作了。
第三步：本地test用例操作（四级）
20250818周一时间段：15:13-17:00看一下，是哪里出问题了，现在还是报错，截图如下：
注：真不知道是哪里写进去的，相关代码如下：
#筛选出未完成的任务
unfinished_task_array = [task for task in parse_task_array if task["progress"] < 1.0]
# 将未完成的任务加入Redis队列
for unfinished_task in unfinished_task_array:
assert REDIS_CONN.queue_product(
SVR_QUEUE_NAME, message=unfinished_task
), "Can't access Redis. Please check the Redis' status."
注：这里面也没有写库的操作呀。
二、生活照片（三级）
拍摄于2023年10月2日，10:12:15，媳妇带二宝参加她同事婚礼拍的，当时二宝一岁。其实，编程而言，确实有牛人，有庸才，但既然这个项目让这哥们来做，说明他是能做成这个项目的，要不然，领导也不会给他。而至于说这个项目客户那边为啥两周能开发完的，偏要外包出去，两个月还没结果。那就不知道了，可能客户夸大了自己团队的技术水平，也可能是哥们领导关系比较硬，拿下了项目，但不管咋说，既然项目让哥们开发，领导只是催催进度，哥们就放平心态，好好开发就行了，别的不要多想，作为一个程序员，思想还是简单点，接项目是领导的事，领导不喊停，自己好好开发就行了，别的想它干啥，更不要说让客户团队自己开发的气话，没有意义。
图7b-1
《本文完》