"""正文提取服务(document_content_service)纯单元测试。 不依赖数据库与请求上下文: - _merge_wrapped_lines:PDF 段内软换行合并启发式(纯文本兜底); - _merge_line_items:块内合并规则(含字号突变切分); - sanitize_extract_error:失败原因脱敏(去除绝对路径); - extract_document_content:UNSUPPORTED/FAILED 分支、PDF 版面块分段与退化回退。 """ from __future__ import annotations from pathlib import Path import fitz from dms.common.enums import ContentExtractStatus from dms.services.document_content_service import ( _merge_line_items, _merge_wrapped_lines, extract_document_content, sanitize_extract_error, ) def test_merge_chinese_wrapped_line_without_space(): text = "这是一段在段内被换行\n打断的中文正文,合并后不应有空格" assert _merge_wrapped_lines(text) == "这是一段在段内被换行打断的中文正文,合并后不应有空格" def test_merge_english_words_with_space(): text = "emergency response\nplan continues here" assert _merge_wrapped_lines(text) == "emergency response plan continues here" def test_sentence_end_punctuation_keeps_paragraph_break(): text = "第一条意见如下。\n第二条意见如下:\n具体内容见附件。" assert _merge_wrapped_lines(text) == "第一条意见如下。\n第二条意见如下:\n具体内容见附件。" def test_blank_line_keeps_paragraph_break(): text = "第一段未结束\n仍在同一段\n\n第二段开始" assert _merge_wrapped_lines(text) == "第一段未结束仍在同一段\n第二段开始" def test_list_marker_start_keeps_paragraph_break(): text = "总体要求如下\n1. 第一条\n(一)细分条款\n一、中文序号\n第2章范围" assert _merge_wrapped_lines(text) == "总体要求如下\n1. 第一条\n(一)细分条款\n一、中文序号\n第2章范围" def test_mixed_cjk_and_ascii_merge_directly(): # 一侧为中文时不补空格 text = "使用 PDF\n格式存储文件" assert _merge_wrapped_lines(text) == "使用 PDF格式存储文件" def test_sanitize_error_strips_windows_path(): exc = RuntimeError("cannot open D:/dms/storage/uploads/a/b.pdf: permission denied") sanitized = sanitize_extract_error(exc) assert sanitized.startswith("RuntimeError:") assert "D:/" not in sanitized assert "" in sanitized def test_sanitize_error_strips_posix_path_and_truncates(): exc = ValueError("failed at /var/lib/dms/storage/file.pdf " + "x" * 600) sanitized = sanitize_extract_error(exc) assert "/var/lib" not in sanitized assert len(sanitized) <= 480 def test_sanitize_error_multiline_uses_first_line(): exc = RuntimeError("第一行原因\n第二行堆栈") assert sanitize_extract_error(exc) == "RuntimeError: 第一行原因" def test_unsupported_extension_returns_unsupported(tmp_path: Path): target = tmp_path / "image.png" target.write_bytes(b"\x89PNG\r\n\x1a\n") result = extract_document_content(target, "png") assert result.status == ContentExtractStatus.UNSUPPORTED.value assert result.text is None assert result.error is None assert result.extracted_at is None def test_corrupt_pdf_returns_failed_with_error(tmp_path: Path): target = tmp_path / "broken.pdf" target.write_bytes(b"not a real pdf content") result = extract_document_content(target, "pdf") assert result.status == ContentExtractStatus.FAILED.value assert result.text is None assert result.error # 必须携带失败原因 assert result.extracted_at is not None def _write_pdf(path: Path, lines: list[str], gap: int = 16, fontsize: int = 11) -> None: document = fitz.open() page = document.new_page() y = 72 for line in lines: # china-s:PyMuPDF 内嵌简体中文字体(默认 helv 无法编码 CJK) page.insert_text((72, y), line, fontsize=fontsize, fontname="china-s") y += gap document.save(path) document.close() def test_pdf_extraction_merges_wrapped_lines(tmp_path: Path): target = tmp_path / "wrapped.pdf" _write_pdf( target, ["这是一段在段内被换行", "打断的中文正文。", "另一段独立内容。"], ) result = extract_document_content(target, "pdf") assert result.status == ContentExtractStatus.SUCCESS.value assert result.error is None assert result.text is not None # 第一行未以句末标点结尾:与第二行合并;第二行以"。"结尾:保留段落边界 assert "这是一段在段内被换行打断的中文正文。" in result.text assert "打断的中文正文。\n另一段独立内容。" in result.text def test_pdf_heading_in_separate_block_not_merged_with_body(tmp_path: Path): """标题行与正文行距较大(独立成块)时,即使标题无句末标点也不并入正文。""" target = tmp_path / "heading.pdf" document = fitz.open() page = document.new_page() page.insert_text((72, 72), "第一章 总则", fontsize=11, fontname="china-s") page.insert_text((72, 72 + 24), "为规范方案管理,特制定本办法。", fontsize=11, fontname="china-s") document.save(target) document.close() result = extract_document_content(target, "pdf") assert result.status == ContentExtractStatus.SUCCESS.value assert result.text is not None assert "第一章 总则\n为规范方案管理,特制定本办法。" in result.text def test_pdf_different_font_size_splits_paragraph(tmp_path: Path): """标题与正文字号不同时独立成块(或块内字号突变切分),不合并为一段。""" target = tmp_path / "fontsize.pdf" document = fitz.open() page = document.new_page() page.insert_text((72, 72), "方案名称", fontsize=18, fontname="china-s") page.insert_text((72, 72 + 24), "本方案用于说明字号差异的段落切分", fontsize=11, fontname="china-s") document.save(target) document.close() result = extract_document_content(target, "pdf") assert result.status == ContentExtractStatus.SUCCESS.value assert result.text is not None assert "方案名称\n本方案用于说明字号差异的段落切分" in result.text def test_pdf_degenerate_layout_falls_back_to_heuristic(tmp_path: Path): """每行独立成块(行距≥20pt 逐行定位)时回退纯文本启发式,行为与现状一致。""" target = tmp_path / "degenerate.pdf" _write_pdf( target, ["这是一段在段内被换行", "打断的中文正文。", "另一段独立内容。"], gap=24, ) result = extract_document_content(target, "pdf") assert result.status == ContentExtractStatus.SUCCESS.value assert result.text is not None assert "这是一段在段内被换行打断的中文正文。" in result.text assert "打断的中文正文。\n另一段独立内容。" in result.text def test_merge_line_items_splits_on_font_size_jump(): items = [("方案名称", 18.0), ("本方案用于说明字号差异", 11.0)] assert _merge_line_items(items) == ["方案名称", "本方案用于说明字号差异"] def test_merge_line_items_merges_same_font_size(): items = [("段内被换行", 11.0), ("打断的中文正文", 11.0)] assert _merge_line_items(items) == ["段内被换行打断的中文正文"] def test_merge_line_items_ignores_font_size_when_unavailable(): # 字号为 0(纯文本兜底路径)不参与字号比较,仍按启发式合并 items = [("段内被换行", 0.0), ("打断的中文正文", 0.0)] assert _merge_line_items(items) == ["段内被换行打断的中文正文"]