package com.dms.service; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.pdfbox.Loader; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.text.PDFTextStripper; import org.apache.pdfbox.text.TextPosition; import org.apache.poi.xwpf.usermodel.XWPFDocument; import org.apache.poi.xwpf.usermodel.XWPFParagraph; import org.apache.poi.xwpf.usermodel.XWPFTable; import org.apache.poi.xwpf.usermodel.XWPFTableCell; import org.apache.poi.xwpf.usermodel.XWPFTableRow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; /** * 不依赖AI或Office进程的PDF/DOCX正文提取与检索文本构建, * 对齐 backend/dms/services/document_content_service.py * (Python fitz/PyMuPDF → PDFBox,python-docx → POI)。 * *

阶段10同步:① PDF正文按"版面块"识别段落(块边界即段落边界,块内合并 * 段内软换行,相邻行主字号突变亦视为段落边界);② 整页为逐行等距同字号的 * 退化网格(扫描件文本层典型特征)时回退纯文本启发式合并;③ 提取结果携带 * 脱敏失败原因(content_extract_error),供管理员"编辑"界面排障与重新解析。

*/ @Service public class DocumentContentService { private static final Logger logger = LoggerFactory.getLogger(DocumentContentService.class); /** 提取结果(对齐 ContentExtraction dataclass,error 缺省 null)。 */ public record ContentExtraction(String text, String status, LocalDateTime extractedAt, String error) { public ContentExtraction(String text, String status, LocalDateTime extractedAt) { this(text, status, extractedAt, null); } } // ------------------------------------------------------------ 提取失败原因脱敏 private static final int ERROR_MAX_LENGTH = 480; /** Windows 盘符绝对路径(C:\... 或 C:/...)。 */ private static final Pattern WINDOWS_PATH = Pattern.compile("[A-Za-z]:[\\/][^\\s,;]*"); /** POSIX 根路径(前一个字符非单词字符,避免误伤 URL/比例线)。 */ private static final Pattern POSIX_PATH = Pattern.compile("(?= 0 ? raw.substring(0, lineBreak) : raw; } message = WINDOWS_PATH.matcher(message).replaceAll(""); message = POSIX_PATH.matcher(message).replaceAll(""); String text = message.isEmpty() ? ex.getClass().getSimpleName() : ex.getClass().getSimpleName() + ": " + message; return text.length() > ERROR_MAX_LENGTH ? text.substring(0, ERROR_MAX_LENGTH) : text; } /** 对齐 build_search_text:非空部分以空格连接后 strip。 */ public static String buildSearchText(String name, String summary, List tags, String contentText) { List parts = new ArrayList<>(); if (name != null && !name.isEmpty()) { parts.add(name); } if (summary != null && !summary.isEmpty()) { parts.add(summary); } if (tags != null) { for (String tag : tags) { if (tag != null && !tag.isEmpty()) { parts.add(tag); } } } if (contentText != null && !contentText.isEmpty()) { parts.add(contentText); } return String.join(" ", parts).strip(); } // ------------------------------------------------------------ DOCX 提取 /** 对齐 _word_text:段落 → 表格递归单元格 → 各节页眉页脚。 */ private static String wordText(Path path) throws IOException { try (XWPFDocument document = new XWPFDocument(Files.newInputStream(path))) { List values = new ArrayList<>(); addParagraphs(document.getParagraphs(), values); addTables(document.getTables(), values); for (var header : document.getHeaderList()) { addParagraphs(header.getParagraphs(), values); addTables(header.getTables(), values); } for (var footer : document.getFooterList()) { addParagraphs(footer.getParagraphs(), values); addTables(footer.getTables(), values); } return String.join("\n", values).strip(); } } private static void addParagraphs(List paragraphs, List values) { for (XWPFParagraph paragraph : paragraphs) { String text = paragraph.getText(); String stripped = text == null ? "" : text.strip(); if (!stripped.isEmpty()) { values.add(stripped); } } } private static void addTables(List tables, List values) { for (XWPFTable table : tables) { for (XWPFTableRow row : table.getRows()) { for (XWPFTableCell cell : row.getTableCells()) { addParagraphs(cell.getParagraphs(), values); addTables(cell.getTables(), values); } } } } // ------------------------------------------------------------ PDF 提取(版面块段落化) /** * 块内相邻行主字号相对差达到该比例时视为段落边界(标题与正文同块的兜底)。 * 注:PDFBox 的字形高度(getHeightDir)较 fitz 的字号度量压缩了相对差异 * (实测16pt/11pt标题正文 ≈ 0.195),故取 0.15 以对齐 Python 版 0.2 的效果。 */ private static final double FONT_SIZE_SPLIT_RATIO = 0.15; /** 退化版面判定:逐行等距网格的间距容差(pt)与字号容差(pt)。 */ private static final double DEGENERATE_GAP_TOLERANCE = 2.0; private static final double DEGENERATE_SIZE_TOLERANCE = 0.5; /** * 块切分阈值:行距(top-to-top)超出中位行距该倍数视为版面块边界—— * 段间距显著大于行间距,标题/落款因前后留白自然独立成块; * 以自适应中位数为基准可兼容不同字体度量(PDFBox 高度 ≠ 字号)。 */ private static final double BLOCK_PITCH_RATIO = 1.4; /** 阈值下限附加量(pt):抑制极小行距页面的比例失真。 */ private static final double BLOCK_PITCH_FLOOR = 2.0; /** 句末标点:一行以此结尾视为自然段落结束,不再与下一行合并。 */ private static final Pattern SENTENCE_END = Pattern.compile( "[。!?!?;;::…..\u201d\u2019」』))\\]}>》]+$"); /** 条目/章节起始标记:下一行以此开头视为新段落(如 "1." "(一)" "第2章")。 */ private static final Pattern LIST_START = Pattern.compile( "^(?:\\d{1,3}[.、))]" + "|[((][一二三四五六七八九十百\\d]{1,4}[))]" + "|[一二三四五六七八九十]{1,3}、" + "|第[一二三四五六七八九十百千\\d]{1,5}[章节条部分篇])"); /** 版面行:(文本, 主字号, bbox顶, bbox底)。 */ private record LineItem(String text, double mainSize, double top, double bottom) { } /** 版面块:(行列表, bbox顶, bbox底)。 */ private record TextBlock(List items, double top, double bottom) { } /** * 逐字符收集版面行(对齐 fitz page.get_text("dict") 的 lines 结构): * PDFTextStripper 按 sortByPosition 顺序回调 writeString, * writeLineSeparator 处收行;行主字号取行内非空白字符的最大高度。 */ private static final class LineCollector extends PDFTextStripper { private final List lines = new ArrayList<>(); private final StringBuilder current = new StringBuilder(); private final List sizes = new ArrayList<>(); private double top = Double.MAX_VALUE; private double bottom = -Double.MAX_VALUE; LineCollector() throws IOException { setSortByPosition(true); } @Override protected void writeString(String text, List positions) { for (TextPosition position : positions) { String unicode = position.getUnicode(); if (unicode == null) { continue; } current.append(unicode); if (!unicode.isBlank()) { sizes.add((double) position.getHeightDir()); } double y = position.getYDirAdj(); double height = position.getHeightDir(); top = Math.min(top, y); bottom = Math.max(bottom, y + height); } } @Override protected void writeLineSeparator() { closeLine(); } @Override public String getText(PDDocument document) throws IOException { String result = super.getText(document); closeLine(); return result; } private void closeLine() { String text = current.toString().strip(); if (!text.isEmpty()) { double mainSize = 0.0; for (double size : sizes) { mainSize = Math.max(mainSize, size); } lines.add(new LineItem(text, mainSize, top, bottom)); } current.setLength(0); sizes.clear(); top = Double.MAX_VALUE; bottom = -Double.MAX_VALUE; } List drainLines() { List drained = new ArrayList<>(lines); lines.clear(); return drained; } } /** 对齐 _pdf_page_text:按版面块识别段落;退化网格回退纯文本启发式。 */ private static String pdfPageText(PDDocument document, int pageNumber) throws IOException { LineCollector collector = new LineCollector(); collector.setStartPage(pageNumber); collector.setEndPage(pageNumber); collector.getText(document); List lines = collector.drainLines(); if (lines.isEmpty()) { return ""; } List blocks = groupBlocks(lines); if (isDegenerateLineGrid(blocks, lines.size())) { PDFTextStripper plain = new PDFTextStripper(); plain.setStartPage(pageNumber); plain.setEndPage(pageNumber); return mergeWrappedLines(plain.getText(document)); } List paragraphs = new ArrayList<>(); for (TextBlock block : blocks) { paragraphs.addAll(mergeLineItems(block.items())); } return String.join("\n", paragraphs); } /** * 版面块分组(近似 fitz 按邻近度聚块):行距超过"中位行距× * BLOCK_PITCH_RATIO(且至少多 BLOCK_PITCH_FLOOR pt)"时视为块边界。 * 行数不足(无中位样本)时不切分,整页视为一个块, * 由 mergeLineItems 的字号/标点规则兜底分段。 */ private static List groupBlocks(List lines) { List pitches = new ArrayList<>(); for (int index = 0; index + 1 < lines.size(); index++) { pitches.add(lines.get(index + 1).top() - lines.get(index).top()); } double threshold = Double.MAX_VALUE; if (pitches.size() >= 2) { List sorted = new ArrayList<>(pitches); java.util.Collections.sort(sorted); double median = sorted.get(sorted.size() / 2); threshold = Math.max(median * BLOCK_PITCH_RATIO, median + BLOCK_PITCH_FLOOR); } List blocks = new ArrayList<>(); List current = new ArrayList<>(); for (int index = 0; index < lines.size(); index++) { LineItem line = lines.get(index); if (!current.isEmpty() && index > 0) { double pitch = line.top() - lines.get(index - 1).top(); if (pitch > threshold) { blocks.add(closeBlock(current)); current = new ArrayList<>(); } } current.add(line); } if (!current.isEmpty()) { blocks.add(closeBlock(current)); } return blocks; } private static TextBlock closeBlock(List items) { double top = Double.MAX_VALUE; double bottom = -Double.MAX_VALUE; for (LineItem item : items) { top = Math.min(top, item.top()); bottom = Math.max(bottom, item.bottom()); } return new TextBlock(List.copyOf(items), top, bottom); } /** * 对齐 _is_degenerate_line_grid:整页"每行一块、等距同字号"的退化网格 * (扫描件文本层逐行定位的典型特征)时块结构无分组价值。 */ private static boolean isDegenerateLineGrid(List blocks, int lineCount) { if (blocks.size() < 3 || blocks.size() < lineCount) { return false; } for (TextBlock block : blocks) { if (block.items().size() != 1) { return false; } } double minGap = Double.MAX_VALUE; double maxGap = -Double.MAX_VALUE; for (int index = 0; index + 1 < blocks.size(); index++) { double gap = blocks.get(index + 1).top() - blocks.get(index).top(); minGap = Math.min(minGap, gap); maxGap = Math.max(maxGap, gap); } if (maxGap - minGap > DEGENERATE_GAP_TOLERANCE) { return false; } double minSize = Double.MAX_VALUE; double maxSize = -Double.MAX_VALUE; for (TextBlock block : blocks) { double size = block.items().get(0).mainSize(); minSize = Math.min(minSize, size); maxSize = Math.max(maxSize, size); } return maxSize - minSize <= DEGENERATE_SIZE_TOLERANCE; } /** 对齐 _merge_line_items:条目编号/字号突变/句末标点切分,其余按软换行合并。 */ private static List mergeLineItems(List items) { List paragraphs = new ArrayList<>(); StringBuilder buffer = new StringBuilder(); double previousSize = 0.0; for (LineItem item : items) { String text = item.text(); double size = item.mainSize(); boolean split = false; if (!buffer.isEmpty() && LIST_START.matcher(text).find()) { split = true; } else if (!buffer.isEmpty() && previousSize > 0 && size > 0 && Math.abs(size - previousSize) / Math.max(size, previousSize) >= FONT_SIZE_SPLIT_RATIO) { split = true; } if (buffer.isEmpty()) { buffer.append(text); } else if (split) { paragraphs.add(buffer.toString()); buffer.setLength(0); buffer.append(text); } else { joinLines(buffer, text); } previousSize = size; Matcher endMatcher = SENTENCE_END.matcher(buffer); if (!buffer.isEmpty() && endMatcher.find()) { paragraphs.add(buffer.toString()); buffer.setLength(0); } } if (!buffer.isEmpty()) { paragraphs.add(buffer.toString()); } return paragraphs; } /** 对齐 _join_lines:两侧均为 ASCII 字母数字时补空格,其余直接相连。 */ private static void joinLines(StringBuilder previous, String following) { if (previous.isEmpty() || following.isEmpty()) { previous.append(following); return; } char left = previous.charAt(previous.length() - 1); char right = following.charAt(0); if (left < 128 && Character.isLetterOrDigit(left) && right < 128 && Character.isLetterOrDigit(right)) { previous.append(' ').append(following); return; } previous.append(following); } /** 对齐 _merge_wrapped_lines:纯文本兜底,空行/句末标点/条目编号视为段落边界。 */ private static String mergeWrappedLines(String pageText) { List paragraphs = new ArrayList<>(); List group = new ArrayList<>(); for (String rawLine : pageText.split("\n")) { String line = rawLine.strip(); if (line.isEmpty()) { paragraphs.addAll(mergeLineItems(group)); group = new ArrayList<>(); } else { group.add(new LineItem(line, 0.0, 0.0, 0.0)); } } paragraphs.addAll(mergeLineItems(group)); return String.join("\n", paragraphs); } /** 对齐 _pdf_text:逐页版面段落化,页与页之间以空行分隔(页边界即段落边界)。 */ private static String pdfText(Path path) throws IOException { try (PDDocument document = Loader.loadPDF(Files.readAllBytes(path))) { List pages = new ArrayList<>(); for (int index = 1; index <= document.getNumberOfPages(); index++) { pages.add(pdfPageText(document, index)); } List nonEmpty = pages.stream().filter(page -> !page.isEmpty()).toList(); return String.join("\n\n", nonEmpty).strip(); } } // ------------------------------------------------------------ 提取入口 /** 对齐 extract_document_content:仅 pdf/docx,异常 FAILED(含脱敏原因)。 */ public ContentExtraction extractDocumentContent(Path path, String extension) { String normalized = extension == null ? "" : extension.toLowerCase(Locale.ROOT).replaceFirst("^\\.+", ""); if (!"pdf".equals(normalized) && !"docx".equals(normalized)) { return new ContentExtraction(null, "UNSUPPORTED", null, null); } LocalDateTime extractedAt = LocalDateTime.now(ZoneOffset.UTC); String text; try { text = "pdf".equals(normalized) ? pdfText(path) : wordText(path); } catch (Exception ex) { logger.error("文档正文提取失败:extension={}", normalized, ex); return new ContentExtraction(null, "FAILED", extractedAt, sanitizeExtractError(ex)); } boolean hasText = text != null && !text.isEmpty(); return new ContentExtraction( hasText ? text : null, hasText ? "SUCCESS" : "EMPTY", extractedAt, null); } }