document-management/backend-java/src/main/java/com/dms/service/DocumentContentService.java

package com.dms.service;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.TextPosition;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.apache.poi.xwpf.usermodel.XWPFTableCell;
import org.apache.poi.xwpf.usermodel.XWPFTableRow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

/**
 * 不依赖AI或Office进程的PDF/DOCX正文提取与检索文本构建，
 * 对齐 backend/dms/services/document_content_service.py
 * （Python fitz/PyMuPDF → PDFBox，python-docx → POI）。
 *
 * <p>阶段10同步：① PDF正文按"版面块"识别段落（块边界即段落边界，块内合并
 * 段内软换行，相邻行主字号突变亦视为段落边界）；② 整页为逐行等距同字号的
 * 退化网格（扫描件文本层典型特征）时回退纯文本启发式合并；③ 提取结果携带
 * 脱敏失败原因（content_extract_error），供管理员"编辑"界面排障与重新解析。</p>
 */
@Service
public class DocumentContentService {

    private static final Logger logger = LoggerFactory.getLogger(DocumentContentService.class);

    /** 提取结果（对齐 ContentExtraction dataclass，error 缺省 null）。 */
    public record ContentExtraction(String text, String status, LocalDateTime extractedAt,
                                    String error) {

        public ContentExtraction(String text, String status, LocalDateTime extractedAt) {
            this(text, status, extractedAt, null);
        }
    }

    // ------------------------------------------------------------ 提取失败原因脱敏

    private static final int ERROR_MAX_LENGTH = 480;
    /** Windows 盘符绝对路径（C:\... 或 C:/...）。 */
    private static final Pattern WINDOWS_PATH = Pattern.compile("[A-Za-z]:[\\/][^\\s,;]*");
    /** POSIX 根路径（前一个字符非单词字符，避免误伤 URL/比例线）。 */
    private static final Pattern POSIX_PATH = Pattern.compile("(?<![\\w])/[A-Za-z0-9_./-]+");

    /** 对齐 sanitize_extract_error：异常类型+首行信息，绝对路径替换为 &lt;path&gt;。 */
    public static String sanitizeExtractError(Throwable ex) {
        String raw = String.valueOf(ex.getMessage());
        String message = "";
        if (!raw.isBlank()) {
            int lineBreak = raw.indexOf('\n');
            message = lineBreak >= 0 ? raw.substring(0, lineBreak) : raw;
        }
        message = WINDOWS_PATH.matcher(message).replaceAll("<path>");
        message = POSIX_PATH.matcher(message).replaceAll("<path>");
        String text = message.isEmpty()
                ? ex.getClass().getSimpleName()
                : ex.getClass().getSimpleName() + ": " + message;
        return text.length() > ERROR_MAX_LENGTH ? text.substring(0, ERROR_MAX_LENGTH) : text;
    }

    /** 对齐 build_search_text：非空部分以空格连接后 strip。 */
    public static String buildSearchText(String name, String summary, List<String> tags,
                                         String contentText) {
        List<String> parts = new ArrayList<>();
        if (name != null && !name.isEmpty()) {
            parts.add(name);
        }
        if (summary != null && !summary.isEmpty()) {
            parts.add(summary);
        }
        if (tags != null) {
            for (String tag : tags) {
                if (tag != null && !tag.isEmpty()) {
                    parts.add(tag);
                }
            }
        }
        if (contentText != null && !contentText.isEmpty()) {
            parts.add(contentText);
        }
        return String.join(" ", parts).strip();
    }

    // ------------------------------------------------------------ DOCX 提取

    /** 对齐 _word_text：段落 → 表格递归单元格 → 各节页眉页脚。 */
    private static String wordText(Path path) throws IOException {
        try (XWPFDocument document = new XWPFDocument(Files.newInputStream(path))) {
            List<String> values = new ArrayList<>();
            addParagraphs(document.getParagraphs(), values);
            addTables(document.getTables(), values);
            for (var header : document.getHeaderList()) {
                addParagraphs(header.getParagraphs(), values);
                addTables(header.getTables(), values);
            }
            for (var footer : document.getFooterList()) {
                addParagraphs(footer.getParagraphs(), values);
                addTables(footer.getTables(), values);
            }
            return String.join("\n", values).strip();
        }
    }

    private static void addParagraphs(List<XWPFParagraph> paragraphs, List<String> values) {
        for (XWPFParagraph paragraph : paragraphs) {
            String text = paragraph.getText();
            String stripped = text == null ? "" : text.strip();
            if (!stripped.isEmpty()) {
                values.add(stripped);
            }
        }
    }

    private static void addTables(List<XWPFTable> tables, List<String> values) {
        for (XWPFTable table : tables) {
            for (XWPFTableRow row : table.getRows()) {
                for (XWPFTableCell cell : row.getTableCells()) {
                    addParagraphs(cell.getParagraphs(), values);
                    addTables(cell.getTables(), values);
                }
            }
        }
    }

    // ------------------------------------------------------------ PDF 提取（版面块段落化）

    /**
     * 块内相邻行主字号相对差达到该比例时视为段落边界（标题与正文同块的兜底）。
     * 注：PDFBox 的字形高度（getHeightDir）较 fitz 的字号度量压缩了相对差异
     * （实测16pt/11pt标题正文 ≈ 0.195），故取 0.15 以对齐 Python 版 0.2 的效果。
     */
    private static final double FONT_SIZE_SPLIT_RATIO = 0.15;

    /** 退化版面判定：逐行等距网格的间距容差（pt）与字号容差（pt）。 */
    private static final double DEGENERATE_GAP_TOLERANCE = 2.0;
    private static final double DEGENERATE_SIZE_TOLERANCE = 0.5;

    /**
     * 块切分阈值：行距（top-to-top）超出中位行距该倍数视为版面块边界——
     * 段间距显著大于行间距，标题/落款因前后留白自然独立成块；
     * 以自适应中位数为基准可兼容不同字体度量（PDFBox 高度 ≠ 字号）。
     */
    private static final double BLOCK_PITCH_RATIO = 1.4;
    /** 阈值下限附加量（pt）：抑制极小行距页面的比例失真。 */
    private static final double BLOCK_PITCH_FLOOR = 2.0;

    /** 句末标点：一行以此结尾视为自然段落结束，不再与下一行合并。 */
    private static final Pattern SENTENCE_END = Pattern.compile(
            "[。！？!?；;：:…．.\u201d\u2019」』）)\\]}>》]+$");

    /** 条目/章节起始标记：下一行以此开头视为新段落（如 "1." "（一）" "第2章"）。 */
    private static final Pattern LIST_START = Pattern.compile(
            "^(?:\\d{1,3}[.、)）]"
                    + "|[（(][一二三四五六七八九十百\\d]{1,4}[)）]"
                    + "|[一二三四五六七八九十]{1,3}、"
                    + "|第[一二三四五六七八九十百千\\d]{1,5}[章节条部分篇])");

    /** 版面行：(文本, 主字号, bbox顶, bbox底)。 */
    private record LineItem(String text, double mainSize, double top, double bottom) {
    }

    /** 版面块：(行列表, bbox顶, bbox底)。 */
    private record TextBlock(List<LineItem> items, double top, double bottom) {
    }

    /**
     * 逐字符收集版面行（对齐 fitz page.get_text("dict") 的 lines 结构）：
     * PDFTextStripper 按 sortByPosition 顺序回调 writeString，
     * writeLineSeparator 处收行；行主字号取行内非空白字符的最大高度。
     */
    private static final class LineCollector extends PDFTextStripper {

        private final List<LineItem> lines = new ArrayList<>();
        private final StringBuilder current = new StringBuilder();
        private final List<Double> sizes = new ArrayList<>();
        private double top = Double.MAX_VALUE;
        private double bottom = -Double.MAX_VALUE;

        LineCollector() throws IOException {
            setSortByPosition(true);
        }

        @Override
        protected void writeString(String text, List<TextPosition> positions) {
            for (TextPosition position : positions) {
                String unicode = position.getUnicode();
                if (unicode == null) {
                    continue;
                }
                current.append(unicode);
                if (!unicode.isBlank()) {
                    sizes.add((double) position.getHeightDir());
                }
                double y = position.getYDirAdj();
                double height = position.getHeightDir();
                top = Math.min(top, y);
                bottom = Math.max(bottom, y + height);
            }
        }

        @Override
        protected void writeLineSeparator() {
            closeLine();
        }

        @Override
        public String getText(PDDocument document) throws IOException {
            String result = super.getText(document);
            closeLine();
            return result;
        }

        private void closeLine() {
            String text = current.toString().strip();
            if (!text.isEmpty()) {
                double mainSize = 0.0;
                for (double size : sizes) {
                    mainSize = Math.max(mainSize, size);
                }
                lines.add(new LineItem(text, mainSize, top, bottom));
            }
            current.setLength(0);
            sizes.clear();
            top = Double.MAX_VALUE;
            bottom = -Double.MAX_VALUE;
        }

        List<LineItem> drainLines() {
            List<LineItem> drained = new ArrayList<>(lines);
            lines.clear();
            return drained;
        }
    }

    /** 对齐 _pdf_page_text：按版面块识别段落；退化网格回退纯文本启发式。 */
    private static String pdfPageText(PDDocument document, int pageNumber)
            throws IOException {
        LineCollector collector = new LineCollector();
        collector.setStartPage(pageNumber);
        collector.setEndPage(pageNumber);
        collector.getText(document);
        List<LineItem> lines = collector.drainLines();
        if (lines.isEmpty()) {
            return "";
        }
        List<TextBlock> blocks = groupBlocks(lines);
        if (isDegenerateLineGrid(blocks, lines.size())) {
            PDFTextStripper plain = new PDFTextStripper();
            plain.setStartPage(pageNumber);
            plain.setEndPage(pageNumber);
            return mergeWrappedLines(plain.getText(document));
        }
        List<String> paragraphs = new ArrayList<>();
        for (TextBlock block : blocks) {
            paragraphs.addAll(mergeLineItems(block.items()));
        }
        return String.join("\n", paragraphs);
    }

    /**
     * 版面块分组（近似 fitz 按邻近度聚块）：行距超过"中位行距×
     * BLOCK_PITCH_RATIO（且至少多 BLOCK_PITCH_FLOOR pt）"时视为块边界。
     * 行数不足（无中位样本）时不切分，整页视为一个块，
     * 由 mergeLineItems 的字号/标点规则兜底分段。
     */
    private static List<TextBlock> groupBlocks(List<LineItem> lines) {
        List<Double> pitches = new ArrayList<>();
        for (int index = 0; index + 1 < lines.size(); index++) {
            pitches.add(lines.get(index + 1).top() - lines.get(index).top());
        }
        double threshold = Double.MAX_VALUE;
        if (pitches.size() >= 2) {
            List<Double> sorted = new ArrayList<>(pitches);
            java.util.Collections.sort(sorted);
            double median = sorted.get(sorted.size() / 2);
            threshold = Math.max(median * BLOCK_PITCH_RATIO,
                    median + BLOCK_PITCH_FLOOR);
        }
        List<TextBlock> blocks = new ArrayList<>();
        List<LineItem> current = new ArrayList<>();
        for (int index = 0; index < lines.size(); index++) {
            LineItem line = lines.get(index);
            if (!current.isEmpty() && index > 0) {
                double pitch = line.top() - lines.get(index - 1).top();
                if (pitch > threshold) {
                    blocks.add(closeBlock(current));
                    current = new ArrayList<>();
                }
            }
            current.add(line);
        }
        if (!current.isEmpty()) {
            blocks.add(closeBlock(current));
        }
        return blocks;
    }

    private static TextBlock closeBlock(List<LineItem> items) {
        double top = Double.MAX_VALUE;
        double bottom = -Double.MAX_VALUE;
        for (LineItem item : items) {
            top = Math.min(top, item.top());
            bottom = Math.max(bottom, item.bottom());
        }
        return new TextBlock(List.copyOf(items), top, bottom);
    }

    /**
     * 对齐 _is_degenerate_line_grid：整页"每行一块、等距同字号"的退化网格
     * （扫描件文本层逐行定位的典型特征）时块结构无分组价值。
     */
    private static boolean isDegenerateLineGrid(List<TextBlock> blocks, int lineCount) {
        if (blocks.size() < 3 || blocks.size() < lineCount) {
            return false;
        }
        for (TextBlock block : blocks) {
            if (block.items().size() != 1) {
                return false;
            }
        }
        double minGap = Double.MAX_VALUE;
        double maxGap = -Double.MAX_VALUE;
        for (int index = 0; index + 1 < blocks.size(); index++) {
            double gap = blocks.get(index + 1).top() - blocks.get(index).top();
            minGap = Math.min(minGap, gap);
            maxGap = Math.max(maxGap, gap);
        }
        if (maxGap - minGap > DEGENERATE_GAP_TOLERANCE) {
            return false;
        }
        double minSize = Double.MAX_VALUE;
        double maxSize = -Double.MAX_VALUE;
        for (TextBlock block : blocks) {
            double size = block.items().get(0).mainSize();
            minSize = Math.min(minSize, size);
            maxSize = Math.max(maxSize, size);
        }
        return maxSize - minSize <= DEGENERATE_SIZE_TOLERANCE;
    }

    /** 对齐 _merge_line_items：条目编号/字号突变/句末标点切分，其余按软换行合并。 */
    private static List<String> mergeLineItems(List<LineItem> items) {
        List<String> paragraphs = new ArrayList<>();
        StringBuilder buffer = new StringBuilder();
        double previousSize = 0.0;
        for (LineItem item : items) {
            String text = item.text();
            double size = item.mainSize();
            boolean split = false;
            if (!buffer.isEmpty() && LIST_START.matcher(text).find()) {
                split = true;
            } else if (!buffer.isEmpty() && previousSize > 0 && size > 0
                    && Math.abs(size - previousSize) / Math.max(size, previousSize)
                    >= FONT_SIZE_SPLIT_RATIO) {
                split = true;
            }
            if (buffer.isEmpty()) {
                buffer.append(text);
            } else if (split) {
                paragraphs.add(buffer.toString());
                buffer.setLength(0);
                buffer.append(text);
            } else {
                joinLines(buffer, text);
            }
            previousSize = size;
            Matcher endMatcher = SENTENCE_END.matcher(buffer);
            if (!buffer.isEmpty() && endMatcher.find()) {
                paragraphs.add(buffer.toString());
                buffer.setLength(0);
            }
        }
        if (!buffer.isEmpty()) {
            paragraphs.add(buffer.toString());
        }
        return paragraphs;
    }

    /** 对齐 _join_lines：两侧均为 ASCII 字母数字时补空格，其余直接相连。 */
    private static void joinLines(StringBuilder previous, String following) {
        if (previous.isEmpty() || following.isEmpty()) {
            previous.append(following);
            return;
        }
        char left = previous.charAt(previous.length() - 1);
        char right = following.charAt(0);
        if (left < 128 && Character.isLetterOrDigit(left)
                && right < 128 && Character.isLetterOrDigit(right)) {
            previous.append(' ').append(following);
            return;
        }
        previous.append(following);
    }

    /** 对齐 _merge_wrapped_lines：纯文本兜底，空行/句末标点/条目编号视为段落边界。 */
    private static String mergeWrappedLines(String pageText) {
        List<String> paragraphs = new ArrayList<>();
        List<LineItem> group = new ArrayList<>();
        for (String rawLine : pageText.split("\n")) {
            String line = rawLine.strip();
            if (line.isEmpty()) {
                paragraphs.addAll(mergeLineItems(group));
                group = new ArrayList<>();
            } else {
                group.add(new LineItem(line, 0.0, 0.0, 0.0));
            }
        }
        paragraphs.addAll(mergeLineItems(group));
        return String.join("\n", paragraphs);
    }

    /** 对齐 _pdf_text：逐页版面段落化，页与页之间以空行分隔（页边界即段落边界）。 */
    private static String pdfText(Path path) throws IOException {
        try (PDDocument document = Loader.loadPDF(Files.readAllBytes(path))) {
            List<String> pages = new ArrayList<>();
            for (int index = 1; index <= document.getNumberOfPages(); index++) {
                pages.add(pdfPageText(document, index));
            }
            List<String> nonEmpty = pages.stream().filter(page -> !page.isEmpty()).toList();
            return String.join("\n\n", nonEmpty).strip();
        }
    }

    // ------------------------------------------------------------ 提取入口

    /** 对齐 extract_document_content：仅 pdf/docx，异常 FAILED（含脱敏原因）。 */
    public ContentExtraction extractDocumentContent(Path path, String extension) {
        String normalized = extension == null
                ? ""
                : extension.toLowerCase(Locale.ROOT).replaceFirst("^\\.+", "");
        if (!"pdf".equals(normalized) && !"docx".equals(normalized)) {
            return new ContentExtraction(null, "UNSUPPORTED", null, null);
        }
        LocalDateTime extractedAt = LocalDateTime.now(ZoneOffset.UTC);
        String text;
        try {
            text = "pdf".equals(normalized) ? pdfText(path) : wordText(path);
        } catch (Exception ex) {
            logger.error("文档正文提取失败：extension={}", normalized, ex);
            return new ContentExtraction(null, "FAILED", extractedAt, sanitizeExtractError(ex));
        }
        boolean hasText = text != null && !text.isEmpty();
        return new ContentExtraction(
                hasText ? text : null,
                hasText ? "SUCCESS" : "EMPTY",
                extractedAt,
                null);
    }
}