package com.dms.service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.TextPosition;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.apache.poi.xwpf.usermodel.XWPFTableCell;
import org.apache.poi.xwpf.usermodel.XWPFTableRow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
/**
* 不依赖AI或Office进程的PDF/DOCX正文提取与检索文本构建,
* 对齐 backend/dms/services/document_content_service.py
* (Python fitz/PyMuPDF → PDFBox,python-docx → POI)。
*
*
阶段10同步:① PDF正文按"版面块"识别段落(块边界即段落边界,块内合并
* 段内软换行,相邻行主字号突变亦视为段落边界);② 整页为逐行等距同字号的
* 退化网格(扫描件文本层典型特征)时回退纯文本启发式合并;③ 提取结果携带
* 脱敏失败原因(content_extract_error),供管理员"编辑"界面排障与重新解析。
*/
@Service
public class DocumentContentService {
private static final Logger logger = LoggerFactory.getLogger(DocumentContentService.class);
/** 提取结果(对齐 ContentExtraction dataclass,error 缺省 null)。 */
public record ContentExtraction(String text, String status, LocalDateTime extractedAt,
String error) {
public ContentExtraction(String text, String status, LocalDateTime extractedAt) {
this(text, status, extractedAt, null);
}
}
// ------------------------------------------------------------ 提取失败原因脱敏
private static final int ERROR_MAX_LENGTH = 480;
/** Windows 盘符绝对路径(C:\... 或 C:/...)。 */
private static final Pattern WINDOWS_PATH = Pattern.compile("[A-Za-z]:[\\/][^\\s,;]*");
/** POSIX 根路径(前一个字符非单词字符,避免误伤 URL/比例线)。 */
private static final Pattern POSIX_PATH = Pattern.compile("(?= 0 ? raw.substring(0, lineBreak) : raw;
}
message = WINDOWS_PATH.matcher(message).replaceAll("");
message = POSIX_PATH.matcher(message).replaceAll("");
String text = message.isEmpty()
? ex.getClass().getSimpleName()
: ex.getClass().getSimpleName() + ": " + message;
return text.length() > ERROR_MAX_LENGTH ? text.substring(0, ERROR_MAX_LENGTH) : text;
}
/** 对齐 build_search_text:非空部分以空格连接后 strip。 */
public static String buildSearchText(String name, String summary, List tags,
String contentText) {
List parts = new ArrayList<>();
if (name != null && !name.isEmpty()) {
parts.add(name);
}
if (summary != null && !summary.isEmpty()) {
parts.add(summary);
}
if (tags != null) {
for (String tag : tags) {
if (tag != null && !tag.isEmpty()) {
parts.add(tag);
}
}
}
if (contentText != null && !contentText.isEmpty()) {
parts.add(contentText);
}
return String.join(" ", parts).strip();
}
// ------------------------------------------------------------ DOCX 提取
/** 对齐 _word_text:段落 → 表格递归单元格 → 各节页眉页脚。 */
private static String wordText(Path path) throws IOException {
try (XWPFDocument document = new XWPFDocument(Files.newInputStream(path))) {
List values = new ArrayList<>();
addParagraphs(document.getParagraphs(), values);
addTables(document.getTables(), values);
for (var header : document.getHeaderList()) {
addParagraphs(header.getParagraphs(), values);
addTables(header.getTables(), values);
}
for (var footer : document.getFooterList()) {
addParagraphs(footer.getParagraphs(), values);
addTables(footer.getTables(), values);
}
return String.join("\n", values).strip();
}
}
private static void addParagraphs(List paragraphs, List values) {
for (XWPFParagraph paragraph : paragraphs) {
String text = paragraph.getText();
String stripped = text == null ? "" : text.strip();
if (!stripped.isEmpty()) {
values.add(stripped);
}
}
}
private static void addTables(List tables, List values) {
for (XWPFTable table : tables) {
for (XWPFTableRow row : table.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
addParagraphs(cell.getParagraphs(), values);
addTables(cell.getTables(), values);
}
}
}
}
// ------------------------------------------------------------ PDF 提取(版面块段落化)
/**
* 块内相邻行主字号相对差达到该比例时视为段落边界(标题与正文同块的兜底)。
* 注:PDFBox 的字形高度(getHeightDir)较 fitz 的字号度量压缩了相对差异
* (实测16pt/11pt标题正文 ≈ 0.195),故取 0.15 以对齐 Python 版 0.2 的效果。
*/
private static final double FONT_SIZE_SPLIT_RATIO = 0.15;
/** 退化版面判定:逐行等距网格的间距容差(pt)与字号容差(pt)。 */
private static final double DEGENERATE_GAP_TOLERANCE = 2.0;
private static final double DEGENERATE_SIZE_TOLERANCE = 0.5;
/**
* 块切分阈值:行距(top-to-top)超出中位行距该倍数视为版面块边界——
* 段间距显著大于行间距,标题/落款因前后留白自然独立成块;
* 以自适应中位数为基准可兼容不同字体度量(PDFBox 高度 ≠ 字号)。
*/
private static final double BLOCK_PITCH_RATIO = 1.4;
/** 阈值下限附加量(pt):抑制极小行距页面的比例失真。 */
private static final double BLOCK_PITCH_FLOOR = 2.0;
/** 句末标点:一行以此结尾视为自然段落结束,不再与下一行合并。 */
private static final Pattern SENTENCE_END = Pattern.compile(
"[。!?!?;;::…..\u201d\u2019」』))\\]}>》]+$");
/** 条目/章节起始标记:下一行以此开头视为新段落(如 "1." "(一)" "第2章")。 */
private static final Pattern LIST_START = Pattern.compile(
"^(?:\\d{1,3}[.、))]"
+ "|[((][一二三四五六七八九十百\\d]{1,4}[))]"
+ "|[一二三四五六七八九十]{1,3}、"
+ "|第[一二三四五六七八九十百千\\d]{1,5}[章节条部分篇])");
/** 版面行:(文本, 主字号, bbox顶, bbox底)。 */
private record LineItem(String text, double mainSize, double top, double bottom) {
}
/** 版面块:(行列表, bbox顶, bbox底)。 */
private record TextBlock(List items, double top, double bottom) {
}
/**
* 逐字符收集版面行(对齐 fitz page.get_text("dict") 的 lines 结构):
* PDFTextStripper 按 sortByPosition 顺序回调 writeString,
* writeLineSeparator 处收行;行主字号取行内非空白字符的最大高度。
*/
private static final class LineCollector extends PDFTextStripper {
private final List lines = new ArrayList<>();
private final StringBuilder current = new StringBuilder();
private final List sizes = new ArrayList<>();
private double top = Double.MAX_VALUE;
private double bottom = -Double.MAX_VALUE;
LineCollector() throws IOException {
setSortByPosition(true);
}
@Override
protected void writeString(String text, List positions) {
for (TextPosition position : positions) {
String unicode = position.getUnicode();
if (unicode == null) {
continue;
}
current.append(unicode);
if (!unicode.isBlank()) {
sizes.add((double) position.getHeightDir());
}
double y = position.getYDirAdj();
double height = position.getHeightDir();
top = Math.min(top, y);
bottom = Math.max(bottom, y + height);
}
}
@Override
protected void writeLineSeparator() {
closeLine();
}
@Override
public String getText(PDDocument document) throws IOException {
String result = super.getText(document);
closeLine();
return result;
}
private void closeLine() {
String text = current.toString().strip();
if (!text.isEmpty()) {
double mainSize = 0.0;
for (double size : sizes) {
mainSize = Math.max(mainSize, size);
}
lines.add(new LineItem(text, mainSize, top, bottom));
}
current.setLength(0);
sizes.clear();
top = Double.MAX_VALUE;
bottom = -Double.MAX_VALUE;
}
List drainLines() {
List drained = new ArrayList<>(lines);
lines.clear();
return drained;
}
}
/** 对齐 _pdf_page_text:按版面块识别段落;退化网格回退纯文本启发式。 */
private static String pdfPageText(PDDocument document, int pageNumber)
throws IOException {
LineCollector collector = new LineCollector();
collector.setStartPage(pageNumber);
collector.setEndPage(pageNumber);
collector.getText(document);
List lines = collector.drainLines();
if (lines.isEmpty()) {
return "";
}
List blocks = groupBlocks(lines);
if (isDegenerateLineGrid(blocks, lines.size())) {
PDFTextStripper plain = new PDFTextStripper();
plain.setStartPage(pageNumber);
plain.setEndPage(pageNumber);
return mergeWrappedLines(plain.getText(document));
}
List paragraphs = new ArrayList<>();
for (TextBlock block : blocks) {
paragraphs.addAll(mergeLineItems(block.items()));
}
return String.join("\n", paragraphs);
}
/**
* 版面块分组(近似 fitz 按邻近度聚块):行距超过"中位行距×
* BLOCK_PITCH_RATIO(且至少多 BLOCK_PITCH_FLOOR pt)"时视为块边界。
* 行数不足(无中位样本)时不切分,整页视为一个块,
* 由 mergeLineItems 的字号/标点规则兜底分段。
*/
private static List groupBlocks(List lines) {
List pitches = new ArrayList<>();
for (int index = 0; index + 1 < lines.size(); index++) {
pitches.add(lines.get(index + 1).top() - lines.get(index).top());
}
double threshold = Double.MAX_VALUE;
if (pitches.size() >= 2) {
List sorted = new ArrayList<>(pitches);
java.util.Collections.sort(sorted);
double median = sorted.get(sorted.size() / 2);
threshold = Math.max(median * BLOCK_PITCH_RATIO,
median + BLOCK_PITCH_FLOOR);
}
List blocks = new ArrayList<>();
List current = new ArrayList<>();
for (int index = 0; index < lines.size(); index++) {
LineItem line = lines.get(index);
if (!current.isEmpty() && index > 0) {
double pitch = line.top() - lines.get(index - 1).top();
if (pitch > threshold) {
blocks.add(closeBlock(current));
current = new ArrayList<>();
}
}
current.add(line);
}
if (!current.isEmpty()) {
blocks.add(closeBlock(current));
}
return blocks;
}
private static TextBlock closeBlock(List items) {
double top = Double.MAX_VALUE;
double bottom = -Double.MAX_VALUE;
for (LineItem item : items) {
top = Math.min(top, item.top());
bottom = Math.max(bottom, item.bottom());
}
return new TextBlock(List.copyOf(items), top, bottom);
}
/**
* 对齐 _is_degenerate_line_grid:整页"每行一块、等距同字号"的退化网格
* (扫描件文本层逐行定位的典型特征)时块结构无分组价值。
*/
private static boolean isDegenerateLineGrid(List blocks, int lineCount) {
if (blocks.size() < 3 || blocks.size() < lineCount) {
return false;
}
for (TextBlock block : blocks) {
if (block.items().size() != 1) {
return false;
}
}
double minGap = Double.MAX_VALUE;
double maxGap = -Double.MAX_VALUE;
for (int index = 0; index + 1 < blocks.size(); index++) {
double gap = blocks.get(index + 1).top() - blocks.get(index).top();
minGap = Math.min(minGap, gap);
maxGap = Math.max(maxGap, gap);
}
if (maxGap - minGap > DEGENERATE_GAP_TOLERANCE) {
return false;
}
double minSize = Double.MAX_VALUE;
double maxSize = -Double.MAX_VALUE;
for (TextBlock block : blocks) {
double size = block.items().get(0).mainSize();
minSize = Math.min(minSize, size);
maxSize = Math.max(maxSize, size);
}
return maxSize - minSize <= DEGENERATE_SIZE_TOLERANCE;
}
/** 对齐 _merge_line_items:条目编号/字号突变/句末标点切分,其余按软换行合并。 */
private static List mergeLineItems(List items) {
List paragraphs = new ArrayList<>();
StringBuilder buffer = new StringBuilder();
double previousSize = 0.0;
for (LineItem item : items) {
String text = item.text();
double size = item.mainSize();
boolean split = false;
if (!buffer.isEmpty() && LIST_START.matcher(text).find()) {
split = true;
} else if (!buffer.isEmpty() && previousSize > 0 && size > 0
&& Math.abs(size - previousSize) / Math.max(size, previousSize)
>= FONT_SIZE_SPLIT_RATIO) {
split = true;
}
if (buffer.isEmpty()) {
buffer.append(text);
} else if (split) {
paragraphs.add(buffer.toString());
buffer.setLength(0);
buffer.append(text);
} else {
joinLines(buffer, text);
}
previousSize = size;
Matcher endMatcher = SENTENCE_END.matcher(buffer);
if (!buffer.isEmpty() && endMatcher.find()) {
paragraphs.add(buffer.toString());
buffer.setLength(0);
}
}
if (!buffer.isEmpty()) {
paragraphs.add(buffer.toString());
}
return paragraphs;
}
/** 对齐 _join_lines:两侧均为 ASCII 字母数字时补空格,其余直接相连。 */
private static void joinLines(StringBuilder previous, String following) {
if (previous.isEmpty() || following.isEmpty()) {
previous.append(following);
return;
}
char left = previous.charAt(previous.length() - 1);
char right = following.charAt(0);
if (left < 128 && Character.isLetterOrDigit(left)
&& right < 128 && Character.isLetterOrDigit(right)) {
previous.append(' ').append(following);
return;
}
previous.append(following);
}
/** 对齐 _merge_wrapped_lines:纯文本兜底,空行/句末标点/条目编号视为段落边界。 */
private static String mergeWrappedLines(String pageText) {
List paragraphs = new ArrayList<>();
List group = new ArrayList<>();
for (String rawLine : pageText.split("\n")) {
String line = rawLine.strip();
if (line.isEmpty()) {
paragraphs.addAll(mergeLineItems(group));
group = new ArrayList<>();
} else {
group.add(new LineItem(line, 0.0, 0.0, 0.0));
}
}
paragraphs.addAll(mergeLineItems(group));
return String.join("\n", paragraphs);
}
/** 对齐 _pdf_text:逐页版面段落化,页与页之间以空行分隔(页边界即段落边界)。 */
private static String pdfText(Path path) throws IOException {
try (PDDocument document = Loader.loadPDF(Files.readAllBytes(path))) {
List pages = new ArrayList<>();
for (int index = 1; index <= document.getNumberOfPages(); index++) {
pages.add(pdfPageText(document, index));
}
List nonEmpty = pages.stream().filter(page -> !page.isEmpty()).toList();
return String.join("\n\n", nonEmpty).strip();
}
}
// ------------------------------------------------------------ 提取入口
/** 对齐 extract_document_content:仅 pdf/docx,异常 FAILED(含脱敏原因)。 */
public ContentExtraction extractDocumentContent(Path path, String extension) {
String normalized = extension == null
? ""
: extension.toLowerCase(Locale.ROOT).replaceFirst("^\\.+", "");
if (!"pdf".equals(normalized) && !"docx".equals(normalized)) {
return new ContentExtraction(null, "UNSUPPORTED", null, null);
}
LocalDateTime extractedAt = LocalDateTime.now(ZoneOffset.UTC);
String text;
try {
text = "pdf".equals(normalized) ? pdfText(path) : wordText(path);
} catch (Exception ex) {
logger.error("文档正文提取失败:extension={}", normalized, ex);
return new ContentExtraction(null, "FAILED", extractedAt, sanitizeExtractError(ex));
}
boolean hasText = text != null && !text.isEmpty();
return new ContentExtraction(
hasText ? text : null,
hasText ? "SUCCESS" : "EMPTY",
extractedAt,
null);
}
}