272 lines
10 KiB
Python
272 lines
10 KiB
Python
|
"""HTML / MD / TXT / JSON formatters from IDM."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
|
||
|
|
from app.services.convert.formatters.text_postprocess import (
|
||
|
|
postprocess_inline,
|
||
|
|
postprocess_plain_text,
|
||
|
|
)
|
||
|
|
from app.services.convert.idm.model import Block, BlockType, Document
|
||
|
|
from app.services.convert.idm.serialize import document_to_dict
|
||
|
|
from app.services.convert.text.arabic_logical import contains_arabic, to_logical
|
||
|
|
|
||
|
|
|
||
|
|
def _md_cell(text: str) -> str:
|
||
|
|
"""Escape a value so it cannot break out of a Markdown table cell."""
|
||
|
|
return (
|
||
|
|
(text or "")
|
||
|
|
.replace("\\", "\\\\")
|
||
|
|
.replace("|", "\\|")
|
||
|
|
.replace("\r\n", " ")
|
||
|
|
.replace("\n", "<br>")
|
||
|
|
.replace("\r", " ")
|
||
|
|
.strip()
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _md_table(cells: list[list[str]], visual: bool) -> list[str]:
|
||
|
|
"""Render a grid as a valid Markdown table.
|
||
|
|
|
||
|
|
Every row is padded to the widest row so the column count matches the
|
||
|
|
delimiter row; ragged input otherwise produces a table no parser accepts.
|
||
|
|
"""
|
||
|
|
grid = [row for row in cells if row is not None]
|
||
|
|
if not grid:
|
||
|
|
return []
|
||
|
|
width = max(len(row) for row in grid)
|
||
|
|
if width == 0:
|
||
|
|
return []
|
||
|
|
out: list[str] = []
|
||
|
|
for idx, row in enumerate(grid):
|
||
|
|
values = [_md_cell(postprocess_inline(to_logical(c, visual=visual))) for c in row]
|
||
|
|
values += [""] * (width - len(values))
|
||
|
|
out.append("| " + " | ".join(values) + " |")
|
||
|
|
if idx == 0:
|
||
|
|
out.append("| " + " | ".join(["---"] * width) + " |")
|
||
|
|
out.append("")
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def md_table_lines(cells: list[list[str]]) -> list[str]:
|
||
|
|
"""Public entry to the Markdown table renderer, for non-IDM callers."""
|
||
|
|
return _md_table(cells, False)
|
||
|
|
|
||
|
|
|
||
|
|
def _escape(s: str) -> str:
|
||
|
|
return (
|
||
|
|
s.replace("&", "&")
|
||
|
|
.replace("<", "<")
|
||
|
|
.replace(">", ">")
|
||
|
|
.replace('"', """)
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _block_visual(block) -> bool:
|
||
|
|
"""True when a block's text is still in visual order and needs repairing.
|
||
|
|
|
||
|
|
Ordering is now resolved at extraction time — ``layout.glyphs`` and the OCR
|
||
|
|
adapter sort each line by its own reading direction — so a block's source
|
||
|
|
no longer implies visual order. Only an explicit span flag does, which is
|
||
|
|
set when a producer genuinely could not determine direction.
|
||
|
|
"""
|
||
|
|
return any(getattr(s, "visual_order", False) for s in (getattr(block, "spans", None) or []))
|
||
|
|
|
||
|
|
|
||
|
|
def _norm_text(block: Block) -> str:
|
||
|
|
return postprocess_inline(to_logical(block.plain_text(), visual=_block_visual(block)))
|
||
|
|
|
||
|
|
|
||
|
|
def format_txt(doc: Document) -> bytes:
|
||
|
|
parts: list[str] = []
|
||
|
|
for page in doc.pages:
|
||
|
|
for block in sorted(page.blocks, key=lambda b: b.reading_order):
|
||
|
|
t = _norm_text(block).strip()
|
||
|
|
if t:
|
||
|
|
parts.append(t)
|
||
|
|
return postprocess_plain_text("\n".join(parts)).encode("utf-8")
|
||
|
|
|
||
|
|
|
||
|
|
def format_md(doc: Document, *, page_markers: bool | None = None) -> bytes:
|
||
|
|
"""IDM → Markdown.
|
||
|
|
|
||
|
|
``page_markers`` inserts a ``## Page N`` heading per page. It defaults to
|
||
|
|
off because page headings fragment semantic chunking for search and LLM
|
||
|
|
ingest; set ``CONVERT_MD_PAGE_MARKERS=1`` or pass True to restore them.
|
||
|
|
"""
|
||
|
|
if page_markers is None:
|
||
|
|
page_markers = os.environ.get("CONVERT_MD_PAGE_MARKERS", "0").strip().lower() in (
|
||
|
|
"1",
|
||
|
|
"true",
|
||
|
|
"yes",
|
||
|
|
"on",
|
||
|
|
)
|
||
|
|
parts: list[str] = []
|
||
|
|
for page in doc.pages:
|
||
|
|
if page_markers:
|
||
|
|
parts.append(f"## Page {page.index + 1}\n")
|
||
|
|
in_list = False
|
||
|
|
for block in sorted(page.blocks, key=lambda b: b.reading_order):
|
||
|
|
if block.type == BlockType.heading:
|
||
|
|
if in_list:
|
||
|
|
parts.append("")
|
||
|
|
in_list = False
|
||
|
|
level = min(max(block.level, 1), 3)
|
||
|
|
parts.append("#" * level + f" {_norm_text(block)}\n")
|
||
|
|
elif block.type == BlockType.table and block.cells:
|
||
|
|
if in_list:
|
||
|
|
parts.append("")
|
||
|
|
in_list = False
|
||
|
|
parts.extend(_md_table(block.cells, _block_visual(block)))
|
||
|
|
elif block.type == BlockType.list_item:
|
||
|
|
in_list = True
|
||
|
|
marker = "1." if block.list_style == "number" else "-"
|
||
|
|
parts.append(f"{marker} {_norm_text(block)}")
|
||
|
|
else:
|
||
|
|
if in_list:
|
||
|
|
parts.append("")
|
||
|
|
in_list = False
|
||
|
|
t = _norm_text(block).strip()
|
||
|
|
if t:
|
||
|
|
parts.append(t)
|
||
|
|
parts.append("")
|
||
|
|
if in_list:
|
||
|
|
parts.append("")
|
||
|
|
# Document-level passes for cross-block hyphenation / page# / leaders
|
||
|
|
return postprocess_plain_text("\n".join(parts)).encode("utf-8")
|
||
|
|
|
||
|
|
|
||
|
|
def format_html(doc: Document) -> bytes:
|
||
|
|
chunks = [
|
||
|
|
'<!DOCTYPE html><html><head><meta charset="utf-8">'
|
||
|
|
"<title>Converted PDF</title></head><body>"
|
||
|
|
]
|
||
|
|
for page in doc.pages:
|
||
|
|
chunks.append(f'<section data-page="{page.index + 1}">')
|
||
|
|
chunks.append(f"<h2>Page {page.index + 1}</h2>")
|
||
|
|
open_list: str | None = None # "ul" | "ol"
|
||
|
|
|
||
|
|
def _close_list() -> None:
|
||
|
|
nonlocal open_list
|
||
|
|
if open_list:
|
||
|
|
chunks.append(f"</{open_list}>")
|
||
|
|
open_list = None
|
||
|
|
|
||
|
|
for block in sorted(page.blocks, key=lambda b: b.reading_order):
|
||
|
|
if block.type == BlockType.heading:
|
||
|
|
_close_list()
|
||
|
|
level = min(max(block.level, 1), 3)
|
||
|
|
txt = _escape(_norm_text(block))
|
||
|
|
rtl = ' dir="rtl" lang="ar"' if contains_arabic(block.plain_text()) else ""
|
||
|
|
chunks.append(f"<h{level}{rtl}>{txt}</h{level}>")
|
||
|
|
elif block.type == BlockType.table and block.cells:
|
||
|
|
_close_list()
|
||
|
|
visual = _block_visual(block)
|
||
|
|
chunks.append('<table border="1" cellpadding="4">')
|
||
|
|
for ri, row in enumerate(block.cells):
|
||
|
|
tag = "th" if ri == 0 else "td"
|
||
|
|
cells_html = []
|
||
|
|
for c in row:
|
||
|
|
cell_txt = _escape(to_logical(c, visual=visual))
|
||
|
|
rtl = ' dir="rtl"' if contains_arabic(c) else ""
|
||
|
|
cells_html.append(f"<{tag}{rtl}>{cell_txt}</{tag}>")
|
||
|
|
chunks.append("<tr>" + "".join(cells_html) + "</tr>")
|
||
|
|
chunks.append("</table>")
|
||
|
|
elif block.type == BlockType.list_item:
|
||
|
|
want = "ol" if block.list_style == "number" else "ul"
|
||
|
|
if open_list != want:
|
||
|
|
_close_list()
|
||
|
|
chunks.append(f"<{want}>")
|
||
|
|
open_list = want
|
||
|
|
txt = _escape(_norm_text(block))
|
||
|
|
rtl = ' dir="rtl"' if contains_arabic(block.plain_text()) else ""
|
||
|
|
chunks.append(f"<li{rtl}>{txt}</li>")
|
||
|
|
else:
|
||
|
|
_close_list()
|
||
|
|
t = _norm_text(block).strip()
|
||
|
|
if t:
|
||
|
|
rtl = ' dir="rtl" lang="ar"' if contains_arabic(block.plain_text()) else ""
|
||
|
|
chunks.append(f"<p{rtl}>{_escape(t)}</p>")
|
||
|
|
_close_list()
|
||
|
|
chunks.append("</section>")
|
||
|
|
chunks.append("</body></html>")
|
||
|
|
return "\n".join(chunks).encode("utf-8")
|
||
|
|
|
||
|
|
|
||
|
|
def format_json(doc: Document) -> bytes:
|
||
|
|
payload = document_to_dict(doc)
|
||
|
|
# Compatibility fields for consumers expecting page.text
|
||
|
|
for page_dict, page in zip(payload["pages"], doc.pages, strict=True):
|
||
|
|
page_dict["text"] = page.all_text()
|
||
|
|
page_dict["paragraphs"] = [
|
||
|
|
_norm_text(b)
|
||
|
|
for b in page.blocks
|
||
|
|
if b.type in (BlockType.paragraph, BlockType.heading, BlockType.list_item)
|
||
|
|
]
|
||
|
|
page_dict["tables"] = [b.cells for b in page.blocks if b.type == BlockType.table]
|
||
|
|
return json.dumps(payload, indent=2, ensure_ascii=False).encode("utf-8")
|
||
|
|
|
||
|
|
|
||
|
|
def format_csv(doc: Document, *, dialect: str = "excel") -> bytes:
|
||
|
|
"""IDM Document -> RFC 4180 compliant CSV bytes with UTF-8 BOM.
|
||
|
|
|
||
|
|
Extracts all detected tables across document pages.
|
||
|
|
- Normalizes ragged rows to the maximum column width per table.
|
||
|
|
- Normalizes Arabic/RTL text to logical order.
|
||
|
|
- Preserves embedded commas, quotes, and newlines per RFC 4180.
|
||
|
|
- Emits UTF-8 BOM (\\xef\\xbb\\xbf) for seamless Excel display on Windows.
|
||
|
|
- If multiple tables exist, separates them with clean section headers.
|
||
|
|
- If zero tables exist, safely exports non-empty paragraphs as single-column records.
|
||
|
|
"""
|
||
|
|
import codecs
|
||
|
|
import csv
|
||
|
|
import io
|
||
|
|
|
||
|
|
stream = io.StringIO()
|
||
|
|
writer = csv.writer(stream, dialect=dialect, quoting=csv.QUOTE_MINIMAL, lineterminator="\r\n")
|
||
|
|
|
||
|
|
# Collect all tables across pages
|
||
|
|
tables_found: list[tuple[int, list[list[str]], bool]] = []
|
||
|
|
for page in doc.pages:
|
||
|
|
for block in sorted(page.blocks, key=lambda b: b.reading_order):
|
||
|
|
if block.type == BlockType.table and block.cells:
|
||
|
|
tables_found.append((page.index + 1, block.cells, _block_visual(block)))
|
||
|
|
|
||
|
|
if tables_found:
|
||
|
|
for idx, (page_num, raw_cells, visual) in enumerate(tables_found, 1):
|
||
|
|
if idx > 1:
|
||
|
|
writer.writerow([])
|
||
|
|
writer.writerow([f"# --- Table {idx} (Page {page_num}) ---"])
|
||
|
|
elif len(tables_found) > 1:
|
||
|
|
writer.writerow([f"# --- Table 1 (Page {page_num}) ---"])
|
||
|
|
|
||
|
|
grid = [row for row in raw_cells if row is not None]
|
||
|
|
if not grid:
|
||
|
|
continue
|
||
|
|
width = max(len(row) for row in grid)
|
||
|
|
if width == 0:
|
||
|
|
continue
|
||
|
|
|
||
|
|
for row in grid:
|
||
|
|
cleaned_row = []
|
||
|
|
for c in row:
|
||
|
|
val = postprocess_inline(to_logical(str(c or ""), visual=visual)).strip()
|
||
|
|
val = val.replace("\r\n", "\n").replace("\r", "\n").replace("<br>", "\n")
|
||
|
|
cleaned_row.append(val)
|
||
|
|
if len(cleaned_row) < width:
|
||
|
|
cleaned_row.extend([""] * (width - len(cleaned_row)))
|
||
|
|
writer.writerow(cleaned_row)
|
||
|
|
else:
|
||
|
|
# Fallback for documents with no detected tables: export paragraphs
|
||
|
|
for page in doc.pages:
|
||
|
|
for block in sorted(page.blocks, key=lambda b: b.reading_order):
|
||
|
|
t = _norm_text(block).strip()
|
||
|
|
if t:
|
||
|
|
writer.writerow([t])
|
||
|
|
|
||
|
|
csv_text = stream.getvalue()
|
||
|
|
return codecs.BOM_UTF8 + csv_text.encode("utf-8")
|
||
|
|
|