1279 lines
52 KiB
Python
1279 lines
52 KiB
Python
"""Layout pipeline: PDF bytes → IDM Document."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
|
|
from app.services.convert import cancellation, doc_cache
|
|
from app.services.convert.idm.model import (
|
|
PAGE_RASTER_SOURCE,
|
|
BBox,
|
|
Block,
|
|
BlockType,
|
|
Document,
|
|
Page,
|
|
PageKind,
|
|
)
|
|
from app.services.convert.idm.serialize import maybe_dump_idm
|
|
from app.services.convert.layout.blocks import (
|
|
estimate_body_size,
|
|
lines_to_block,
|
|
)
|
|
from app.services.convert.layout.doc_wide import apply_document_wide
|
|
from app.services.convert.layout.glyphs import lines_from_glyphs, lines_from_plain_text
|
|
from app.services.convert.layout.idm_optimize import optimize_document
|
|
from app.services.convert.layout.images import (
|
|
display_list_ops_from_engine_page,
|
|
figure_blocks_from_pypdf_page,
|
|
image_blocks_from_engine_page,
|
|
image_blocks_from_pypdf_page,
|
|
)
|
|
from app.services.convert.layout.page_classify import (
|
|
classify_page,
|
|
estimate_image_coverage,
|
|
glyph_size_variance,
|
|
)
|
|
from app.services.convert.layout.paragraphs import group_lines as group_paragraph_lines
|
|
from app.services.convert.layout.paragraphs import lexical_hyphen_pairs
|
|
from app.services.convert.layout.paragraphs import measure as measure_page
|
|
from app.services.convert.layout.reading_order import order_lines
|
|
from app.services.convert.layout.tables import (
|
|
continue_table_across_pages,
|
|
extract_tables,
|
|
extract_vertical_rulings_from_display_list,
|
|
)
|
|
from app.services.convert.layout.tables_rects import extract_rects_from_display_list
|
|
from app.services.convert.layout.text_quality import (
|
|
filename_suggests_arabic,
|
|
page_text_needs_ocr,
|
|
)
|
|
from app.services.convert.options import LayoutMode, OcrPolicy, get_options
|
|
from app.services.convert.pdf_bridge import open_pdf_bytes
|
|
from app.services.convert.validation import convert_timeout_seconds, run_with_timeout
|
|
|
|
# The pipeline's own deadline as a fraction of the hard timeout. The gap is
|
|
# what pays for finishing the document — merging tables, tagging headers and
|
|
# footers, formatting — after OCR stops early.
|
|
SOFT_DEADLINE_RATIO = 0.80
|
|
|
|
|
|
def _record_auto_policy(document: Document) -> None:
|
|
"""Log the automatic routing decision on the document and its warnings.
|
|
|
|
Support needs an answer to "why did page 7 come out as a picture?", and the
|
|
customer never chose any of this, so the reasoning has to live with the
|
|
output rather than in a request nobody kept.
|
|
"""
|
|
from app.services.convert import auto_policy
|
|
|
|
decision = auto_policy.get_active_decision()
|
|
if decision is None:
|
|
return
|
|
meta = document.meta if document.meta is not None else {}
|
|
meta["convert_policy"] = decision.as_dict()
|
|
document.meta = meta
|
|
document.warnings.insert(0, decision.summary())
|
|
|
|
|
|
def _recognition_passes() -> float:
|
|
"""How many recognition passes a page costs.
|
|
|
|
RapidOCR runs once per loaded script model. With the Arabic weights
|
|
present every page is recognised twice, which the OCR time budget has to
|
|
allow for or it stops partway through a bilingual document.
|
|
"""
|
|
try:
|
|
from app.services import ocr as ocr_service
|
|
|
|
return 2.0 if ocr_service.is_ocr_arabic_available() else 1.0
|
|
except Exception: # budgeting must never fail a conversion
|
|
return 1.0
|
|
|
|
|
|
def _env_float(name: str, default: float) -> float:
|
|
try:
|
|
return float(os.environ.get(name, default))
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def set_cancel_check(fn) -> None:
|
|
"""Bind a soft-cancel callback for the current conversion.
|
|
|
|
Kept for callers and tests that manage the lifetime themselves; the
|
|
pipeline itself is driven by :func:`cancellation.conversion_scope`. The
|
|
token lives in a context variable, not a module global — a global is shared
|
|
by every thread in the process, so concurrent jobs would overwrite each
|
|
other's callback and cancelling one would abort another.
|
|
"""
|
|
cancellation.set_cancel_check(fn)
|
|
|
|
|
|
def _raise_if_cancelled(stage: str = "layout") -> None:
|
|
"""Stop at a page boundary if the job was cancelled or ran out of time."""
|
|
cancellation.check(stage)
|
|
|
|
|
|
def _glyphs_from_engine_page(page) -> list[dict]:
|
|
glyphs: list[dict] = []
|
|
try:
|
|
raw = page.extract_text_with_bounds()
|
|
except Exception:
|
|
raw = None
|
|
if raw:
|
|
for g in raw:
|
|
if isinstance(g, dict):
|
|
d = dict(g)
|
|
if float(d.get("fontSize") or 0) <= 2.0 and float(d.get("h") or 0) > 2.0:
|
|
d["fontSize"] = float(d["h"])
|
|
glyphs.append(d)
|
|
else:
|
|
h = float(getattr(g, "h", 0))
|
|
fs = float(getattr(g, "fontSize", h or 12))
|
|
if fs <= 2.0 and h > 2.0:
|
|
fs = h
|
|
glyphs.append(
|
|
{
|
|
"text": getattr(g, "text", str(g)),
|
|
"x": float(getattr(g, "x", 0)),
|
|
"y": float(getattr(g, "y", 0)),
|
|
"w": float(getattr(g, "w", 0)),
|
|
"h": h,
|
|
"fontSize": fs,
|
|
}
|
|
)
|
|
return glyphs
|
|
|
|
try:
|
|
model = page.extract_document_model()
|
|
for para in getattr(model, "paragraphs", []) or []:
|
|
for line in getattr(para, "lines", []) or []:
|
|
runs = getattr(line, "runs", []) or []
|
|
text = "".join(getattr(r, "text", "") or "" for r in runs)
|
|
if not text.strip():
|
|
continue
|
|
glyphs.append(
|
|
{
|
|
"text": text,
|
|
"x": float(getattr(line, "x", 0)),
|
|
"y": float(getattr(line, "y", 0)),
|
|
"w": float(getattr(line, "w", len(text) * 6)),
|
|
"h": float(getattr(line, "h", 12)),
|
|
"fontSize": float(getattr(runs[0], "font_size", 12) if runs else 12),
|
|
"fontName": str(getattr(runs[0], "font_name", "") if runs else ""),
|
|
}
|
|
)
|
|
except Exception:
|
|
pass
|
|
return glyphs
|
|
|
|
|
|
def _page_from_lines(
|
|
index: int,
|
|
width: float,
|
|
height: float,
|
|
kind: PageKind,
|
|
lines,
|
|
warnings: list[str],
|
|
*,
|
|
vertical_rulings: list[float] | None = None,
|
|
page_rects: list | None = None,
|
|
path_ops: list[dict] | None = None,
|
|
ml_regions: list | None = None,
|
|
page_png: bytes | None = None,
|
|
) -> Page:
|
|
if ml_regions:
|
|
from app.services.convert.layout.ml_merge import merge_regions_with_lines
|
|
|
|
blocks = merge_regions_with_lines(
|
|
lines,
|
|
ml_regions,
|
|
page_width=width,
|
|
page_index=index,
|
|
kind=kind,
|
|
warnings=warnings,
|
|
vertical_rulings=vertical_rulings,
|
|
page_rects=page_rects,
|
|
path_ops=path_ops,
|
|
page_png=page_png,
|
|
page_height=height,
|
|
)
|
|
if blocks:
|
|
if not any(b.plain_text().strip() or b.cells for b in blocks) and kind in (
|
|
PageKind.scan,
|
|
PageKind.hybrid,
|
|
PageKind.blank,
|
|
):
|
|
warnings.append(f"Page {index + 1}: low text; OCR rebuild may apply.")
|
|
return Page(index=index, width=width, height=height, kind=kind, blocks=blocks)
|
|
|
|
opts = get_options()
|
|
if opts.layout_mode == LayoutMode.nocolumns:
|
|
ordered = sorted(lines, key=lambda ln: (-float(ln.y), float(ln.x0)))
|
|
else:
|
|
ordered = order_lines(lines, width, page_height=height)
|
|
body = estimate_body_size(ordered)
|
|
if not opts.detect_tables:
|
|
table_blocks, remaining, conf = [], ordered, 0.0
|
|
else:
|
|
table_blocks, remaining, conf = extract_tables(
|
|
ordered,
|
|
start_order=0,
|
|
vertical_rulings=vertical_rulings,
|
|
page_rects=page_rects,
|
|
path_ops=path_ops,
|
|
)
|
|
if table_blocks:
|
|
# A line consumed by a table has to actually appear in it.
|
|
#
|
|
# Nothing enforced that. ``extract_tables`` withdraws every line it
|
|
# believes belongs to a grid and then emits the cells it managed to
|
|
# reconstruct, so a line whose text no cell reproduced was simply gone --
|
|
# and gone silently, because recall is computed over the whole document
|
|
# and a few missing field labels barely move it. Measured on the IRS 1040
|
|
# sample: splitting its address block into more lines made the cells
|
|
# collide, and both "Last name" labels, "Credits" and "jointly"
|
|
# disappeared with nothing added in their place.
|
|
#
|
|
# Orphans go back to the prose. A label that lands in a stray paragraph
|
|
# is untidy; a label that is absent is a form the reader cannot fill in,
|
|
# and the reader cannot even tell it is missing. The costs are not
|
|
# symmetric, so this errs towards keeping the text.
|
|
printed = _squash("\n".join(b.plain_text() for b in table_blocks))
|
|
kept_ids = {id(ln) for ln in remaining}
|
|
rescued = {
|
|
id(ln)
|
|
for ln in ordered
|
|
if id(ln) not in kept_ids
|
|
and ln.text.strip()
|
|
and _squash(ln.text) not in printed
|
|
}
|
|
if rescued:
|
|
remaining = [ln for ln in ordered if id(ln) in kept_ids or id(ln) in rescued]
|
|
warnings.append(
|
|
f"Page {index + 1}: {len(rescued)} line(s) the table reconstruction "
|
|
"dropped were kept as text."
|
|
)
|
|
if table_blocks and conf < 0.85:
|
|
warnings.append(f"Page {index + 1}: table confidence={conf:.2f} (heuristic).")
|
|
if conf and conf < 0.65 and not table_blocks:
|
|
warnings.append(f"Page {index + 1}: table_low_confidence={conf:.2f}; emitted paragraphs.")
|
|
# Tables used to be emitted first and the prose after, whatever the page
|
|
# looked like — so a report's heading and its opening paragraph appeared
|
|
# *below* a table printed further down the page, on every document with a
|
|
# table in it. Each block is anchored to the position of its earliest line
|
|
# in the reading order the XY-cut produced, and the page is emitted in that
|
|
# order, which keeps multi-column documents correct too.
|
|
at = {id(ln): i for i, ln in enumerate(ordered)}
|
|
kept = {id(ln) for ln in remaining}
|
|
consumed = [(i, ln) for i, ln in enumerate(ordered) if id(ln) not in kept]
|
|
|
|
def _table_anchor(block: Block) -> int:
|
|
box = block.bbox
|
|
if box is not None:
|
|
inside = [
|
|
i
|
|
for i, ln in consumed
|
|
if box.x - 2 <= (ln.x0 + ln.x1) / 2.0 <= box.x + box.w + 2
|
|
and box.y - 2 <= ln.y <= box.y + box.h + 2
|
|
]
|
|
if inside:
|
|
return min(inside)
|
|
return min((i for i, _ln in consumed), default=len(ordered))
|
|
|
|
anchored: list[tuple[int, Block]] = [(_table_anchor(tb), tb) for tb in table_blocks]
|
|
|
|
# Rebuild paragraphs from wrapped lines. Emitting one block per visual line
|
|
# produces a DOCX that cannot reflow; grouping restores editability.
|
|
metrics = measure_page(remaining, body_size=body, page_width=width)
|
|
pairs = lexical_hyphen_pairs("\n".join(ln.text for ln in remaining))
|
|
for run in group_paragraph_lines(remaining, metrics):
|
|
anchor = min((at.get(id(ln), len(ordered)) for ln in run), default=len(ordered))
|
|
anchored.append((anchor, lines_to_block(run, 0, body, lexical_pairs=pairs)))
|
|
|
|
blocks: list[Block] = []
|
|
for order, (_anchor, block) in enumerate(sorted(anchored, key=lambda pair: pair[0])):
|
|
block.reading_order = order
|
|
blocks.append(block)
|
|
if not blocks and kind in (PageKind.scan, PageKind.hybrid, PageKind.blank):
|
|
warnings.append(f"Page {index + 1}: low text; OCR rebuild may apply.")
|
|
return Page(index=index, width=width, height=height, kind=kind, blocks=blocks)
|
|
|
|
|
|
# Every kind of whitespace, including the newlines ``extract_text()`` inserts
|
|
# between visual lines. Used to compare a text length against a glyph walker's
|
|
# own character count, which carries no whitespace at all.
|
|
_WS_RE = re.compile(r"\s+")
|
|
|
|
|
|
def _squash(text: str) -> str:
|
|
"""Only the letters and digits of *text*, for presence tests that ignore layout.
|
|
|
|
Whitespace alone is not enough to strip. Rebuilding a row into cells drops
|
|
whatever separated the columns on the page, so the line ``"Item | Qty |
|
|
Price"`` becomes the cells ``Item``, ``Qty``, ``Price`` -- and a test that
|
|
kept the pipes concluded the row had been lost and emitted it a second time
|
|
as a paragraph beside the table it had just been read into. Dot leaders in a
|
|
form do the same thing. ``\\W`` is Unicode-aware, so Arabic and CJK text
|
|
survives this rather than reducing to the empty string and matching anything.
|
|
"""
|
|
return re.sub(r"[\W_]+", "", text, flags=re.UNICODE)
|
|
|
|
|
|
def _image_blocks_with_fallback(engine_page, index: int, reader) -> list[dict]:
|
|
"""Images on a page, asking pypdf when the engine reports none.
|
|
|
|
The engine only reports images it recognises as image XObjects. A page whose
|
|
picture is wrapped in a Form XObject — which is how most producers emit a
|
|
placed image — comes back with an empty list, coverage is measured as zero,
|
|
and :func:`classify_page` calls a full-page scan *blank*. A blank page is
|
|
then skipped by OCR, gets no figures, and reaches Word as an empty sheet.
|
|
|
|
pypdf resolves nested XObjects, so it sees those pictures. Asking it only
|
|
when the engine found nothing keeps the engine's richer geometry wherever
|
|
the engine works, and costs nothing on ordinary text pages.
|
|
"""
|
|
blocks = image_blocks_from_engine_page(engine_page)
|
|
if blocks:
|
|
return blocks
|
|
try:
|
|
if reader is not None and index < len(reader.pages):
|
|
return image_blocks_from_pypdf_page(reader.pages[index])
|
|
except Exception:
|
|
pass
|
|
return blocks
|
|
|
|
|
|
def _raster_has_marks(png_bytes: bytes) -> bool:
|
|
"""Whether a rendered page actually has ink on it.
|
|
|
|
The last word on "is this page empty": a page that renders to solid white
|
|
really is blank, whatever the classifier decided; anything else has content
|
|
the reader is entitled to see.
|
|
"""
|
|
if not png_bytes:
|
|
return False
|
|
try:
|
|
import io as _io
|
|
|
|
from PIL import Image
|
|
|
|
img = Image.open(_io.BytesIO(png_bytes)).convert("L")
|
|
img.thumbnail((240, 240))
|
|
lo, hi = img.getextrema()
|
|
# Near-white paper is ~255; any appreciably darker pixel is a mark.
|
|
return lo < 245 or hi < 245
|
|
except Exception:
|
|
# Unreadable raster is not evidence of emptiness.
|
|
return True
|
|
|
|
|
|
def _page_has_visual_content(
|
|
page: Page, document: Document, reader, pdf_bytes: bytes
|
|
) -> tuple[bool, bytes | None]:
|
|
"""Is there something on this page the reader should still get?
|
|
|
|
Returns the verdict plus the page raster when one was rendered, so the
|
|
caller can embed the very image the decision was made from.
|
|
|
|
Classification is not trusted here. A page reaches this function only after
|
|
reconstruction produced nothing, which is itself a sign that something
|
|
upstream misread it, so the question is answered from evidence about the
|
|
source rather than from the label the page was given.
|
|
|
|
The rendered page is the arbiter, and it is consulted first. Every cheaper
|
|
signal is ambiguous in exactly the case that matters: the router reports a
|
|
*blank* one-page PDF as ``image_based`` with the page needing OCR, because
|
|
"no extractable text" looks identical whether the page is a photograph or
|
|
is empty. Only the pixels distinguish them. The cheap signals are kept for
|
|
the case where the page cannot be rendered at all.
|
|
"""
|
|
png = _render_page_for_fallback(pdf_bytes, page.index)
|
|
if png is not None:
|
|
return _raster_has_marks(png), png
|
|
|
|
# Could not render. Fall back to what the parsers saw, and lean towards
|
|
# "there is something here": a page wrongly kept is a warning, a page
|
|
# wrongly dropped is the customer's content gone without a word.
|
|
meta = document.meta or {}
|
|
route = meta.get("pdf_route") or {}
|
|
if page.index in set(meta.get("encoding_broken_pages") or ()):
|
|
return True, None
|
|
try:
|
|
if reader is not None and page.index < len(reader.pages):
|
|
if image_blocks_from_pypdf_page(reader.pages[page.index]):
|
|
return True, None
|
|
except Exception:
|
|
pass
|
|
if str(route.get("doc_type") or "") in ("image_based", "scanned"):
|
|
return True, None
|
|
return False, None
|
|
|
|
|
|
def _render_page_for_fallback(pdf_bytes: bytes, index: int) -> bytes | None:
|
|
try:
|
|
from app.services.convert.layout.page_raster import raster_page
|
|
|
|
# Through the document cache: on a scanned page OCR has already
|
|
# rendered this image, so this is a lookup, not a second render.
|
|
return doc_cache.page_raster(
|
|
pdf_bytes, index, 150, lambda: raster_page(pdf_bytes, index, dpi=150)
|
|
)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _page_is_empty_of_content(page: Page) -> bool:
|
|
"""No text and no picture — the page would arrive in Word as a blank sheet."""
|
|
for block in page.blocks:
|
|
if block.plain_text().strip():
|
|
return False
|
|
if getattr(block, "image_png", None):
|
|
return False
|
|
if block.type == BlockType.table and block.cells:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _embed_unreconstructable_pages(document: Document, pdf_bytes: bytes) -> None:
|
|
"""A page that reconstructed to nothing gets its own picture, or a warning.
|
|
|
|
This is the floor under the whole automatic-routing promise. Routing picks
|
|
the best in-house path, but some pages have no path: a raster with no text
|
|
layer converted under ``ocr_policy=never``, a page whose OCR engine is
|
|
missing, a vector-art page with no glyphs at all. Before this pass those
|
|
pages arrived in Word as blank sheets — the customer's page silently gone,
|
|
with nothing in the document saying so.
|
|
|
|
The rule is the honest one: show the page as it looked, or say why it is
|
|
missing. Never invent a layout for it.
|
|
|
|
A genuinely blank source page stays blank; embedding a white rectangle for
|
|
it would be noise, not fidelity. But "blank" here means *evidence* of
|
|
emptiness, never the ``PageKind.blank`` label alone: a full-page picture
|
|
wrapped in a Form XObject is routinely mislabelled blank, and trusting that
|
|
label is precisely how a scanned page became an empty Word sheet.
|
|
"""
|
|
empty = [p for p in document.pages if _page_is_empty_of_content(p)]
|
|
if not empty:
|
|
return
|
|
|
|
try:
|
|
reader = doc_cache.get_reader(pdf_bytes)
|
|
except Exception:
|
|
reader = None
|
|
|
|
for page in empty:
|
|
has_content, png = _page_has_visual_content(page, document, reader, pdf_bytes)
|
|
if not has_content:
|
|
# Solid white with nothing in the source saying otherwise. The
|
|
# faithful reproduction of a blank page is a blank page.
|
|
continue
|
|
if png is None:
|
|
png = _render_page_for_fallback(pdf_bytes, page.index)
|
|
if not png:
|
|
document.warnings.append(
|
|
f"Page {page.index + 1}: could not be reconstructed and no page image "
|
|
"was available; the page is empty in the output."
|
|
)
|
|
continue
|
|
page.blocks.append(
|
|
Block(
|
|
type=BlockType.figure,
|
|
text=f"[Page {page.index + 1} image]",
|
|
bbox=BBox(x=0.0, y=0.0, w=float(page.width or 612.0), h=float(page.height or 792.0)),
|
|
image_png=png,
|
|
reading_order=max((b.reading_order for b in page.blocks), default=-1) + 1,
|
|
)
|
|
)
|
|
document.warnings.append(
|
|
f"Page {page.index + 1}: no text could be reconstructed; embedded the page "
|
|
"image so the page is not lost. It is a picture, not editable text."
|
|
)
|
|
|
|
|
|
# A flowing DOCX is valuable only when the source really has a flow. Pages
|
|
# made from many short, positioned fragments are usually multi-column forms,
|
|
# tables, brochures, or desktop-published instructions. Reflowing those
|
|
# fragments produces wrong reading order, blank continuation pages, and global
|
|
# headers made from unrelated page furniture. For an entire document of this
|
|
# shape, a page-sized image is the honest high-visual-fidelity fallback.
|
|
_DOCX_RASTER_MIN_PAGES = 4
|
|
_DOCX_RASTER_MIN_BLOCKS = 14
|
|
_DOCX_RASTER_MIN_TEXT_BLOCKS = 10
|
|
_DOCX_RASTER_SHORT_TEXT_RATIO = 0.60
|
|
_DOCX_RASTER_MIN_COMPLEX_SHARE = 0.60
|
|
# Preflight uses the PDF content stream instead of waiting for the full IDM
|
|
# reconstruction. These values identify a page made from many individually
|
|
# positioned labels/lines, rather than ordinary flowing report text.
|
|
_DOCX_RASTER_PREFLIGHT_MIN_PAGES = 10
|
|
_DOCX_RASTER_PREFLIGHT_SAMPLE_PAGES = 24
|
|
_DOCX_RASTER_PREFLIGHT_TEXT_OPS = 200
|
|
_DOCX_RASTER_PREFLIGHT_GRAPHIC_OPS = 60
|
|
# 75% requires a strong majority of complex pages before giving up
|
|
# editability. The old 60% cutoff was too aggressive: IRS instruction PDFs
|
|
# have many text-show ops per page (each line is a separate positioned
|
|
# block) but the text IS extractable and valuable as editable content.
|
|
_DOCX_RASTER_PREFLIGHT_COMPLEX_SHARE = 0.75
|
|
# Minimum average extractable words per page for the text-density bypass.
|
|
# When a PDF carries this much readable text, reconstruction is preferred
|
|
# over page images regardless of content-stream complexity.
|
|
_DOCX_RASTER_TEXT_DENSITY_BYPASS = 80
|
|
|
|
|
|
def _docx_page_raster_mode() -> str:
|
|
"""Return ``off``, ``auto``, or ``always`` for the DOCX safety fallback."""
|
|
raw = (os.environ.get("CONVERT_DOCX_PAGE_RASTER_FALLBACK") or "auto").strip().lower()
|
|
if raw in {"0", "false", "off", "no", "never"}:
|
|
return "off"
|
|
if raw in {"1", "true", "on", "yes", "always", "force"}:
|
|
return "always"
|
|
return "auto"
|
|
|
|
|
|
def _page_has_fragmented_positioned_layout(page: Page) -> bool:
|
|
"""Whether flowing this page is likely to invent a document layout.
|
|
|
|
The signal deliberately requires both *many* blocks and *short* measured
|
|
text blocks. A normal report can have a long paragraph split over several
|
|
blocks, while a table of contents or tax instruction page has dozens of
|
|
small labels spread around two or more columns. Synthetic geometry is
|
|
rejected: it cannot describe a page well enough to justify a visual claim.
|
|
"""
|
|
body = [
|
|
b
|
|
for b in page.blocks
|
|
if b.type not in (BlockType.header, BlockType.footer)
|
|
and (b.plain_text().strip() or b.image_png or b.cells)
|
|
]
|
|
if len(body) < _DOCX_RASTER_MIN_BLOCKS:
|
|
return False
|
|
|
|
text_blocks = [
|
|
b
|
|
for b in body
|
|
if b.plain_text().strip() and b.type not in (BlockType.figure, BlockType.table)
|
|
]
|
|
if len(text_blocks) < _DOCX_RASTER_MIN_TEXT_BLOCKS:
|
|
return False
|
|
|
|
measured = [
|
|
b
|
|
for b in text_blocks
|
|
if not getattr(b, "synthetic_geometry", False)
|
|
and float(getattr(b.bbox, "w", 0) or 0) > 6.0
|
|
and float(getattr(b.bbox, "h", 0) or 0) > 6.0
|
|
]
|
|
if len(measured) / len(text_blocks) < 0.75:
|
|
return False
|
|
|
|
short = sum(1 for b in measured if len(b.plain_text().strip()) <= 110)
|
|
return short / len(measured) >= _DOCX_RASTER_SHORT_TEXT_RATIO
|
|
|
|
|
|
def _reconstructed_words_per_page(document: Document) -> float:
|
|
"""Editable words the reconstruction actually recovered, per page.
|
|
|
|
The preflight has to guess from the source content stream. By the time
|
|
this runs the reconstruction exists, so the question "is the editable
|
|
version worth keeping?" can be answered by measuring it rather than by
|
|
predicting it.
|
|
"""
|
|
if not document.pages:
|
|
return 0.0
|
|
words = 0
|
|
for page in document.pages:
|
|
for block in page.blocks:
|
|
if block.type is BlockType.figure:
|
|
continue
|
|
if block.cells:
|
|
words += sum(len(str(c).split()) for row in block.cells for c in row)
|
|
words += len(block.plain_text().split())
|
|
return words / len(document.pages)
|
|
|
|
|
|
def _document_has_unsafe_flow_layout(document: Document) -> bool:
|
|
"""Require a document-wide pattern before giving up editability."""
|
|
if document.page_count < _DOCX_RASTER_MIN_PAGES:
|
|
return False
|
|
if (document.meta or {}).get("geometry_degraded"):
|
|
# The fallback needs a working renderer; without measured geometry this
|
|
# heuristic would be guessing, exactly what the fallback is designed
|
|
# not to do.
|
|
return False
|
|
# Same bypass the preflight applies, on better evidence. A tax
|
|
# instruction booklet trips the fragmentation heuristic on every page —
|
|
# each line of a two-column layout is its own positioned block — while
|
|
# carrying exactly the dense prose and tables a customer opens a DOCX to
|
|
# edit. Without this the preflight and the post-hoc check disagree about
|
|
# the same document, and the post-hoc one silently wins.
|
|
if _reconstructed_words_per_page(document) >= _DOCX_RASTER_TEXT_DENSITY_BYPASS:
|
|
return False
|
|
complex_pages = sum(_page_has_fragmented_positioned_layout(p) for p in document.pages)
|
|
threshold = max(3, round(document.page_count * _DOCX_RASTER_MIN_COMPLEX_SHARE))
|
|
return complex_pages >= threshold
|
|
|
|
|
|
def _preflight_sample_indices(page_count: int) -> list[int]:
|
|
"""Evenly distributed indices, bounded so preflight stays inexpensive."""
|
|
if page_count <= _DOCX_RASTER_PREFLIGHT_SAMPLE_PAGES:
|
|
return list(range(page_count))
|
|
last = page_count - 1
|
|
return sorted(
|
|
{
|
|
round(position * last / (_DOCX_RASTER_PREFLIGHT_SAMPLE_PAGES - 1))
|
|
for position in range(_DOCX_RASTER_PREFLIGHT_SAMPLE_PAGES)
|
|
}
|
|
)
|
|
|
|
|
|
def should_prefer_docx_page_rasters(pdf_bytes: bytes) -> bool:
|
|
"""Fast, conservative preflight for visual-fidelity DOCX output.
|
|
|
|
This prevents a known bad path: first spending minutes reconstructing a
|
|
dense desktop-published PDF, then discovering the reconstruction cannot
|
|
preserve its columns and replacing it with page images anyway. It inspects
|
|
a bounded sample of PDF drawing operations; normal reports have a small
|
|
number of text-show operations per page, whereas forms/instruction layouts
|
|
paint hundreds of separately positioned labels and rules.
|
|
|
|
A text-density bypass ensures PDFs with substantial extractable text are
|
|
always reconstructed as editable content, even when the content stream
|
|
looks complex. IRS instruction manuals, government reports, and
|
|
multi-column technical documents have many text-show ops per page but
|
|
carry real prose that the customer expects to edit.
|
|
"""
|
|
mode = _docx_page_raster_mode()
|
|
if mode == "off":
|
|
return False
|
|
try:
|
|
reader = doc_cache.get_reader(pdf_bytes)
|
|
page_count = len(reader.pages)
|
|
except Exception:
|
|
return False
|
|
if mode == "always":
|
|
return page_count > 0
|
|
if page_count < _DOCX_RASTER_PREFLIGHT_MIN_PAGES:
|
|
return False
|
|
|
|
# Text-density bypass: if the PDF has rich extractable text, prefer
|
|
# reconstruction over page images. A document averaging >=80 words per
|
|
# page has enough prose to make an editable DOCX valuable.
|
|
try:
|
|
total_words = 0
|
|
sample = _preflight_sample_indices(page_count)
|
|
for idx in sample:
|
|
text = doc_cache.page_text(pdf_bytes, idx)
|
|
total_words += len(text.split())
|
|
avg_words = total_words / max(len(sample), 1)
|
|
if avg_words >= _DOCX_RASTER_TEXT_DENSITY_BYPASS:
|
|
return False
|
|
except Exception:
|
|
pass # text probe failure does not block the content-stream check
|
|
|
|
try:
|
|
from pypdf.generic import ContentStream
|
|
|
|
complex_pages = 0
|
|
indices = _preflight_sample_indices(page_count)
|
|
for index in indices:
|
|
operations = ContentStream(reader.pages[index].get_contents(), reader).operations
|
|
text_ops = sum(op in (b"Tj", b"TJ") for _operands, op in operations)
|
|
graphic_ops = sum(
|
|
op in (b"re", b"m", b"l", b"S", b"s", b"f", b"F", b"f*")
|
|
for _operands, op in operations
|
|
)
|
|
if text_ops >= _DOCX_RASTER_PREFLIGHT_TEXT_OPS and (
|
|
graphic_ops >= _DOCX_RASTER_PREFLIGHT_GRAPHIC_OPS
|
|
or text_ops >= _DOCX_RASTER_PREFLIGHT_TEXT_OPS * 2
|
|
):
|
|
complex_pages += 1
|
|
return complex_pages / max(len(indices), 1) >= _DOCX_RASTER_PREFLIGHT_COMPLEX_SHARE
|
|
except Exception:
|
|
# Preflight is an optimisation/safety choice, never a conversion
|
|
# failure. The normal IDM route remains available.
|
|
return False
|
|
|
|
|
|
def build_docx_page_raster_document(pdf_bytes: bytes) -> Document | None:
|
|
"""Build an all-page visual DOCX IDM without reconstructing text layout.
|
|
|
|
Returns ``None`` if even one nonblank source page cannot be rendered, so
|
|
the caller can safely take the normal editable reconstruction path instead
|
|
of returning a partial document.
|
|
"""
|
|
try:
|
|
reader = doc_cache.get_reader(pdf_bytes)
|
|
except Exception:
|
|
return None
|
|
|
|
pages: list[Page] = []
|
|
for index, source_page in enumerate(reader.pages):
|
|
try:
|
|
width = float(source_page.mediabox.width)
|
|
height = float(source_page.mediabox.height)
|
|
except Exception:
|
|
width, height = 612.0, 792.0
|
|
png = _render_page_for_fallback(pdf_bytes, index)
|
|
if png and _raster_has_marks(png):
|
|
blocks = [
|
|
Block(
|
|
type=BlockType.figure,
|
|
bbox=BBox(x=0.0, y=0.0, w=width, h=height),
|
|
image_png=png,
|
|
reading_order=0,
|
|
source=PAGE_RASTER_SOURCE,
|
|
)
|
|
]
|
|
elif png and not _raster_has_marks(png):
|
|
blocks = []
|
|
else:
|
|
return None
|
|
pages.append(Page(index=index, width=width, height=height, blocks=blocks))
|
|
|
|
if not pages:
|
|
return None
|
|
document = Document(
|
|
pages=pages,
|
|
warnings=[
|
|
f"Complex source layout preserved as {sum(bool(p.blocks) for p in pages)} "
|
|
"page image(s) in DOCX; visual fidelity is retained, but those pages are not editable text."
|
|
],
|
|
meta={
|
|
"docx_page_raster_fallback": "preflight-full",
|
|
"docx_page_raster_pages": sum(bool(p.blocks) for p in pages),
|
|
},
|
|
)
|
|
return document
|
|
|
|
|
|
def apply_docx_page_raster_fallback(document: Document, pdf_bytes: bytes) -> bool:
|
|
"""Replace a whole unsafe PDF layout with faithful, page-sized DOCX images.
|
|
|
|
This is deliberately all-or-nothing. A mixed document would still need a
|
|
global Word header/footer and different margins, which can duplicate the
|
|
raster's own furniture or create page-count drift. If any non-blank source
|
|
page cannot be rendered, the original editable reconstruction is retained.
|
|
"""
|
|
mode = _docx_page_raster_mode()
|
|
if mode == "off" or not document.pages:
|
|
return False
|
|
if mode == "auto" and not _document_has_unsafe_flow_layout(document):
|
|
return False
|
|
|
|
# One Word section cannot faithfully represent a mixed-size/orientation
|
|
# source. Keep the normal reconstruction rather than silently rescaling it.
|
|
first = document.pages[0]
|
|
if any(
|
|
abs(float(p.width) - float(first.width)) > 1.0
|
|
or abs(float(p.height) - float(first.height)) > 1.0
|
|
for p in document.pages[1:]
|
|
):
|
|
document.warnings.append(
|
|
"DOCX faithful-page fallback skipped: source pages have mixed sizes."
|
|
)
|
|
return False
|
|
|
|
original_blocks = {id(page): page.blocks for page in document.pages}
|
|
rendered_pages = 0
|
|
try:
|
|
for page in document.pages:
|
|
png = _render_page_for_fallback(pdf_bytes, page.index)
|
|
if png and _raster_has_marks(png):
|
|
page.blocks = [
|
|
Block(
|
|
type=BlockType.figure,
|
|
bbox=BBox(
|
|
x=0.0,
|
|
y=0.0,
|
|
w=float(page.width or first.width),
|
|
h=float(page.height or first.height),
|
|
),
|
|
image_png=png,
|
|
reading_order=0,
|
|
source=PAGE_RASTER_SOURCE,
|
|
)
|
|
]
|
|
rendered_pages += 1
|
|
continue
|
|
|
|
# A genuinely blank page stays blank. Any other failed page
|
|
# aborts the fallback so customer content is never replaced with a
|
|
# white rectangle or a partial document.
|
|
if _page_is_empty_of_content(page):
|
|
page.blocks = []
|
|
continue
|
|
raise RuntimeError(f"page {page.index + 1} could not be rasterized")
|
|
except Exception as exc:
|
|
for page in document.pages:
|
|
page.blocks = original_blocks[id(page)]
|
|
document.warnings.append(
|
|
f"DOCX faithful-page fallback unavailable; kept editable reconstruction ({exc})."
|
|
)
|
|
return False
|
|
|
|
if not rendered_pages:
|
|
for page in document.pages:
|
|
page.blocks = original_blocks[id(page)]
|
|
return False
|
|
|
|
meta = document.meta if document.meta is not None else {}
|
|
meta["docx_page_raster_fallback"] = "full"
|
|
meta["docx_page_raster_pages"] = rendered_pages
|
|
document.meta = meta
|
|
document.warnings.append(
|
|
f"Complex source layout preserved as {rendered_pages} page image(s) in DOCX; "
|
|
"visual fidelity is retained, but those pages are not editable text."
|
|
)
|
|
return True
|
|
|
|
|
|
def _wants_page_raster() -> bool:
|
|
"""Whether any consumer needs the page image kept after region detection."""
|
|
from app.services.convert.layout.ml_table_structure import table_structure_enabled
|
|
|
|
return table_structure_enabled()
|
|
|
|
|
|
def _try_ml_regions(
|
|
pdf_bytes: bytes,
|
|
page_index: int,
|
|
width: float,
|
|
height: float,
|
|
warnings: list[str],
|
|
table_regions_acc: dict[int, list[list[float]]],
|
|
raster_acc: dict[int, bytes] | None = None,
|
|
) -> list | None:
|
|
from app.services.convert.layout.ml_regions import detect_regions, layout_ml_enabled
|
|
from app.services.convert.layout.page_raster import raster_page
|
|
|
|
if not layout_ml_enabled():
|
|
return None
|
|
try:
|
|
png = raster_page(pdf_bytes, page_index)
|
|
regions, err = detect_regions(png, width, height)
|
|
if err:
|
|
warnings.append(f"Page {page_index + 1}: layout_ml_fallback=heuristic ({err})")
|
|
return None
|
|
warnings.append(f"Page {page_index + 1}: layout_ml=onnx regions={len(regions)}")
|
|
table_regions_acc[page_index] = [
|
|
r.bbox_pdf.as_list() for r in regions if r.label == "table"
|
|
]
|
|
# Keep the raster only when a consumer asked for it: the optional table
|
|
# structure pass reuses this page image instead of rendering it again.
|
|
if raster_acc is not None:
|
|
raster_acc[page_index] = png
|
|
return regions
|
|
except Exception as exc:
|
|
warnings.append(f"Page {page_index + 1}: layout_ml_fallback=heuristic ({exc})")
|
|
return None
|
|
|
|
|
|
def _merge_multipage_tables(document: Document, warnings: list[str]) -> None:
|
|
for i in range(1, len(document.pages)):
|
|
prev = document.pages[i - 1]
|
|
cur = document.pages[i]
|
|
prev_tables = [b for b in prev.blocks if b.type == BlockType.table and b.cells]
|
|
if not prev_tables:
|
|
continue
|
|
from app.services.convert.layout.glyphs import Line
|
|
|
|
probe: list[Line] = []
|
|
for b in sorted(cur.blocks, key=lambda x: x.reading_order):
|
|
if b.type == BlockType.table:
|
|
break
|
|
t = b.plain_text().strip()
|
|
if t:
|
|
probe.append(
|
|
Line(text=t, x0=b.bbox.x, x1=b.bbox.x + b.bbox.w, y=b.bbox.y, font_size=11)
|
|
)
|
|
if not probe:
|
|
continue
|
|
updated, remaining = continue_table_across_pages(prev_tables[-1], probe)
|
|
if updated is None:
|
|
continue
|
|
for idx, b in enumerate(prev.blocks):
|
|
if b is prev_tables[-1]:
|
|
prev.blocks[idx] = updated
|
|
break
|
|
consumed_texts = {ln.text for ln in probe} - {ln.text for ln in remaining}
|
|
cur.blocks = [
|
|
b
|
|
for b in cur.blocks
|
|
if b.type == BlockType.table or b.plain_text().strip() not in consumed_texts
|
|
]
|
|
warnings.append(f"Pages {i}/{i + 1}: continued multi-page table.")
|
|
|
|
|
|
def _build_idm_from_pdf_impl(
|
|
data: bytes, *, apply_ocr: bool = True, filename: str | None = None
|
|
) -> Document:
|
|
|
|
from app.services.convert.layout.pdf_router import route_pdf
|
|
|
|
warnings: list[str] = []
|
|
pages: list[Page] = []
|
|
table_regions_acc: dict[int, list[list[float]]] = {}
|
|
encoding_broken_pages: list[int] = []
|
|
reader = doc_cache.get_reader(data)
|
|
# Pages that fell all the way back to fabricated line geometry, which is
|
|
# the only case where the old "geometry degraded" warning still holds.
|
|
degraded_pages = 0
|
|
|
|
opts = get_options()
|
|
pdf_route = route_pdf(data)
|
|
route_needing = set(pdf_route.pages_needing_ocr)
|
|
if opts.ocr_policy == OcrPolicy.never:
|
|
route_needing = set()
|
|
elif opts.ocr_policy == OcrPolicy.force:
|
|
n_hint = doc_cache.page_count(data) or 0
|
|
route_needing = set(range(max(n_hint, 1)))
|
|
|
|
def _pypdf_plain(i: int) -> str:
|
|
try:
|
|
if i < len(reader.pages):
|
|
return doc_cache.page_text(data, i)
|
|
except Exception:
|
|
pass
|
|
return ""
|
|
|
|
def _best_plain(engine_plain: str, i: int) -> str:
|
|
alt = _pypdf_plain(i)
|
|
if len(alt.strip()) > len((engine_plain or "").strip()):
|
|
return alt
|
|
return engine_plain or alt
|
|
|
|
from app.services.convert.layout.ml_regions import layout_ml_enabled
|
|
|
|
if layout_ml_enabled():
|
|
warnings.append("layout_ml=onnx")
|
|
else:
|
|
warnings.append("layout_ml=off")
|
|
|
|
doc = open_pdf_bytes(data)
|
|
if doc is not None:
|
|
for i in range(doc.page_count):
|
|
_raise_if_cancelled(f"layout page {i + 1}")
|
|
page = doc.get_page(i)
|
|
width = float(page.width)
|
|
height = float(page.height)
|
|
glyphs = _glyphs_from_engine_page(page)
|
|
text_chars = sum(len(str(g.get("text", ""))) for g in glyphs)
|
|
try:
|
|
engine_plain = page.extract_text() or ""
|
|
except Exception:
|
|
engine_plain = ""
|
|
plain = _best_plain(engine_plain, i)
|
|
if len(plain.strip()) > text_chars:
|
|
text_chars = len(plain.strip())
|
|
img_blocks = _image_blocks_with_fallback(page, i, reader)
|
|
coverage = estimate_image_coverage(img_blocks, width, height)
|
|
kind = classify_page(
|
|
glyph_count=max(len(glyphs), len(plain.strip())),
|
|
text_chars=text_chars,
|
|
width=width,
|
|
height=height,
|
|
image_coverage=coverage,
|
|
glyph_size_variance=glyph_size_variance(glyphs),
|
|
)
|
|
# Doc-level router: OCR pages with little text; image+text stays hybrid (keep figures)
|
|
if i in route_needing and kind == PageKind.digital:
|
|
if coverage >= 0.15 or text_chars >= 12:
|
|
kind = PageKind.hybrid
|
|
else:
|
|
kind = PageKind.scan
|
|
line_probe = plain.strip() or " ".join(
|
|
str(g.get("text", "")) for g in glyphs[:200]
|
|
)
|
|
if opts.ocr_policy != OcrPolicy.never and page_text_needs_ocr(
|
|
line_probe,
|
|
filename=filename,
|
|
image_coverage=coverage,
|
|
font_names=[str(g.get("fontName") or g.get("font_name") or "") for g in glyphs[:80]],
|
|
):
|
|
encoding_broken_pages.append(i)
|
|
kind = PageKind.scan
|
|
warnings.append(
|
|
f"Page {i + 1}: broken PDF text layer detected — forcing OCR rebuild."
|
|
)
|
|
elif opts.ocr_policy == OcrPolicy.force and kind == PageKind.digital:
|
|
kind = PageKind.hybrid if coverage >= 0.15 or text_chars >= 12 else PageKind.scan
|
|
dl_ops = display_list_ops_from_engine_page(page)
|
|
rulings = extract_vertical_rulings_from_display_list(dl_ops)
|
|
page_rects = extract_rects_from_display_list(dl_ops)
|
|
glyph_chars = sum(len(str(g.get("text", ""))) for g in glyphs)
|
|
plain_chars = len(plain.strip())
|
|
# Prefer glyphs when multi-X geometry exists — plain extract often
|
|
# collapses columns and destroys table grids.
|
|
distinct_x = {
|
|
round(float(g.get("x", 0)), 0)
|
|
for g in glyphs
|
|
if str(g.get("text", "")).strip()
|
|
}
|
|
has_geo = len(distinct_x) >= 2
|
|
use_glyphs = bool(glyphs) and (
|
|
plain_chars == 0
|
|
or glyph_chars >= max(1, int(plain_chars * 0.85))
|
|
or (has_geo and glyph_chars >= max(1, int(plain_chars * 0.50)))
|
|
)
|
|
if use_glyphs:
|
|
lines = lines_from_glyphs(glyphs)
|
|
if not lines and plain.strip():
|
|
warnings.append(
|
|
f"Page {i + 1}: glyphs present but empty lines; using plain-text fallback."
|
|
)
|
|
lines = lines_from_plain_text(plain, height)
|
|
elif plain.strip():
|
|
if glyphs and glyph_chars < plain_chars:
|
|
warnings.append(
|
|
f"Page {i + 1}: glyph/engine text weaker than pypdf; using plain-text lines."
|
|
)
|
|
lines = lines_from_plain_text(plain, height)
|
|
else:
|
|
lines = lines_from_glyphs(glyphs)
|
|
if not lines:
|
|
lines = lines_from_plain_text(plain, height)
|
|
# One page's raster at a time, and only when something consumes it,
|
|
# so a 500-page document never holds 500 page images.
|
|
raster_acc: dict[int, bytes] = {} if _wants_page_raster() else None # type: ignore[assignment]
|
|
ml_regs = _try_ml_regions(
|
|
data, i, width, height, warnings, table_regions_acc, raster_acc
|
|
)
|
|
pg = _page_from_lines(
|
|
i,
|
|
width,
|
|
height,
|
|
kind,
|
|
lines,
|
|
warnings,
|
|
vertical_rulings=rulings or None,
|
|
page_rects=page_rects or None,
|
|
path_ops=dl_ops or None,
|
|
ml_regions=ml_regs,
|
|
page_png=(raster_acc or {}).get(i),
|
|
)
|
|
# Attach only small decorative images on digital/hybrid; never on OCR-forced pages
|
|
if (
|
|
kind in (PageKind.digital, PageKind.hybrid)
|
|
and i not in encoding_broken_pages
|
|
and i < len(reader.pages)
|
|
):
|
|
order = max((b.reading_order for b in pg.blocks), default=-1) + 1
|
|
figs = figure_blocks_from_pypdf_page(
|
|
reader.pages[i], reading_order=order, warnings=warnings
|
|
)
|
|
if figs:
|
|
pg.blocks.extend(figs)
|
|
pages.append(pg)
|
|
else:
|
|
# No C++ engine — the common case in a Linux container. Page geometry
|
|
# comes from pypdf instead of from a fabricated layout, so this branch
|
|
# still sees columns, indents, ruling lines and cell rectangles.
|
|
from app.services.convert.layout.pypdf_geometry import (
|
|
page_dimensions,
|
|
page_geometry,
|
|
)
|
|
|
|
for i, page in enumerate(reader.pages):
|
|
_raise_if_cancelled(f"layout page {i + 1}")
|
|
width, height = page_dimensions(page)
|
|
plain = doc_cache.page_text(data, i)
|
|
# Non-whitespace, to match what the walker's own counter measures.
|
|
# See ``glyphs_from_page`` for why the raw length silently demoted
|
|
# ordinary two-column pages to estimated-width geometry.
|
|
glyphs, path_ops = doc_cache.page_geometry(
|
|
data,
|
|
i,
|
|
lambda page=page, n=len(_WS_RE.sub("", plain)): page_geometry(
|
|
page, reader, plain_chars=n
|
|
),
|
|
)
|
|
img_blocks = image_blocks_from_pypdf_page(page)
|
|
coverage = estimate_image_coverage(img_blocks, width, height)
|
|
kind = classify_page(
|
|
glyph_count=len(plain),
|
|
text_chars=len(plain.strip()),
|
|
width=width,
|
|
height=height,
|
|
image_coverage=coverage,
|
|
)
|
|
if opts.ocr_policy != OcrPolicy.never and page_text_needs_ocr(
|
|
plain,
|
|
filename=filename,
|
|
image_coverage=coverage,
|
|
font_names=[str(g.get("fontName") or "") for g in glyphs[:80]],
|
|
):
|
|
encoding_broken_pages.append(i)
|
|
kind = PageKind.scan
|
|
warnings.append(
|
|
f"Page {i + 1}: broken PDF text layer detected — forcing OCR rebuild."
|
|
)
|
|
elif opts.ocr_policy == OcrPolicy.force and kind == PageKind.digital:
|
|
kind = PageKind.scan
|
|
lines = lines_from_glyphs(glyphs) if glyphs else []
|
|
if not lines and plain.strip():
|
|
lines = lines_from_plain_text(plain, height)
|
|
degraded_pages += 1
|
|
rulings = extract_vertical_rulings_from_display_list(path_ops)
|
|
page_rects = extract_rects_from_display_list(path_ops)
|
|
raster_acc: dict[int, bytes] = {} if _wants_page_raster() else None # type: ignore[assignment]
|
|
ml_regs = _try_ml_regions(
|
|
data, i, width, height, warnings, table_regions_acc, raster_acc
|
|
)
|
|
pg = _page_from_lines(
|
|
i,
|
|
width,
|
|
height,
|
|
kind,
|
|
lines,
|
|
warnings,
|
|
vertical_rulings=rulings or None,
|
|
page_rects=page_rects or None,
|
|
path_ops=path_ops or None,
|
|
ml_regions=ml_regs,
|
|
page_png=(raster_acc or {}).get(i),
|
|
)
|
|
if (
|
|
kind in (PageKind.digital, PageKind.hybrid)
|
|
and i not in encoding_broken_pages
|
|
):
|
|
order = max((b.reading_order for b in pg.blocks), default=-1) + 1
|
|
figs = figure_blocks_from_pypdf_page(
|
|
page, reading_order=order, warnings=warnings
|
|
)
|
|
if figs:
|
|
pg.blocks.extend(figs)
|
|
pages.append(pg)
|
|
|
|
meta = {"engine": "layout.v3", "pdf_route": pdf_route.as_dict()}
|
|
if filename:
|
|
meta["source_filename"] = filename
|
|
if table_regions_acc:
|
|
meta["ml_table_regions_by_page"] = table_regions_acc
|
|
if encoding_broken_pages:
|
|
meta["encoding_broken_pages"] = encoding_broken_pages
|
|
try:
|
|
from app.services import ocr as ocr_service
|
|
|
|
ar_on = ocr_service.is_ocr_arabic_available()
|
|
except Exception:
|
|
ar_on = False
|
|
if ar_on:
|
|
warnings.insert(
|
|
0,
|
|
"PDF text encoding is broken/unusable on some pages — rebuilt via OCR "
|
|
"(lossy; dual-pass EN+AR RapidOCR when Arabic weights are loaded).",
|
|
)
|
|
else:
|
|
warnings.insert(
|
|
0,
|
|
"PDF text encoding is broken/unusable on some pages — rebuilt via OCR "
|
|
"(lossy; fetch PP-OCRv5 Arabic weights for bilingual script recovery).",
|
|
)
|
|
if filename_suggests_arabic(filename) and encoding_broken_pages:
|
|
try:
|
|
from app.services import ocr as ocr_service
|
|
|
|
ar_on = ocr_service.is_ocr_arabic_available()
|
|
except Exception:
|
|
ar_on = False
|
|
if ar_on:
|
|
warnings.append(
|
|
"Source filename is Arabic: body recovered via dual-pass EN+AR OCR "
|
|
"(lossy layout; Arabic character order may still be imperfect)."
|
|
)
|
|
else:
|
|
warnings.append(
|
|
"Source filename is Arabic: body English is OCR-recovered; Arabic glyphs in the "
|
|
"PDF text layer were not machine-readable."
|
|
)
|
|
if doc is None:
|
|
# The engine is absent, but geometry is no longer fabricated: pypdf
|
|
# supplies real span positions and path operators. Only pages that
|
|
# fell through to plain-text lines are genuinely degraded, and saying
|
|
# so is what keeps X-Warnings worth reading.
|
|
meta["geometry_source"] = "pypdf"
|
|
if degraded_pages:
|
|
meta["geometry_degraded"] = True
|
|
meta["geometry_degraded_pages"] = degraded_pages
|
|
warnings.insert(
|
|
0,
|
|
f"C++ PDF engine unavailable and {degraded_pages} page(s) exposed no glyph "
|
|
"geometry — those pages use plain-text layout (weaker tables/columns).",
|
|
)
|
|
else:
|
|
from app.services import raster
|
|
|
|
if raster.is_available():
|
|
meta["raster_source"] = raster.version()
|
|
warnings.insert(
|
|
0,
|
|
"C++ PDF engine unavailable — page geometry recovered from the content "
|
|
f"stream and pages rendered by {raster.version()}.",
|
|
)
|
|
else:
|
|
warnings.insert(
|
|
0,
|
|
"C++ PDF engine unavailable — page geometry recovered from the content "
|
|
"stream (no rasteriser: page images and OCR are unavailable).",
|
|
)
|
|
document = Document(pages=pages, warnings=warnings, meta=meta)
|
|
_merge_multipage_tables(document, warnings)
|
|
|
|
do_ocr = apply_ocr and opts.ocr_policy != OcrPolicy.never
|
|
if apply_ocr and opts.ocr_policy == OcrPolicy.never:
|
|
warnings.append("ocr_policy=never — skipped OCR (digital text layer kept).")
|
|
if do_ocr:
|
|
from app.services.convert.ocr.rebuild import enrich_scanned_pages
|
|
|
|
enrich_scanned_pages(document, data)
|
|
|
|
apply_document_wide(document)
|
|
optimize_document(document)
|
|
_embed_unreconstructable_pages(document, data)
|
|
_record_auto_policy(document)
|
|
|
|
maybe_dump_idm(document, stem="pdf")
|
|
return document
|
|
|
|
|
|
def build_idm_from_pdf(
|
|
data: bytes, *, apply_ocr: bool = True, filename: str | None = None
|
|
) -> Document:
|
|
timeout = convert_timeout_seconds(120.0)
|
|
if apply_ocr:
|
|
n_pages = doc_cache.page_count(data) or 1
|
|
# Broken-encoding bilingual RFPs need per-page OCR (~15-25s each).
|
|
page_budget = _env_float("CONVERT_OCR_PAGE_BUDGET", 22.0)
|
|
# A bilingual document runs the recogniser twice per page, once per
|
|
# script, so a budget calibrated on single-pass OCR covers only about
|
|
# half the document. Measured on a 26-page Arabic/English RFP the
|
|
# single-pass figure ran out after 19 pages and the remaining seven
|
|
# kept their broken text layer.
|
|
ocr_budget = 90.0 + max(1, n_pages) * page_budget * _recognition_passes()
|
|
timeout = max(timeout, ocr_budget)
|
|
# A long document must not be allowed to plan a run longer than the
|
|
# caller will wait: a 42-page scan asks for ~1000s, but an async job
|
|
# that gives up at 600s throws away every page already recovered.
|
|
wall_budget = _env_float("CONVERT_OCR_WALL_BUDGET_SECONDS", 1200.0)
|
|
if wall_budget > 0:
|
|
timeout = min(timeout, wall_budget)
|
|
timeout = min(timeout, _env_float("CONVERT_TIMEOUT_MAX_SECONDS", 1800.0))
|
|
|
|
# The inner deadline is deliberately shorter than the outer kill switch.
|
|
# Sharing one value makes them race: the pipeline's graceful stop and the
|
|
# hard timeout fire at the same instant, and the hard timeout discards the
|
|
# partial document the graceful stop was preserving.
|
|
soft_deadline = max(1.0, timeout * SOFT_DEADLINE_RATIO)
|
|
|
|
def _run() -> Document:
|
|
# The deadline is checked at page boundaries inside the impl, so a
|
|
# runaway document stops doing work rather than merely having its
|
|
# result discarded when the outer timeout fires.
|
|
with cancellation.deadline_scope(soft_deadline):
|
|
return _build_idm_from_pdf_impl(data, apply_ocr=apply_ocr, filename=filename)
|
|
|
|
# Fail closed — callers must not format empty OOXML as success.
|
|
return run_with_timeout(_run, timeout, label="layout/OCR pipeline")
|