pdf/gateway/app/services/convert/auto_policy.py

"""Automatic reconstruction policy — the engine picks the path, not the user.

The conversion desk asks for one thing only: the output format. Everything
else — whether to OCR, whether to reflow or position, how to treat running
headers, whether to look for tables — is decided here from what the document
actually is.

Why route first rather than run one universal path: a clean digital contract, a
photographed invoice and a bilingual RFP with a broken ToUnicode map need
different treatment, and applying any one of those treatments to the other two
makes the output worse. Routing costs a page scan; guessing costs the document.

Honest limits, stated the same way in the UI and the docs:

* Word and Excel are **reconstructions**. They stay labelled lossy.
* PNG/JPEG/TIFF of pages are exact rasters of what was uploaded.
* A page that cannot be reconstructed is embedded as a page image rather than
  having a layout invented for it.

Every decision is recorded on ``document.meta.convert_policy`` so a support
question ("why did page 7 come out as a picture?") has an answer.
"""

from __future__ import annotations

import os
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, field

from app.services.convert.layout.pdf_router import PdfDocType, PdfRouteResult
from app.services.convert.options import (
    ConvertOptions,
    HeaderFooterMode,
    LayoutMode,
    OcrPolicy,
    RecognitionMode,
)

# Targets that are reconstructions of the document's *content*.
OFFICE_TARGETS = frozenset({"docx", "xlsx", "csv", "pptx", "html", "md", "txt", "json"})
# Targets that reproduce the page as pixels; nothing here is reconstructed.
RASTER_TARGETS = frozenset({"png", "jpeg", "jpg", "tiff", "tif"})

# --- brochure detection -----------------------------------------------------
# A brochure-like page is mostly artwork with captions placed around it.
# Reflowing one produces a single mashed paragraph, so it is emitted positioned.
BROCHURE_IMAGE_COVERAGE = 0.45
# Below this many words a page is not carrying body prose.
BROCHURE_MAX_WORDS = 120
# Short positioned blocks per paragraph: a poster is many small labels, a report
# is a few long paragraphs.
BROCHURE_BLOCK_RATIO = 2.5
# The share of pages that must look like this before the *document* is treated
# as brochure-like. One art page in a report does not change the whole file.
BROCHURE_PAGE_SHARE = 0.6

# A page with no body prose at all does not need table detection; looking for
# grids in running prose is where soup tables come from.
PROSE_ONLY_MAX_TABLE_HINT = 0.02


def auto_enabled() -> bool:
    """Whether omitted knobs should be filled in by this module."""
    raw = (os.environ.get("CONVERT_AUTO_RECONSTRUCT") or "1").strip().lower()
    return raw not in ("0", "false", "off", "no")


@dataclass
class DocumentSignals:
    """What we know about the document before choosing a path."""

    doc_type: PdfDocType = PdfDocType.text_based
    confidence: float = 0.0
    page_count: int = 1
    pages_needing_ocr: list[int] = field(default_factory=list)
    # Mean fraction of page area covered by images.
    image_coverage: float = 0.0
    # Pages that are one full-page raster and nothing else.
    full_page_raster_pages: list[int] = field(default_factory=list)
    # Words of usable digital text across the document.
    text_words: int = 0
    # Positioned blocks vs paragraphs, averaged per page.
    block_to_paragraph_ratio: float = 0.0
    brochure_pages: list[int] = field(default_factory=list)

    @property
    def ocr_fraction(self) -> float:
        return len(self.pages_needing_ocr) / max(self.page_count, 1)

    @property
    def brochure_fraction(self) -> float:
        return len(self.brochure_pages) / max(self.page_count, 1)


@dataclass
class PolicyDecision:
    """The chosen options plus why, for logging and for X-Warnings."""

    options: ConvertOptions
    reasons: list[str] = field(default_factory=list)
    signals: DocumentSignals = field(default_factory=DocumentSignals)

    def as_dict(self) -> dict:
        return {
            "auto": True,
            "options": self.options.as_dict(),
            "reasons": list(self.reasons),
            "doc_type": self.signals.doc_type.value,
            "confidence": round(self.signals.confidence, 3),
            "pages_needing_ocr": len(self.signals.pages_needing_ocr),
            "page_count": self.signals.page_count,
            "image_coverage": round(self.signals.image_coverage, 3),
            "brochure_pages": len(self.signals.brochure_pages),
        }

    def summary(self) -> str:
        """One short line for X-Warnings — no internal jargon."""
        opts = self.options
        bits = [f"auto: {self.signals.doc_type.value}"]
        if opts.layout_mode != LayoutMode.flowing:
            bits.append(f"layout={opts.layout_mode.value}")
        if opts.ocr_policy != OcrPolicy.auto:
            bits.append(f"ocr={opts.ocr_policy.value}")
        if not opts.detect_tables:
            bits.append("tables=off")
        return "; ".join(bits)


def _decide_ocr(signals: DocumentSignals, reasons: list[str]) -> OcrPolicy:
    """OCR the pages that need it — never the whole document out of caution.

    ``auto`` already means "the rebuild step decides per page from
    pages_needing_ocr and the encoding-broken list", which is exactly the
    per-page behaviour wanted. It is escalated to ``force`` only when the
    document is so thoroughly broken that per-page detection would still leave
    corrupt text behind, and dropped to ``never`` when there is nothing to gain.
    """
    if signals.doc_type == PdfDocType.text_based and not signals.pages_needing_ocr:
        reasons.append("clean digital text layer: no OCR")
        return OcrPolicy.never
    if signals.ocr_fraction >= 0.95 and signals.page_count > 1:
        reasons.append(f"every page needs OCR ({signals.doc_type.value})")
        return OcrPolicy.force
    if signals.pages_needing_ocr:
        reasons.append(f"OCR {len(signals.pages_needing_ocr)}/{signals.page_count} pages")
    return OcrPolicy.auto


def _decide_layout(
    signals: DocumentSignals, target: str, reasons: list[str]
) -> tuple[LayoutMode, RecognitionMode]:
    """Flowing keeps the output editable; exact keeps a poster looking like one."""
    if target != "docx":
        # Only DOCX has a positioned emit. Everything else reflows by nature.
        return LayoutMode.flowing, RecognitionMode.enhanced_flow

    brochure = signals.brochure_fraction >= BROCHURE_PAGE_SHARE
    image_doc = signals.doc_type == PdfDocType.image_based
    if brochure or (image_doc and signals.text_words < BROCHURE_MAX_WORDS * signals.page_count):
        why = "image-led layout" if image_doc else "brochure-like pages"
        reasons.append(f"{why}: positioned emit so the design survives")
        return LayoutMode.exact, RecognitionMode.textbox

    return LayoutMode.flowing, RecognitionMode.enhanced_flow


def _decide_tables(signals: DocumentSignals, target: str, reasons: list[str]) -> bool:
    """Table detection off only where it can do nothing but harm."""
    if target in ("xlsx", "csv", "json"):
        # A spreadsheet target exists to carry grids.
        return True
    if signals.doc_type == PdfDocType.image_based and not signals.pages_needing_ocr:
        reasons.append("image-only document: table detection off")
        return False
    return True


def decide(
    signals: DocumentSignals, *, target: str, base: ConvertOptions | None = None
) -> PolicyDecision:
    """Choose reconstruction options for this document and target."""
    target = (target or "").lower()
    reasons: list[str] = []

    if target in RASTER_TARGETS:
        # Rasters reproduce the page; nothing below applies to them.
        options = base or ConvertOptions()
        return PolicyDecision(
            options=options,
            reasons=["raster target: page reproduced exactly, no reconstruction"],
            signals=signals,
        )

    ocr_policy = _decide_ocr(signals, reasons)
    layout_mode, recognition_mode = _decide_layout(signals, target, reasons)
    detect_tables = _decide_tables(signals, target, reasons)

    options = ConvertOptions(
        layout_mode=layout_mode,
        ocr_policy=ocr_policy,
        ocr_engine=(base.ocr_engine if base else ConvertOptions().ocr_engine),
        header_footer_mode=HeaderFooterMode.detect,
        recognition_mode=recognition_mode,
        detect_tables=detect_tables,
    )
    if not reasons:
        reasons.append("digital text: flowing reconstruction")
    return PolicyDecision(options=options, reasons=reasons, signals=signals)


def signals_from_pdf(data: bytes, route: PdfRouteResult | None = None) -> DocumentSignals:
    """Gather the document signals the policy needs, reusing cached work.

    Deliberately cheap: the router result and the page text are already in the
    per-conversion cache by the time this runs, and image placement comes from
    the same content-stream walk the figure extractor uses.
    """
    from app.services.convert import doc_cache
    from app.services.convert.layout.images import image_placements
    from app.services.convert.layout.page_classify import estimate_image_coverage
    from app.services.convert.layout.pdf_router import route_pdf

    if route is None:
        try:
            route = route_pdf(data)
        except Exception:  # a routing failure must not fail the conversion
            route = None

    signals = DocumentSignals()
    if route is not None:
        signals.doc_type = route.doc_type
        signals.confidence = route.confidence
        signals.pages_needing_ocr = list(route.pages_needing_ocr)

    try:
        reader = doc_cache.get_reader(data)
    except Exception:
        return signals

    pages = reader.pages
    signals.page_count = len(pages)
    coverages: list[float] = []
    words = 0

    for index, page in enumerate(pages):
        try:
            width = float(page.mediabox.width)
            height = float(page.mediabox.height)
        except Exception:
            width = height = 0.0
        try:
            blocks = [
                {"x": p.x, "y": p.y, "w": p.w, "h": p.h} for p in image_placements(page)
            ]
        except Exception:
            blocks = []
        coverage = estimate_image_coverage(blocks, width, height) if blocks else 0.0
        coverages.append(coverage)
        if coverage >= 0.98:
            signals.full_page_raster_pages.append(index)

        try:
            text = doc_cache.page_text(data, index)
        except Exception:
            text = ""
        page_words = len(text.split())
        words += page_words

        # Brochure: a lot of artwork and very little running prose.
        if coverage >= BROCHURE_IMAGE_COVERAGE and page_words < BROCHURE_MAX_WORDS:
            signals.brochure_pages.append(index)

    signals.image_coverage = sum(coverages) / max(len(coverages), 1)
    signals.text_words = words
    return signals


def policy_for(data: bytes, *, target: str, base: ConvertOptions | None = None) -> PolicyDecision:
    """Full auto decision for a PDF source."""
    return decide(signals_from_pdf(data), target=target, base=base)


# The decision travels with the conversion so the formatter layer can record it
# on the document without every plugin signature growing a parameter. A context
# variable, not a module global: concurrent jobs must not see each other's.
_active: ContextVar[PolicyDecision | None] = ContextVar("convert_auto_policy", default=None)


def set_active_decision(decision: PolicyDecision | None) -> None:
    _active.set(decision)


def get_active_decision() -> PolicyDecision | None:
    return _active.get()


@contextmanager
def decision_scope(decision: PolicyDecision | None = None) -> Iterator[None]:
    """Bind the routing decision to one conversion and nothing else.

    A ContextVar ``set`` persists in whatever context performed it. Conversions
    run on pooled worker threads whose context outlives the job, so a decision
    left behind by an automatic conversion was still readable by the next one —
    including an explicit conversion that did no routing at all, which would
    then report someone else's ``convert_policy`` as its own.

    Entering resets the decision to *nothing*, so a job can only ever see a
    decision its own run made; leaving restores what was there before.
    """
    token = _active.set(decision)
    try:
        yield
    finally:
        _active.reset(token)