pdf/gateway/app/services/convert/converters.py

"""Concrete OSS conversion plugins."""

from __future__ import annotations

import io
import zipfile

from PIL import Image

from app.schemas.convert import Fidelity
from app.services.convert import doc_cache
from app.services.convert.backends.pdf_exporter import export_office_to_pdf
from app.services.convert.formatters.docx_formatter import format_docx
from app.services.convert.formatters.text_formatters import (
    format_csv,
    format_html,
    format_json,
    format_md,
    format_txt,
)
from app.services.convert.formatters.xlsx_formatter import format_xlsx
from app.services.convert.formatters.xml_sanitize import sanitize_ooxml_text
from app.services.convert.idm.model import PageKind
from app.services.convert.layout.pipeline import (
    apply_docx_page_raster_fallback,
    build_docx_page_raster_document,
    build_idm_from_pdf,
    should_prefer_docx_page_rasters,
)
from app.services.convert.pdf_bridge import iter_pages_png
from app.services.convert.plugins import ConverterPlugin
from app.services.convert.quality.scorer import score_conversion
from app.services.convert.readers.docx_reader import read_docx_text, text_to_json_document
from app.services.convert.validation import assert_valid_ooxml, nonempty_output_or_warn
from app.services.convert.writers.pdf_from_docx import docx_to_pdf as reportlab_docx_to_pdf
from app.services.convert.writers.pdf_from_docx import html_to_pdf
from app.services.convert.writers.pdf_from_docx import xlsx_to_pdf as reportlab_xlsx_to_pdf
from app.services.convert.writers.pdf_from_text import csv_to_pdf, txt_to_pdf


def pdf_to_pdf_convert(data: bytes, filename: str = "") -> tuple[bytes, Fidelity, list[str], str, float | None]:
    from app.services.convert.options import OcrPolicy, get_options
    from app.services.convert.writers.searchable_pdf import scanned_to_searchable_pdf

    opts = get_options()
    if opts.ocr_policy == OcrPolicy.never:
        return data, Fidelity.high, [], "application/pdf", 1.0

    force_ocr = opts.ocr_policy == OcrPolicy.force
    out_pdf, warnings = scanned_to_searchable_pdf(data, filename, force_ocr=force_ocr)
    return out_pdf, Fidelity.high, warnings, "application/pdf", 1.0


def _pdf_plain_source(data: bytes, *, skip_pages: set[int] | None = None) -> str:
    """Whole-document plain text used as the scoring baseline.

    Served from the per-conversion cache: the router and the layout pipeline
    have usually extracted these pages already.

    ``skip_pages`` drops pages whose embedded text layer the engine judged
    unusable. Scoring recall against known-corrupt text measures the wrong
    thing in both directions: the engine is *penalised* for replacing
    ``"2 c11rtograp/1ic perspectives"`` with the words that are actually on the
    page, and *rewarded* for copying the corruption through.
    """
    try:
        if not skip_pages:
            return doc_cache.all_text(data)
        count = doc_cache.page_count(data)
        return "\n".join(
            doc_cache.page_text(data, i) for i in range(count) if i not in skip_pages
        )
    except Exception:
        return ""


def _docx_output_text(data: bytes) -> str:
    from docx import Document as DocxDocument

    doc = DocxDocument(io.BytesIO(data))
    parts: list[str] = []
    for p in doc.paragraphs:
        if p.text.strip():
            parts.append(p.text)
    for table in doc.tables:
        for row in table.rows:
            for cell in row.cells:
                if cell.text.strip():
                    parts.append(cell.text)
    return "\n".join(parts)


def _xlsx_output_payload(data: bytes) -> tuple[str, list[list[list[str]]]]:
    from openpyxl import load_workbook

    wb = load_workbook(io.BytesIO(data), data_only=True)
    parts: list[str] = []
    tables: list[list[list[str]]] = []
    for ws in wb.worksheets:
        grid: list[list[str]] = []
        for row in ws.iter_rows(values_only=True):
            cells = ["" if c is None else str(c) for c in row]
            if any(c.strip() for c in cells):
                grid.append(cells)
                parts.extend(c for c in cells if c.strip())
        if grid:
            tables.append(grid)
    return "\n".join(parts), tables


def _pdf_idm_convert(data: bytes, filename: str = ""):
    idm = build_idm_from_pdf(data, apply_ocr=True, filename=filename or None)
    broken = set((idm.meta or {}).get("encoding_broken_pages") or [])
    # Pages the engine rebuilt from OCR are excluded from the reference: their
    # embedded text is the corruption OCR was run to repair.
    replaced = {
        p.index
        for p in idm.pages
        if p.index in broken and p.kind == PageKind.scan
    }
    plain = _pdf_plain_source(data, skip_pages=replaced)
    # Prefer longer source signal for recall scoring
    source_text = plain if len(plain.strip()) >= len(idm.all_text().strip()) else idm.all_text()
    return idm, source_text


def _source_derived_tables(source_text: str) -> list[list[list[str]]]:
    """Build expected grids from source PDF plain text via tables-v3 (not IDM)."""
    from app.services.convert.layout.glyphs import lines_from_plain_text
    from app.services.convert.layout.tables import extract_tables

    if not (source_text or "").strip():
        return []
    lines = lines_from_plain_text(source_text)
    blocks, _rem, conf = extract_tables(lines, start_order=0)
    if conf < 0.65:
        return []
    return [b.cells for b in blocks if b.cells]


def pdf_to_docx(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    # Skip the expensive, misleading reflow route when bounded content-stream
    # evidence already says this is a dense positioned layout.  If rendering
    # cannot produce every page, fall through to the ordinary reconstruction.
    idm = build_docx_page_raster_document(data) if should_prefer_docx_page_rasters(data) else None
    raster_fallback = idm is not None
    if raster_fallback:
        source_text = ""
    else:
        idm, source_text = _pdf_idm_convert(data, _filename)
        raster_fallback = apply_docx_page_raster_fallback(idm, data)
    out = format_docx(idm, pdf_bytes=data)
    assert_valid_ooxml(out, kind="docx")
    output_text = _docx_output_text(out)
    extra: list[str] = []
    if raster_fallback:
        # Text recall is intentionally inapplicable: Word holds rendered pages
        # rather than a misleading, out-of-order text reconstruction.
        score = None
        extra.append(
            "Quality score omitted: this DOCX uses the faithful page-image fallback."
        )
    else:
        score, _sug, extra = score_conversion(
            source_text=source_text,
            output_text=output_text,
            idm=idm,
            expected_tables=_source_derived_tables(source_text) or None,
            actual_tables=None,  # DOCX tables scored via text; no grid extract here
            prefer_idm_table_confidence=False,
        )
    warnings = [
        "Layout fidelity is lossy; headers/footers/fonts are not fully preserved.",
        *idm.warnings,
        *extra,
    ]
    if not raster_fallback:
        nonempty_output_or_warn(
            source_text=source_text, output_text=output_text, warnings=warnings, label="PDF→DOCX"
        )
    return (
        out,
        Fidelity.lossy,
        warnings,
        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
        score,
    )


def docx_to_pdf_convert(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    out, warnings = export_office_to_pdf(
        data, source_ext="docx", reportlab_fn=reportlab_docx_to_pdf
    )
    return out, Fidelity.medium, warnings, "application/pdf", None


def pdf_to_xlsx(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    idm, source_text = _pdf_idm_convert(data, _filename)
    out, tw = format_xlsx(idm)
    assert_valid_ooxml(out, kind="xlsx")
    output_text, actual_tables = _xlsx_output_payload(out)
    expected = _source_derived_tables(source_text)
    score, _sug, extra = score_conversion(
        source_text=source_text,
        output_text=output_text,
        idm=idm,
        expected_tables=expected or None,
        actual_tables=actual_tables or None,
        prefer_idm_table_confidence=False,
    )
    warnings = ["Table detection is heuristic/lossy.", *tw, *idm.warnings, *extra]
    nonempty_output_or_warn(
        source_text=source_text, output_text=output_text, warnings=warnings, label="PDF→XLSX"
    )
    return (
        out,
        Fidelity.lossy,
        warnings,
        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        score,
    )


def xlsx_to_pdf_convert(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    def _rl(d: bytes) -> bytes:
        pdf, _w = reportlab_xlsx_to_pdf(d)
        return pdf

    out, warnings = export_office_to_pdf(data, source_ext="xlsx", reportlab_fn=_rl)
    return out, Fidelity.medium, warnings, "application/pdf", None


def _csv_output_payload(data: bytes) -> tuple[str, list[list[list[str]]]]:
    import csv
    text = data.decode("utf-8", errors="replace").lstrip("\ufeff")
    parts: list[str] = []
    tables: list[list[list[str]]] = []
    current_grid: list[list[str]] = []
    reader = csv.reader(io.StringIO(text))
    for row in reader:
        if not row:
            if current_grid:
                tables.append(current_grid)
                current_grid = []
            continue
        if len(row) == 1 and str(row[0]).startswith("# --- Table"):
            if current_grid:
                tables.append(current_grid)
                current_grid = []
            continue
        cells = [str(c).strip() for c in row if str(c).strip()]
        if cells:
            current_grid.append(row)
            parts.extend(cells)
    if current_grid:
        tables.append(current_grid)
    return " ".join(parts), tables


def pdf_to_csv(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    idm, source_text = _pdf_idm_convert(data, _filename)
    out = format_csv(idm)
    output_text, actual_tables = _csv_output_payload(out)
    expected = _source_derived_tables(source_text)
    score, _sug, extra = score_conversion(
        source_text=source_text,
        output_text=output_text,
        idm=idm,
        expected_tables=expected or None,
        actual_tables=actual_tables or None,
        prefer_idm_table_confidence=False,
    )
    warnings = ["Table detection is heuristic/lossy for CSV.", *idm.warnings, *extra]
    nonempty_output_or_warn(
        source_text=source_text, output_text=output_text, warnings=warnings, label="PDF→CSV"
    )
    return (
        out,
        Fidelity.lossy,
        warnings,
        "text/csv; charset=utf-8",
        score,
    )


def csv_to_pdf_convert(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    out, warnings = csv_to_pdf(data, filename=_filename)
    return out, Fidelity.high, warnings, "application/pdf", 1.0


def txt_to_pdf_convert(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    out, warnings = txt_to_pdf(data, filename=_filename)
    return out, Fidelity.high, warnings, "application/pdf", 1.0


def json_to_pdf_convert(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    from app.services.convert.writers.pdf_from_json import json_to_pdf

    out, warnings = json_to_pdf(data, filename=_filename)
    return out, Fidelity.high, warnings, "application/pdf", 1.0


def pdf_to_txt(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    idm, source_text = _pdf_idm_convert(data, _filename)
    out = format_txt(idm)
    text = out.decode("utf-8", errors="replace")
    score, fidelity, extra = score_conversion(
        source_text=source_text, output_text=text, idm=idm
    )
    warnings = [*idm.warnings, *extra]
    if not text.strip():
        warnings.append("TXT extract produced no text — PDF may be image-only or empty.")
    return out, fidelity, warnings, "text/plain; charset=utf-8", score


def pdf_to_md(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    from app.services.convert.sidecars.md_sidecar import pdf_to_md_sidecar

    side = pdf_to_md_sidecar(data)
    if side is not None:
        out_b, side_warn = side
        if out_b is not None:
            return out_b, Fidelity.medium, side_warn, "text/markdown; charset=utf-8", None
        # fail-open → IDM with warnings
        idm, source_text = _pdf_idm_convert(data, _filename)
        out = format_md(idm)
        score, fidelity, extra = score_conversion(
            source_text=source_text, output_text=out.decode("utf-8", errors="replace"), idm=idm
        )
        return out, fidelity, [*side_warn, *idm.warnings, *extra], "text/markdown; charset=utf-8", score

    idm, source_text = _pdf_idm_convert(data, _filename)
    out = format_md(idm)
    score, fidelity, extra = score_conversion(
        source_text=source_text, output_text=out.decode("utf-8", errors="replace"), idm=idm
    )
    return out, fidelity, [*idm.warnings, *extra], "text/markdown; charset=utf-8", score


def docx_markdown(data: bytes) -> str:
    """DOCX body as Markdown, in document order, with tables kept as tables."""
    import docx as _docx
    from docx.table import Table as _Table
    from docx.text.paragraph import Paragraph as _Paragraph

    from app.services.convert.formatters.text_formatters import md_table_lines

    document = _docx.Document(io.BytesIO(data))
    lines: list[str] = []
    for child in document.element.body.iterchildren():
        tag = child.tag.split("}")[-1]
        if tag == "p":
            para = _Paragraph(child, document)
            text = (para.text or "").strip()
            if not text:
                continue
            style = (para.style.name or "") if para.style is not None else ""
            if style.startswith("Heading"):
                digits = "".join(ch for ch in style if ch.isdigit())
                level = min(max(int(digits or 1), 1), 6)
                lines.append("#" * level + f" {text}")
            elif style.startswith("List"):
                marker = "1." if "Number" in style else "-"
                lines.append(f"{marker} {text}")
            else:
                lines.append(text)
            lines.append("")
        elif tag == "tbl":
            table = _Table(child, document)
            grid = [[cell.text.strip() for cell in row.cells] for row in table.rows]
            grid = [row for row in grid if any(row)]
            if grid:
                lines.extend(md_table_lines(grid))
    return "\n".join(lines).strip() + "\n"


def docx_to_md(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    from app.services.convert.sidecars.md_sidecar import office_to_md_sidecar

    side = office_to_md_sidecar(data, source_ext="docx")
    if side is not None:
        out_b, side_warn = side
        if out_b is not None:
            return out_b, Fidelity.medium, side_warn, "text/markdown; charset=utf-8", None
    # Fallback: read the document's own structure. mammoth's Markdown writer
    # flattens every table into a run of loose paragraphs — "Region", "Revenue",
    # "North", "1,240.50" each on their own line — which loses the grid that
    # xlsx→md and pdf→md both preserve. Walking the body in order keeps it.
    try:
        md = docx_markdown(data)
        warnings = ["docx→md from document structure (tables preserved)"]
        if side is not None:
            warnings = [*side[1], *warnings]
        if not md.strip():
            md = read_docx_text(data)
        return md.encode("utf-8"), Fidelity.medium, warnings, "text/markdown; charset=utf-8", None
    except Exception as exc:
        text = read_docx_text(data)
        warnings = [f"docx→md mammoth failed ({exc}); plain text fallback"]
        if side is not None:
            warnings = [*side[1], *warnings]
        return text.encode("utf-8"), Fidelity.medium, warnings, "text/markdown; charset=utf-8", None


def xlsx_to_md(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    from app.services.convert.sidecars.md_sidecar import office_to_md_sidecar

    side = office_to_md_sidecar(data, source_ext="xlsx")
    if side is not None:
        out_b, side_warn = side
        if out_b is not None:
            return out_b, Fidelity.medium, side_warn, "text/markdown; charset=utf-8", None
    text, tables = _xlsx_output_payload(data)
    parts: list[str] = []
    for grid in tables:
        if not grid:
            continue
        parts.append("| " + " | ".join(grid[0]) + " |")
        parts.append("| " + " | ".join(["---"] * len(grid[0])) + " |")
        for row in grid[1:]:
            parts.append("| " + " | ".join(row) + " |")
        parts.append("")
    if not parts and text.strip():
        parts.append(text)
    warnings = ["xlsx→md via openpyxl tables (fallback)"]
    if side is not None:
        warnings = [*side[1], *warnings]
    return "\n".join(parts).encode("utf-8"), Fidelity.medium, warnings, "text/markdown; charset=utf-8", None


def pptx_to_md(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    from app.services.convert.sidecars.md_sidecar import office_to_md_sidecar

    side = office_to_md_sidecar(data, source_ext="pptx")
    if side is not None:
        out_b, side_warn = side
        if out_b is not None:
            return out_b, Fidelity.medium, side_warn, "text/markdown; charset=utf-8", None
    # Minimal PPTX text extract from slide XML without LibreOffice
    warnings = ["pptx→md: sidecar missing; extracting slide text from OOXML"]
    if side is not None:
        warnings = [*side[1], *warnings]
    parts: list[str] = []
    try:
        with zipfile.ZipFile(io.BytesIO(data)) as zf:
            slides = sorted(
                n for n in zf.namelist() if n.startswith("ppt/slides/slide") and n.endswith(".xml")
            )
            for i, name in enumerate(slides, 1):
                xml = zf.read(name).decode("utf-8", errors="replace")
                # Pull <a:t> text nodes
                import re

                texts = re.findall(r"<a:t[^>]*>([^<]*)</a:t>", xml)
                body = " ".join(t.strip() for t in texts if t.strip())
                if body:
                    parts.append(f"## Slide {i}\n\n{body}\n")
    except Exception as exc:
        warnings.append(f"pptx OOXML extract failed: {exc}")
    if not parts:
        raise ValueError("PPTX→MD produced no text (enable CONVERT_MD_SIDECAR with anydoc).")
    return "\n".join(parts).encode("utf-8"), Fidelity.medium, warnings, "text/markdown; charset=utf-8", None


def _json_output_text(payload: bytes) -> str:
    """Text actually present in the idm.v1 export, for honest recall scoring."""
    import json as _json

    try:
        doc = _json.loads(payload.decode("utf-8"))
    except Exception:
        return ""
    parts: list[str] = []
    for page in doc.get("pages") or []:
        if page.get("text"):
            parts.append(str(page["text"]))
        for block in page.get("blocks") or []:
            if block.get("text"):
                parts.append(str(block["text"]))
            for row in block.get("cells") or []:
                parts.extend(str(c) for c in row if c)
    return "\n".join(parts)


def pdf_to_json(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    idm, source_text = _pdf_idm_convert(data, _filename)
    out = format_json(idm)
    # Score against the text the export actually carries, never against the
    # source itself — recall(x, x) is 1.0 and tells the caller nothing.
    score, fidelity, extra = score_conversion(
        source_text=source_text, output_text=_json_output_text(out), idm=idm
    )
    return out, fidelity, [*idm.warnings, *extra], "application/json", score


def pdf_to_html(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    idm, source_text = _pdf_idm_convert(data, _filename)
    out = format_html(idm)
    score, fidelity, extra = score_conversion(
        source_text=source_text, output_text=out.decode("utf-8", errors="replace"), idm=idm
    )
    warnings = ["PDF->HTML is structural/lossy.", *idm.warnings, *extra]
    return out, fidelity, warnings, "text/html; charset=utf-8", score


def _escape(s: str) -> str:
    return (
        s.replace("&", "&amp;")
        .replace("<", "&lt;")
        .replace(">", "&gt;")
        .replace('"', "&quot;")
    )


def pdf_to_pptx(data: bytes, filename: str = "") -> tuple[bytes, Fidelity, list[str], str, float | None]:
    from app.services.convert.formatters.pptx_formatter import format_pptx

    idm, source_text = _pdf_idm_convert(data, filename)
    out = format_pptx(idm)
    return (
        out,
        Fidelity.medium,
        [*idm.warnings],
        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
        None,
    )


def pptx_to_pdf_convert(data: bytes, filename: str = "") -> tuple[bytes, Fidelity, list[str], str, float | None]:
    from app.services.convert.writers.pdf_from_pptx import pptx_to_pdf

    out, warnings = pptx_to_pdf(data, filename)
    return out, Fidelity.high, warnings, "application/pdf", 1.0


def pdf_to_jpeg(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    # Keep only one decoded page and one compressed page in memory. The old
    # implementation materialised every PNG before encoding the first JPEG,
    # which made a long raster export scale with page count squared in RSS.
    first_jpeg: bytes | None = None
    zip_buf: io.BytesIO | None = None
    zip_file: zipfile.ZipFile | None = None
    count = 0
    for png in iter_pages_png(data, dpi=144):
        # Close both the decoder and converted image promptly; a long export
        # otherwise leaves one Pillow file object alive per yielded page.
        with Image.open(io.BytesIO(png)) as source:
            img = source.convert("RGB")
        try:
            buf = io.BytesIO()
            try:
                img.save(buf, format="JPEG", quality=90)
                jpeg = buf.getvalue()
            finally:
                buf.close()
        finally:
            img.close()
        count += 1
        if first_jpeg is None:
            first_jpeg = jpeg
        else:
            if zip_file is None:
                zip_buf = io.BytesIO()
                zip_file = zipfile.ZipFile(zip_buf, "w", compression=zipfile.ZIP_DEFLATED)
                zip_file.writestr("page-0001.jpg", first_jpeg)
            zip_file.writestr(f"page-{count:04d}.jpg", jpeg)
    if count == 0 or first_jpeg is None:
        raise ValueError("PDF render produced no page images.")
    if zip_file is None or zip_buf is None:
        return first_jpeg, Fidelity.high, [], "image/jpeg", 1.0
    zip_file.close()
    return (
        zip_buf.getvalue(),
        Fidelity.high,
        [f"Multi-page PDF exported as ZIP of {count} JPEG files."],
        "application/zip",
        1.0,
    )


def pdf_to_tiff(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    """PDF→TIFF, one multi-page TIFF, LZW-compressed.

    This saved with no ``compression`` argument, and Pillow's TIFF default is
    uncompressed, so every page cost width x height x 3 bytes raw. A 14-page
    letter-size document at 144 dpi came out at **77.7 MB** -- the identical pages
    as JPEG are 459 KB. That is not merely wasteful: the gateway rejects uploads
    over 50 MB, so the engine could produce a TIFF it then refused to read back,
    and PDF→TIFF→PDF was impossible for anything longer than about nine pages.

    Measured on that document: uncompressed 77.66 MB, LZW 8.18 MB, Deflate
    6.16 MB, all decoding to 14 correct frames, with LZW and Deflate costing
    ~0.5s against 0.1s. LZW rather than the slightly smaller Deflate because TIFF
    is chosen for archival and legacy-system interchange -- scanners, fax
    gateways, document-management systems -- and LZW is the variant every TIFF
    reader handles. A further 2 MB is not worth a file a customer's system cannot
    open.

    ``dpi`` is written for the same reason: without it the file reported 1 dpi
    when read back, which tells a consumer the page is 1224 inches wide and makes
    print and placement sizing meaningless.
    """
    out = io.BytesIO()
    from PIL import TiffImagePlugin

    pages = iter(iter_pages_png(data, dpi=144))
    try:
        png = next(pages)
    except StopIteration as exc:
        raise ValueError("PDF render produced no page images.") from exc
    writer = TiffImagePlugin.AppendingTiffWriter(out)
    count = 0
    try:
        while True:
            with Image.open(io.BytesIO(png)) as source:
                image = source.convert("RGB")
            try:
                image.save(
                    writer,
                    format="TIFF",
                    compression="tiff_lzw",
                    dpi=(144, 144),
                )
            finally:
                image.close()
            count += 1
            try:
                png = next(pages)
            except StopIteration:
                break
            writer.newFrame()
        writer.finalize()
    finally:
        # AppendingTiffWriter must not close the caller-owned BytesIO.
        try:
            writer.f.flush()
        except Exception:
            pass
    warnings: list[str] = []
    if count > 1:
        warnings.append(f"Multi-page PDF exported as a single {count}-page LZW TIFF.")
    return out.getvalue(), Fidelity.high, warnings, "image/tiff", 1.0


def pdf_to_png(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    first_png: bytes | None = None
    buf: io.BytesIO | None = None
    archive: zipfile.ZipFile | None = None
    count = 0
    for png in iter_pages_png(data, dpi=144):
        count += 1
        if first_png is None:
            first_png = png
        else:
            if archive is None:
                buf = io.BytesIO()
                archive = zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED)
                archive.writestr("page-0001.png", first_png)
            archive.writestr(f"page-{count:04d}.png", png)
    if count == 0 or first_png is None:
        raise ValueError("PDF render produced no page images.")
    if archive is None or buf is None:
        return first_png, Fidelity.high, [], "image/png", 1.0
    archive.close()
    return (
        buf.getvalue(),
        Fidelity.high,
        [f"Multi-page PDF exported as ZIP of {count} PNG files."],
        "application/zip",
        1.0,
    )


# Pillow modes that can be written straight into a PDF.
_PDF_SAFE_MODES = ("1", "L", "RGB", "CMYK")


def _prepare_image_for_pdf(img: Image.Image) -> Image.Image:
    """Normalise orientation and colour mode for PDF embedding."""
    from PIL import ImageOps

    try:
        img = ImageOps.exif_transpose(img) or img
    except Exception:
        pass
    if img.mode not in _PDF_SAFE_MODES:
        img = img.convert("RGB")
    return img


def image_to_pdf(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    """Image → PDF, preserving every frame of a multi-page TIFF."""
    from PIL import ImageSequence

    src = Image.open(io.BytesIO(data))
    frames = [_prepare_image_for_pdf(f.copy()) for f in ImageSequence.Iterator(src)]
    if not frames:
        raise ValueError("Image contained no decodable frames.")

    warnings: list[str] = []
    buf = io.BytesIO()
    if len(frames) == 1:
        frames[0].save(buf, format="PDF")
    else:
        frames[0].save(buf, format="PDF", save_all=True, append_images=frames[1:])
        warnings.append(f"Multi-page image: {len(frames)} frames written as {len(frames)} PDF pages.")
    return buf.getvalue(), Fidelity.high, warnings, "application/pdf", 1.0


def html_to_pdf_convert(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    return (
        html_to_pdf(data),
        Fidelity.medium,
        ["HTML→PDF is a text/layout subset via reportlab (not CSS-accurate)."],
        "application/pdf",
        None,
    )


def md_to_html(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    text = data.decode("utf-8", errors="replace")
    try:
        from markdown_it import MarkdownIt

        html = MarkdownIt().render(text)
    except Exception:
        paras = "".join(f"<p>{_escape(p)}</p>" for p in text.split("\n\n") if p.strip())
        html = paras
    doc = f'<!DOCTYPE html><html><head><meta charset="utf-8"></head><body>{html}</body></html>'
    return doc.encode("utf-8"), Fidelity.high, [], "text/html; charset=utf-8", None


def md_to_pdf(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    html_bytes, _, warnings, _, _ = md_to_html(data, _filename)
    pdf, _, w2, _, _ = html_to_pdf_convert(html_bytes, _filename)
    return pdf, Fidelity.medium, warnings + w2, "application/pdf", None


def html_table_rows(table) -> list[list[str]]:
    """Cell text of an HTML table, as a rectangular grid."""
    rows: list[list[str]] = []
    for tr in table.find_all("tr"):
        cells = tr.find_all(["th", "td"], recursive=False) or tr.find_all(["th", "td"])
        row = [c.get_text(" ", strip=True) for c in cells]
        if any(v for v in row):
            rows.append(row)
    return rows


def html_to_md(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    from bs4 import BeautifulSoup

    from app.services.convert.formatters.text_formatters import md_table_lines

    soup = BeautifulSoup(data.decode("utf-8", errors="replace"), "html.parser")
    for tag in soup(["script", "style"]):
        tag.decompose()

    lines: list[str] = []
    warnings: list[str] = []
    # Tables were not in the element list at all, so every grid in an HTML
    # document was dropped silently — the one conversion failure a user cannot
    # see without comparing the two documents side by side. Headings below h3
    # and block quotes went the same way.
    wanted = ["h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre", "blockquote", "table"]
    for el in soup.find_all(wanted):
        # A paragraph inside a table cell is emitted as part of its table.
        if el.name != "table" and el.find_parent("table") is not None:
            continue
        if el.name == "table":
            grid = html_table_rows(el)
            if grid:
                lines.extend(md_table_lines(grid))
            continue
        text = el.get_text(" ", strip=True)
        if not text:
            continue
        if el.name in ("h1", "h2", "h3", "h4", "h5", "h6"):
            lines.append("#" * int(el.name[1]) + f" {text}")
        elif el.name == "li":
            ordered = el.find_parent("ol") is not None
            lines.append(f"1. {text}" if ordered else f"- {text}")
        elif el.name == "blockquote":
            lines.append(f"> {text}")
        elif el.name == "pre":
            lines.append("```")
            lines.append(el.get_text("\n", strip=False).strip("\n"))
            lines.append("```")
        else:
            lines.append(text)
        lines.append("")
    if not lines:
        lines.append(soup.get_text("\n", strip=True))
    return (
        "\n".join(lines).encode("utf-8"),
        Fidelity.medium,
        warnings,
        "text/markdown; charset=utf-8",
        None,
    )


def docx_to_txt(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    return read_docx_text(data).encode("utf-8"), Fidelity.high, [], "text/plain; charset=utf-8", None


def docx_to_json(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    return text_to_json_document(read_docx_text(data), "docx"), Fidelity.medium, [], "application/json", None


def docx_to_html(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    """DOCX→HTML via mammoth (BSD-2), falling back to the in-house structure walker.

    mammoth was imported and called unguarded, which made this the only converter
    in the registry that turns a single dependency problem into a hard failure.
    Every other optional path here degrades: ``docx_to_md`` catches the same class
    of error and still returns a document. The exposure is not hypothetical --
    mammoth raises on some malformed but openable .docx files, and it is an
    optional wheel, so a partial install took ``docx→html`` down completely while
    ``docx→md`` and ``docx→pdf`` kept working on the very same file. A converter
    that cannot produce its best output should produce its second-best one.

    The fallback goes through ``docx_markdown`` rather than ``read_docx_text``
    because plain text would throw away exactly what HTML is for. The structure
    walker already reads headings, lists and tables in document order -- and
    handles tables *better* than mammoth, whose Markdown writer flattens a grid
    into loose paragraphs -- so rendering its Markdown gives real ``<h1>``,
    ``<ul>`` and ``<table>`` elements. ``markdown_it`` is a declared dependency
    and needs nothing beyond the standard library.
    """
    html_body = ""
    warnings: list[str] = []
    try:
        import mammoth

        result = mammoth.convert_to_html(io.BytesIO(data))
        html_body = result.value or ""
        warnings = [f"mammoth: {m.message}" for m in (result.messages or [])[:8]]
    except Exception as exc:
        # Deliberately broad: the point is that no failure mode below this
        # converter should be able to fail the request. ImportError for a missing
        # wheel and mammoth's own parse errors need identical handling.
        try:
            from markdown_it import MarkdownIt

            html_body = MarkdownIt().render(docx_markdown(data))
        except Exception:
            paras = read_docx_text(data).split("\n")
            html_body = "".join(f"<p>{_escape(p)}</p>" for p in paras if p.strip())
        warnings = [
            f"docx→html mammoth unavailable ({type(exc).__name__}: {exc}); "
            "rebuilt from document structure (headings, lists and tables kept)"
        ]
    doc = (
        '<!DOCTYPE html><html><head><meta charset="utf-8">'
        "<title>Converted DOCX</title></head><body>\n"
        f"{html_body}\n</body></html>"
    )
    return doc.encode("utf-8"), Fidelity.medium, warnings, "text/html; charset=utf-8", None


def html_to_docx(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    """HTML→DOCX via BeautifulSoup → python-docx (lossy / medium)."""
    from bs4 import BeautifulSoup, NavigableString
    from docx import Document as DocxDocument

    soup = BeautifulSoup(data.decode("utf-8", errors="replace"), "lxml")
    for tag in soup(["script", "style", "noscript"]):
        tag.decompose()
    document = DocxDocument()
    root = soup.body if soup.body else soup
    warnings: list[str] = ["HTML→DOCX is structural/lossy (not CSS-accurate)."]

    def _text(el) -> str:
        # Every string entering python-docx must be XML-legal: PDF and HTML
        # sources routinely carry C0 controls and lone surrogates that lxml
        # rejects at save time with an opaque ValueError.
        return sanitize_ooxml_text(el.get_text(" ", strip=True)) if el is not None else ""

    def _walk(node) -> None:
        if node is None or isinstance(node, NavigableString):
            return
        name = getattr(node, "name", None)
        if name in ("h1", "h2", "h3"):
            h = document.add_heading(_text(node), level=int(name[1]))
            return
        if name == "p":
            t = _text(node)
            if t:
                document.add_paragraph(t)
            return
        if name in ("ul", "ol"):
            style = "List Number" if name == "ol" else "List Bullet"
            for li in node.find_all("li", recursive=False):
                try:
                    p = document.add_paragraph(_text(li), style=style)
                except Exception:
                    document.add_paragraph(_text(li))
            return
        if name == "table":
            rows = node.find_all("tr")
            if not rows:
                return
            grid = []
            for tr in rows:
                cells = tr.find_all(["td", "th"])
                grid.append([c.get_text(" ", strip=True) for c in cells])
            if not grid:
                return
            cols = max(len(r) for r in grid)
            table = document.add_table(rows=len(grid), cols=cols)
            table.style = "Table Grid"
            for ri, row in enumerate(grid):
                for ci in range(cols):
                    table.rows[ri].cells[ci].text = (
                        sanitize_ooxml_text(row[ci]) if ci < len(row) else ""
                    )
            document.add_paragraph("")
            return
        if name in ("html", "body", "div", "section", "article", "main", "header", "footer", None):
            for child in getattr(node, "children", []) or []:
                _walk(child)

    _walk(root)
    if len(document.paragraphs) == 0 and len(document.tables) == 0:
        document.add_paragraph(
            sanitize_ooxml_text(soup.get_text("\n", strip=True)) or "(empty)"
        )

    buf = io.BytesIO()
    document.save(buf)
    return buf.getvalue(), Fidelity.medium, warnings, (
        "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
    ), None


def docx_passthrough(data: bytes, _filename: str) -> tuple[bytes, Fidelity, list[str], str, float | None]:
    if not data or data[:2] != b"PK":
        raise ValueError("DOCX passthrough expects OOXML zip bytes")
    return (
        data,
        Fidelity.high,
        [],
        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
        1.0,
    )


def all_plugins() -> list[ConverterPlugin]:
    gaps_office = [
        "lossy layout; headers/footers not fully mapped",
        "in-house ConvertAPI/Aspose-class accuracy climbing (Phase 1 layout/IDM)",
    ]
    return [
        ConverterPlugin(
            "pdf",
            "pdf",
            Fidelity.high,
            pdf_to_pdf_convert,
            "PDF->PDF OCR Searchable PDF rebuild",
            [],
        ),
        ConverterPlugin("pdf", "docx", Fidelity.lossy, pdf_to_docx, "PDF->DOCX via layout IDM", gaps_office),
        ConverterPlugin(
            "docx",
            "pdf",
            Fidelity.medium,
            docx_to_pdf_convert,
            "DOCX->PDF via reportlab (concurrent-safe)",
            ["In-process reportlab; complex Word layout may differ from Word print"],
        ),
        ConverterPlugin("pdf", "xlsx", Fidelity.lossy, pdf_to_xlsx, "PDF->XLSX via layout IDM", gaps_office),
        ConverterPlugin(
            "xlsx",
            "pdf",
            Fidelity.medium,
            xlsx_to_pdf_convert,
            "XLSX->PDF via reportlab (concurrent-safe)",
            ["formulas use cached values; wide sheets scaled"],
        ),
        ConverterPlugin("pdf", "csv", Fidelity.lossy, pdf_to_csv, "PDF->CSV via layout IDM table extraction", gaps_office),
        ConverterPlugin(
            "csv",
            "pdf",
            Fidelity.high,
            csv_to_pdf_convert,
            "CSV->PDF via reportlab styled grid",
            [],
        ),
        ConverterPlugin("pdf", "pptx", Fidelity.medium, pdf_to_pptx, "PDF->PPTX via IDM presentation builder", gaps_office),
        ConverterPlugin(
            "pptx",
            "pdf",
            Fidelity.high,
            pptx_to_pdf_convert,
            "PPTX->PDF via structural slide renderer",
            [],
        ),
        ConverterPlugin("pdf", "txt", Fidelity.medium, pdf_to_txt, "PDF->TXT via IDM"),
        ConverterPlugin(
            "txt",
            "pdf",
            Fidelity.high,
            txt_to_pdf_convert,
            "TXT->PDF via reportlab flowables",
            [],
        ),
        ConverterPlugin(
            "json",
            "pdf",
            Fidelity.high,
            json_to_pdf_convert,
            "JSON->PDF via styled grid or structural inspector",
            [],
        ),
        ConverterPlugin("pdf", "md", Fidelity.medium, pdf_to_md, "PDF->Markdown via IDM (+ optional MIT sidecar)"),
        ConverterPlugin(
            "docx",
            "md",
            Fidelity.medium,
            docx_to_md,
            "DOCX->Markdown (anydoc sidecar or mammoth)",
            ["medium; sidecar optional"],
        ),
        ConverterPlugin(
            "xlsx",
            "md",
            Fidelity.medium,
            xlsx_to_md,
            "XLSX->Markdown (anydoc sidecar or openpyxl)",
            ["medium; sidecar optional"],
        ),
        ConverterPlugin(
            "pptx",
            "md",
            Fidelity.medium,
            pptx_to_md,
            "PPTX->Markdown (anydoc sidecar or OOXML text)",
            ["medium; sidecar optional"],
        ),
        ConverterPlugin(
            "pdf",
            "json",
            Fidelity.medium,
            pdf_to_json,
            "PDF->JSON idm.v1 product schema",
            ["idm.v1 is the product Intermediate Document Model export"],
        ),
        ConverterPlugin("pdf", "html", Fidelity.medium, pdf_to_html, "PDF->HTML via IDM", ["structural/lossy"]),
        ConverterPlugin("pdf", "png", Fidelity.high, pdf_to_png, "PDF->PNG (ZIP if multi-page)"),
        ConverterPlugin("pdf", "jpeg", Fidelity.high, pdf_to_jpeg, "PDF->JPEG (ZIP if multi-page)"),
        ConverterPlugin("pdf", "tiff", Fidelity.high, pdf_to_tiff, "PDF->TIFF"),
        ConverterPlugin("png", "pdf", Fidelity.high, image_to_pdf, "PNG->PDF"),
        ConverterPlugin("jpeg", "pdf", Fidelity.high, image_to_pdf, "JPEG->PDF"),
        ConverterPlugin("tiff", "pdf", Fidelity.high, image_to_pdf, "TIFF->PDF"),
        ConverterPlugin(
            "html",
            "pdf",
            Fidelity.medium,
            html_to_pdf_convert,
            "HTML->PDF via reportlab structural subset",
            ["not CSS-accurate; WeasyPrint deferred; remote images denied"],
        ),
        ConverterPlugin("md", "html", Fidelity.high, md_to_html, "Markdown->HTML"),
        ConverterPlugin("md", "pdf", Fidelity.medium, md_to_pdf, "Markdown->PDF"),
        ConverterPlugin("html", "md", Fidelity.medium, html_to_md, "HTML->Markdown"),
        ConverterPlugin("docx", "txt", Fidelity.high, docx_to_txt, "DOCX->TXT"),
        ConverterPlugin("docx", "json", Fidelity.medium, docx_to_json, "DOCX->JSON"),
        ConverterPlugin(
            "docx",
            "html",
            Fidelity.medium,
            docx_to_html,
            "DOCX->HTML via mammoth",
            ["styles/layout approximated"],
        ),
        ConverterPlugin(
            "html",
            "docx",
            Fidelity.medium,
            html_to_docx,
            "HTML->DOCX via BeautifulSoup",
            ["structural/lossy; not CSS-accurate"],
        ),
        ConverterPlugin("docx", "docx", Fidelity.high, docx_passthrough, "DOCX passthrough", []),
    ]