1037 lines
42 KiB
Python
1037 lines
42 KiB
Python
"""DOCX formatter from IDM."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import io
|
||
import re
|
||
|
||
from docx import Document as DocxDocument
|
||
from docx.enum.text import WD_ALIGN_PARAGRAPH
|
||
from docx.oxml import OxmlElement
|
||
from docx.oxml.ns import qn
|
||
from docx.shared import Inches, Pt, RGBColor, Twips
|
||
|
||
from app.services.convert.formatters import docx_exact
|
||
from app.services.convert.formatters.xml_sanitize import sanitize_ooxml_text
|
||
from app.services.convert.idm.model import (
|
||
PAGE_RASTER_SOURCE,
|
||
Block,
|
||
BlockType,
|
||
Document,
|
||
PageKind,
|
||
TextSpan,
|
||
)
|
||
from app.services.convert.layout.headers_footers import looks_like_page_number
|
||
from app.services.convert.layout.text_quality import is_ocr_noise_line
|
||
from app.services.convert.text.arabic_logical import (
|
||
contains_arabic,
|
||
contains_complex_script,
|
||
is_rtl_dominant,
|
||
to_logical,
|
||
)
|
||
|
||
_ALIGN = {
|
||
"left": WD_ALIGN_PARAGRAPH.LEFT,
|
||
"center": WD_ALIGN_PARAGRAPH.CENTER,
|
||
"right": WD_ALIGN_PARAGRAPH.RIGHT,
|
||
"justify": WD_ALIGN_PARAGRAPH.JUSTIFY,
|
||
}
|
||
|
||
_HYPERLINK_REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink"
|
||
|
||
# Plausible typographic range. Values outside it are bounding-box heights
|
||
# that leaked into font_size, not point sizes.
|
||
MIN_FONT_PT = 4.0
|
||
MAX_FONT_PT = 48.0
|
||
|
||
|
||
def _normalize_url(url: str) -> str:
|
||
u = (url or "").strip()
|
||
if not u:
|
||
return ""
|
||
if "://" not in u:
|
||
return f"http://{u}"
|
||
return u
|
||
|
||
|
||
def _add_hyperlink_run(paragraph, text: str, url: str, *, bold: bool = False, italic: bool = False) -> None:
|
||
"""Insert a real w:hyperlink containing a styled run (Word-clickable)."""
|
||
part = paragraph.part
|
||
r_id = part.relate_to(_normalize_url(url), _HYPERLINK_REL, is_external=True)
|
||
hyperlink = OxmlElement("w:hyperlink")
|
||
hyperlink.set(qn("r:id"), r_id)
|
||
|
||
run = OxmlElement("w:r")
|
||
r_pr = OxmlElement("w:rPr")
|
||
color = OxmlElement("w:color")
|
||
color.set(qn("w:val"), "0563C1")
|
||
r_pr.append(color)
|
||
u = OxmlElement("w:u")
|
||
u.set(qn("w:val"), "single")
|
||
r_pr.append(u)
|
||
if bold:
|
||
r_pr.append(OxmlElement("w:b"))
|
||
if italic:
|
||
r_pr.append(OxmlElement("w:i"))
|
||
run.append(r_pr)
|
||
t = OxmlElement("w:t")
|
||
t.set("{http://www.w3.org/XML/1998/namespace}space", "preserve")
|
||
t.text = sanitize_ooxml_text(text)
|
||
run.append(t)
|
||
hyperlink.append(run)
|
||
paragraph._p.append(hyperlink)
|
||
|
||
|
||
# Near-white text is not written as a run colour. See ``_apply_span_color``.
|
||
_NEAR_WHITE = 235
|
||
|
||
|
||
def _apply_span_color(run, color: str | None) -> None:
|
||
"""Set a run's colour from an ``RRGGBB`` string, ignoring anything malformed.
|
||
|
||
A span with no colour is left alone rather than set to black, so the run
|
||
keeps Word's "automatic" colour -- which is what an author who never chose a
|
||
colour expects, and what lets the text stay readable in a dark theme.
|
||
|
||
Near-white is deliberately dropped, and this is the one place where being
|
||
less faithful is the right answer. White text in a PDF means one of two
|
||
things. It is either deliberately invisible -- the GPO's ``govinfo`` pages
|
||
carry printer job tickets like ``Jkt 247004`` and ``VerDate Sep<11>2014`` in
|
||
white, and 37 such runs appear in this corpus -- or it is a heading reversed
|
||
out of a dark banner. This writer does not reproduce background shading, so
|
||
the second case would put white text on a white Word page and the content
|
||
would silently vanish. There is no way to tell the two apart from the text
|
||
colour alone, and the costs are not symmetric: leaving job-ticket noise
|
||
visible is untidy, while losing a heading is losing content, in the way that
|
||
is hardest for anyone to notice. So near-white falls back to automatic and
|
||
stays readable either way.
|
||
"""
|
||
if not color:
|
||
return
|
||
text = str(color).lstrip("#").strip()
|
||
if len(text) != 6:
|
||
return
|
||
try:
|
||
r, g, b = (int(text[i : i + 2], 16) for i in (0, 2, 4))
|
||
except (ValueError, TypeError):
|
||
return
|
||
if min(r, g, b) >= _NEAR_WHITE:
|
||
return
|
||
run.font.color.rgb = RGBColor(r, g, b)
|
||
|
||
|
||
def _apply_spans(
|
||
paragraph,
|
||
spans: list[TextSpan],
|
||
*,
|
||
fallback: str = "",
|
||
warnings: list[str] | None = None,
|
||
visual: bool = False,
|
||
) -> None:
|
||
if not spans:
|
||
cleaned = sanitize_ooxml_text(to_logical(fallback, visual=visual))
|
||
if cleaned.strip():
|
||
paragraph.add_run(cleaned)
|
||
return
|
||
for span in spans:
|
||
if span.text is None:
|
||
continue
|
||
span_visual = visual or bool(getattr(span, "visual_order", False))
|
||
text = sanitize_ooxml_text(to_logical(span.text, visual=span_visual))
|
||
if span.url:
|
||
try:
|
||
_add_hyperlink_run(
|
||
paragraph,
|
||
text,
|
||
span.url,
|
||
bold=bool(span.bold),
|
||
italic=bool(span.italic),
|
||
)
|
||
continue
|
||
except Exception as exc:
|
||
if warnings is not None:
|
||
warnings.append(f"hyperlink omitted: {exc}")
|
||
# Fall through to plain run so text is not lost
|
||
run = paragraph.add_run(text)
|
||
if span.bold:
|
||
run.bold = True
|
||
if span.italic:
|
||
run.italic = True
|
||
_apply_span_color(run, getattr(span, "color", None))
|
||
size = _sane_font_size(span.font_size)
|
||
# A 1–3 character OCR token (digit, "@", stray mark) often inherits a
|
||
# title-sized box. Writing that size produces the giant mid-sentence
|
||
# glyphs; inherit Normal instead.
|
||
if size is not None and len(text.strip()) <= 3 and size > 16:
|
||
size = None
|
||
if size is not None:
|
||
run.font.size = Pt(size)
|
||
if span.font_name:
|
||
face = span.font_name.split("-")[0].split(",")[0].strip()
|
||
if face:
|
||
run.font.name = face
|
||
# Complex-script properties. Word resolves Arabic, Hebrew and other
|
||
# complex scripts through w:rtl / w:cs / w:szCs, not through w:ascii —
|
||
# without these the glyphs are laid out left-to-right and rendered with
|
||
# the Latin font, which is why correct Unicode still looks wrong.
|
||
if contains_complex_script(text):
|
||
_set_run_complex_script(run, size=size)
|
||
if span.url:
|
||
try:
|
||
run.font.color.rgb = RGBColor(0x05, 0x63, 0xC1)
|
||
run.underline = True
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _sane_font_size(value: float | None) -> float | None:
|
||
"""Clamp a span's font size, rejecting values that are clearly not point sizes.
|
||
|
||
Glyph and OCR bounding-box *heights* leak into ``font_size`` on some paths.
|
||
A height in device pixels is an order of magnitude larger than a point size,
|
||
and clamping it to the 72 pt ceiling produced runs of 72 pt body text. Values
|
||
outside a plausible typographic range are dropped so the style inherits.
|
||
"""
|
||
if not value or value <= 0:
|
||
return None
|
||
if value < MIN_FONT_PT or value > MAX_FONT_PT:
|
||
return None
|
||
return round(float(value), 1)
|
||
|
||
|
||
def _set_run_complex_script(run, *, size: float | None) -> None:
|
||
"""Mark a run as complex-script so Word applies bidi layout and the CS font."""
|
||
try:
|
||
rPr = run._r.get_or_add_rPr()
|
||
rtl = OxmlElement("w:rtl")
|
||
rtl.set(qn("w:val"), "1")
|
||
rPr.append(rtl)
|
||
|
||
face = run.font.name or "Arial"
|
||
rfonts = rPr.find(qn("w:rFonts"))
|
||
if rfonts is None:
|
||
rfonts = OxmlElement("w:rFonts")
|
||
rPr.append(rfonts)
|
||
rfonts.set(qn("w:cs"), face)
|
||
|
||
if size is not None:
|
||
sz_cs = OxmlElement("w:szCs")
|
||
# OOXML expresses size in half-points.
|
||
sz_cs.set(qn("w:val"), str(int(round(size * 2))))
|
||
rPr.append(sz_cs)
|
||
if run.bold:
|
||
b_cs = OxmlElement("w:bCs")
|
||
rPr.append(b_cs)
|
||
if run.italic:
|
||
i_cs = OxmlElement("w:iCs")
|
||
rPr.append(i_cs)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _block_visual(block) -> bool:
|
||
"""True when a block's text is still in visual order and needs repairing.
|
||
|
||
Ordering is now resolved at extraction time — ``layout.glyphs`` and the OCR
|
||
adapter sort each line by its own reading direction — so a block's source
|
||
no longer implies visual order. Only an explicit span flag does, which is
|
||
set when a producer genuinely could not determine direction.
|
||
"""
|
||
return any(getattr(s, "visual_order", False) for s in (getattr(block, "spans", None) or []))
|
||
|
||
|
||
# A recovered letterhead mark is a mark, not an illustration. These are the
|
||
# outer bounds any masthead artwork is fitted inside, whatever its raster size.
|
||
MAX_MASTHEAD_WIDTH_IN = 2.0
|
||
MAX_MASTHEAD_HEIGHT_IN = 1.4
|
||
# Fallback rendering density for artwork that reached the writer with no
|
||
# geometry. 150 dpi matches the page raster the crops are cut from.
|
||
FALLBACK_IMAGE_DPI = 150.0
|
||
# A small physical margin keeps an inline image from spilling to an otherwise
|
||
# blank second page in Word, while remaining visually indistinguishable from a
|
||
# source PDF page at normal zoom.
|
||
PAGE_RASTER_MARGIN_PT = 10.8 # 0.15 inch
|
||
PAGE_RASTER_PARAGRAPH_SAFETY_PT = 6.0
|
||
|
||
|
||
def _image_pixel_size(png_bytes: bytes) -> tuple[int, int] | None:
|
||
try:
|
||
from PIL import Image
|
||
|
||
with Image.open(io.BytesIO(png_bytes)) as im:
|
||
return im.size
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _figure_width_inches(block: Block, page_width: float) -> float:
|
||
"""How wide to draw a figure, in inches.
|
||
|
||
A figure with measured geometry is drawn at its measured width. A figure
|
||
without it used to be drawn at a flat 5.5in — which is most of a page, and
|
||
is why a small recovered emblem arrived as a full-width banner. With no
|
||
geometry the image's own pixel size now decides, so a small picture stays
|
||
small; 5.5in survives only as the last resort for an image whose size
|
||
cannot be read at all.
|
||
"""
|
||
try:
|
||
w_pt = float(block.bbox.w or 0)
|
||
if w_pt > 20:
|
||
return max(0.2, min(w_pt / 72.0, max(page_width / 72.0 - 1.0, 1.5)))
|
||
except Exception:
|
||
pass
|
||
|
||
size = _image_pixel_size(block.image_png) if block.image_png else None
|
||
if size:
|
||
inches = size[0] / FALLBACK_IMAGE_DPI
|
||
return max(0.2, min(inches, max(page_width / 72.0 - 1.0, 1.5)))
|
||
return 5.5
|
||
|
||
|
||
def _masthead_size_inches(block: Block, page_width: float) -> tuple[float, float]:
|
||
"""Width and height for a letterhead mark, fitted inside the masthead box.
|
||
|
||
Aspect ratio is preserved: a wordmark is wide and short, an emblem is
|
||
roughly square, and stretching either to a fixed width is what makes a
|
||
recovered letterhead look wrong.
|
||
"""
|
||
w_in = _figure_width_inches(block, page_width)
|
||
try:
|
||
h_pt = float(block.bbox.h or 0)
|
||
w_pt = float(block.bbox.w or 0)
|
||
ratio = (h_pt / w_pt) if (w_pt > 0 and h_pt > 0) else 0.0
|
||
except Exception:
|
||
ratio = 0.0
|
||
if ratio <= 0:
|
||
size = _image_pixel_size(block.image_png) if block.image_png else None
|
||
ratio = (size[1] / size[0]) if size and size[0] else 0.5
|
||
h_in = w_in * ratio
|
||
|
||
if w_in > MAX_MASTHEAD_WIDTH_IN:
|
||
h_in *= MAX_MASTHEAD_WIDTH_IN / w_in
|
||
w_in = MAX_MASTHEAD_WIDTH_IN
|
||
if h_in > MAX_MASTHEAD_HEIGHT_IN:
|
||
w_in *= MAX_MASTHEAD_HEIGHT_IN / h_in
|
||
h_in = MAX_MASTHEAD_HEIGHT_IN
|
||
return max(0.15, w_in), max(0.15, h_in)
|
||
|
||
|
||
def _set_paragraph_rtl(paragraph) -> None:
|
||
"""Mark paragraph as right-to-left for Arabic runs."""
|
||
try:
|
||
pPr = paragraph._p.get_or_add_pPr()
|
||
bidi = OxmlElement("w:bidi")
|
||
bidi.set(qn("w:val"), "1")
|
||
pPr.append(bidi)
|
||
paragraph.alignment = WD_ALIGN_PARAGRAPH.RIGHT
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
# A running banner is a few lines — an agency name in each language, a country.
|
||
# Beyond this the "header" is page content that was mis-detected, and writing it
|
||
# all into every page's margin would be worse than leaving it in the body.
|
||
MAX_HEADER_LINES = 4
|
||
|
||
|
||
# Set by the OCR rebuild on artwork segmented out of a page raster's masthead.
|
||
MASTHEAD_SOURCE = "masthead"
|
||
|
||
|
||
def _is_masthead_mark(block) -> bool:
|
||
return (
|
||
block.type == BlockType.figure
|
||
and bool(block.image_png)
|
||
and (block.source or "") == MASTHEAD_SOURCE
|
||
)
|
||
|
||
|
||
def _add_page_number_field(part) -> None:
|
||
"""A real Word PAGE field in the footer.
|
||
|
||
A scanned document stamps a different number on every page, so there is no
|
||
single literal text a footer could carry. Writing the numbers as text would
|
||
put page one's "1" on all forty pages; leaving them in the body scatters a
|
||
stray digit through the prose. The field is what Word itself uses, and it
|
||
renumbers correctly when the document is edited.
|
||
"""
|
||
paragraphs = list(part.paragraphs)
|
||
para = paragraphs[0] if paragraphs else part.add_paragraph()
|
||
para.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||
run = para.add_run()
|
||
|
||
begin = OxmlElement("w:fldChar")
|
||
begin.set(qn("w:fldCharType"), "begin")
|
||
instr = OxmlElement("w:instrText")
|
||
instr.set(qn("xml:space"), "preserve")
|
||
instr.text = " PAGE "
|
||
end = OxmlElement("w:fldChar")
|
||
end.set(qn("w:fldCharType"), "end")
|
||
run._r.append(begin)
|
||
run._r.append(instr)
|
||
run._r.append(end)
|
||
|
||
|
||
def _set_tbl_borders_nil(table) -> None:
|
||
"""A letterhead row is not a data grid — hide every rule."""
|
||
tbl = table._tbl
|
||
tbl_pr = tbl.tblPr
|
||
if tbl_pr is None:
|
||
tbl_pr = OxmlElement("w:tblPr")
|
||
tbl.insert(0, tbl_pr)
|
||
borders = tbl_pr.find(qn("w:tblBorders"))
|
||
if borders is None:
|
||
borders = OxmlElement("w:tblBorders")
|
||
tbl_pr.append(borders)
|
||
for edge in ("top", "left", "bottom", "right", "insideH", "insideV"):
|
||
el = borders.find(qn(f"w:{edge}"))
|
||
if el is None:
|
||
el = OxmlElement(f"w:{edge}")
|
||
borders.append(el)
|
||
el.set(qn("w:val"), "nil")
|
||
el.set(qn("w:sz"), "0")
|
||
el.set(qn("w:space"), "0")
|
||
el.set(qn("w:color"), "auto")
|
||
|
||
|
||
def _add_header_picture(paragraph, png: bytes, w_in: float, h_in: float, align, warnings: list[str]) -> None:
|
||
paragraph.alignment = align
|
||
try:
|
||
paragraph.add_run().add_picture(io.BytesIO(png), width=Inches(w_in), height=Inches(h_in))
|
||
except Exception as exc:
|
||
warnings.append(f"masthead mark omitted from header: {exc}")
|
||
|
||
|
||
def _fill_header_marks(part, marks: list[tuple[bytes, float, float]], warnings: list[str]) -> None:
|
||
"""Put letterhead artwork in the Word header.
|
||
|
||
One mark is centred. Two or more sit in a borderless two-cell row so an
|
||
emblem on the left and a seal on the right stay on their sides instead of
|
||
stacking in the middle of the margin.
|
||
"""
|
||
if not marks:
|
||
return
|
||
if len(marks) == 1:
|
||
paragraphs = list(part.paragraphs)
|
||
para = paragraphs[0] if paragraphs else part.add_paragraph()
|
||
png, w_in, h_in = marks[0]
|
||
_add_header_picture(para, png, w_in, h_in, WD_ALIGN_PARAGRAPH.CENTER, warnings)
|
||
return
|
||
table = part.add_table(1, 2, width=Inches(6.5))
|
||
_set_tbl_borders_nil(table)
|
||
try:
|
||
table.autofit = True
|
||
except Exception:
|
||
pass
|
||
left, right = marks[0], marks[-1]
|
||
_add_header_picture(
|
||
table.cell(0, 0).paragraphs[0], left[0], left[1], left[2], WD_ALIGN_PARAGRAPH.LEFT, warnings
|
||
)
|
||
_add_header_picture(
|
||
table.cell(0, 1).paragraphs[0], right[0], right[1], right[2], WD_ALIGN_PARAGRAPH.RIGHT, warnings
|
||
)
|
||
|
||
|
||
def _fill_header_footer(part, texts: list[str]) -> None:
|
||
"""Write every banner line into the Word header/footer, not just the first.
|
||
|
||
The banner on a bilingual document is several lines — the agency in Arabic,
|
||
the agency in English, the country. Writing only ``texts[0]`` dropped the
|
||
rest from the document altogether: they had been retagged out of the body,
|
||
so nothing else carried them.
|
||
|
||
Arabic lines get paragraph-level bidi and complex-script run properties, or
|
||
Word lays them out left to right in the margin.
|
||
"""
|
||
if not texts:
|
||
return
|
||
from app.services.convert.layout.text_quality import (
|
||
is_cmap_garbled_line,
|
||
is_ocr_noise_line,
|
||
ocr_text_quality_score,
|
||
)
|
||
|
||
scored = [
|
||
t
|
||
for t in texts
|
||
if ocr_text_quality_score([t]) >= 0.34
|
||
and not is_ocr_noise_line(t)
|
||
and not is_cmap_garbled_line(t)
|
||
]
|
||
# Short Latin stamps next to recovered Arabic are usually OCR of a
|
||
# country line. Keep real English banners (5+ words) and any Arabic.
|
||
if any("\u0600" <= ch <= "\u06FF" for t in scored for ch in t):
|
||
kept = [
|
||
t
|
||
for t in scored
|
||
if any("\u0600" <= ch <= "\u06FF" for ch in t)
|
||
or len(t.split()) >= 5
|
||
or not is_cmap_garbled_line(t, min_tokens=1)
|
||
]
|
||
if kept:
|
||
scored = kept
|
||
lines = (scored or list(texts))[:MAX_HEADER_LINES]
|
||
# Only reuse paragraphs that are actually empty. Assigning ``para.text``
|
||
# replaces every run in the paragraph, so reusing one that already holds a
|
||
# letterhead mark silently deleted the picture — the image stayed in the
|
||
# package while nothing referenced it.
|
||
paragraphs = [p for p in part.paragraphs if not p.runs and not p.text.strip()]
|
||
for i, text in enumerate(lines):
|
||
para = paragraphs[i] if i < len(paragraphs) else part.add_paragraph()
|
||
para.text = sanitize_ooxml_text(text)
|
||
if is_rtl_dominant(text):
|
||
_set_paragraph_rtl(para)
|
||
for run in para.runs:
|
||
_set_run_complex_script(run, size=None)
|
||
else:
|
||
para.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||
|
||
|
||
def _compress_figure_bytes(png_bytes: bytes, *, max_side: int = 1600, quality: int = 72) -> bytes:
|
||
"""Downscale/compress page images so DOCX stays usable."""
|
||
try:
|
||
from PIL import Image
|
||
|
||
im = Image.open(io.BytesIO(png_bytes))
|
||
if max(im.size) > max_side:
|
||
scale = max_side / max(im.size)
|
||
im = im.resize((max(1, int(im.width * scale)), max(1, int(im.height * scale))))
|
||
if im.mode not in ("RGB", "L"):
|
||
im = im.convert("RGB")
|
||
buf = io.BytesIO()
|
||
im.save(buf, format="JPEG", quality=quality, optimize=True)
|
||
out = buf.getvalue()
|
||
# Prefer compressed when meaningfully smaller
|
||
return out if len(out) < len(png_bytes) * 0.9 else png_bytes
|
||
except Exception:
|
||
return png_bytes
|
||
|
||
|
||
def _exact_layout_requested() -> bool:
|
||
"""Whether this conversion asked for a positioned emit.
|
||
|
||
Read from the conversion's options rather than passed in, so the flowing
|
||
path — every existing caller — is untouched.
|
||
"""
|
||
try:
|
||
from app.services.convert.options import LayoutMode, RecognitionMode, get_options
|
||
|
||
opts = get_options()
|
||
return (
|
||
opts.layout_mode == LayoutMode.exact
|
||
or opts.recognition_mode == RecognitionMode.textbox
|
||
)
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _uses_full_page_raster_fallback(doc: Document) -> bool:
|
||
return bool((doc.meta or {}).get("docx_page_raster_fallback"))
|
||
|
||
|
||
# Nothing narrower than a quarter inch: below that Word's own printable area
|
||
# takes over and the number is decoration.
|
||
MIN_MARGIN_PT = 18.0
|
||
# And nothing wider than this share of the page — a document whose only text
|
||
# is a centred title should not come out with four-inch margins.
|
||
MAX_MARGIN_RATIO = 0.25
|
||
|
||
|
||
_PLAIN_NUMBER_MARKER = re.compile(r"^\s*(\d{1,3})[.)]\s+")
|
||
# Marker forms a list *style* supplies for us, and so may be stripped.
|
||
_LIST_MARKER_STRIP = r"^(\u2022|\-|\*|\u25cf|\u25aa|\d+[\.\)]|[A-Za-z][\.\)])\s+"
|
||
_ANY_LIST_MARKER = re.compile(r"^\s*(?:[•●▪\-\*•]|\(?[0-9A-Za-z]{1,4}[.)])\s+")
|
||
|
||
|
||
def _list_marker(block: Block) -> str:
|
||
match = _ANY_LIST_MARKER.match(block.plain_text() or "")
|
||
return match.group(0).strip() if match else ""
|
||
|
||
|
||
def _auto_numberable(blocks: list[Block]) -> set[int]:
|
||
"""Ids of numbered list items Word may renumber without changing them.
|
||
|
||
Word's "List Number" style throws the document's own marker away and
|
||
counts from one. That is right for a plain 1., 2., 3. list — the reader
|
||
gets a real editable list and the numbers are identical — and wrong for
|
||
everything else: a clause numbered 3.2, a list that starts at 5 because it
|
||
continued from the previous page, an (a)/(b)/(c) run. Those keep their own
|
||
markers as text with a hanging indent, because a renumbered legal clause
|
||
is a changed document, not a reformatted one.
|
||
"""
|
||
ok: set[int] = set()
|
||
run: list[tuple[Block, int]] = []
|
||
|
||
def flush() -> None:
|
||
# A run of one starting at 1 is still safe: Word renders "1." and the
|
||
# source said "1.". What is never safe is a run starting anywhere else,
|
||
# or one whose numbers are not consecutive.
|
||
if run and run[0][1] == 1:
|
||
expected = 1
|
||
for _block, value in run:
|
||
if value != expected:
|
||
return
|
||
expected += 1
|
||
ok.update(id(block) for block, _v in run)
|
||
|
||
for block in blocks:
|
||
if block.type is not BlockType.list_item or block.list_style != "number":
|
||
flush()
|
||
run = []
|
||
continue
|
||
match = _PLAIN_NUMBER_MARKER.match(block.plain_text() or "")
|
||
if not match:
|
||
flush()
|
||
run = []
|
||
continue
|
||
run.append((block, int(match.group(1))))
|
||
flush()
|
||
return ok
|
||
|
||
|
||
def _page_text_extent(page) -> tuple[float, float, float, float] | None:
|
||
"""Bounding box of a page's *text*, in PDF points, or None.
|
||
|
||
Figures are excluded deliberately. A full-bleed masthead or a background
|
||
image touches all four edges, and letting it set the margins would put
|
||
every paragraph of a report hard against the paper.
|
||
"""
|
||
xs0: list[float] = []
|
||
xs1: list[float] = []
|
||
ys0: list[float] = []
|
||
ys1: list[float] = []
|
||
for block in page.blocks or []:
|
||
if block.type is BlockType.figure or not block.bbox:
|
||
continue
|
||
if not (block.plain_text().strip() or block.cells):
|
||
continue
|
||
box = block.bbox
|
||
width = float(box.w or 0.0)
|
||
height = float(box.h or 0.0)
|
||
if width <= 1.0 or height <= 0.0:
|
||
continue
|
||
xs0.append(float(box.x))
|
||
xs1.append(float(box.x) + width)
|
||
ys0.append(float(box.y))
|
||
ys1.append(float(box.y) + height)
|
||
if not xs0:
|
||
return None
|
||
return min(xs0), min(ys0), max(xs1), max(ys1)
|
||
|
||
|
||
def _percentile(values: list[float], fraction: float) -> float:
|
||
ordered = sorted(values)
|
||
index = min(len(ordered) - 1, max(0, int(round(fraction * (len(ordered) - 1)))))
|
||
return ordered[index]
|
||
|
||
|
||
def _margins_from_content(doc: Document) -> tuple[float, float, float, float] | None:
|
||
"""Margins the source document actually uses, or None to keep the defaults.
|
||
|
||
Every converted DOCX got Word's one-inch defaults whatever the source did,
|
||
so a government report typeset with 22pt margins had 68 points shaved off
|
||
every line and rewrapped, and a fact sheet laid out 541 points wide was
|
||
squeezed into 432. The page size was already taken from the MediaBox;
|
||
this is the other half of making a page look like the page.
|
||
|
||
A percentile rather than a minimum: one page number in the gutter, or a
|
||
rule that overhangs the text, should not set the margin for the document.
|
||
"""
|
||
width = float(doc.pages[0].width or 612.0)
|
||
height = float(doc.pages[0].height or 792.0)
|
||
if width <= 0 or height <= 0:
|
||
return None
|
||
|
||
# Per page, then a low percentile of each margin — not of each coordinate.
|
||
# A page's vertical extent says almost nothing on its own: a title page
|
||
# and a short final page both leave inches of white below the text, and
|
||
# averaging their coordinates gives a document a three-inch bottom margin.
|
||
# The pages that *fill* the text area are the ones that describe it, and
|
||
# taking the tightest margins is how they get the last word.
|
||
lefts: list[float] = []
|
||
rights: list[float] = []
|
||
tops: list[float] = []
|
||
bottoms: list[float] = []
|
||
for page in doc.pages or []:
|
||
extent = _page_text_extent(page)
|
||
if not extent:
|
||
continue
|
||
page_w = float(page.width or width)
|
||
page_h = float(page.height or height)
|
||
x0, y0, x1, y1 = extent
|
||
lefts.append(x0)
|
||
rights.append(page_w - x1)
|
||
tops.append(page_h - y1)
|
||
bottoms.append(y0)
|
||
if not lefts:
|
||
return None
|
||
|
||
left = _percentile(lefts, 0.15)
|
||
right = _percentile(rights, 0.15)
|
||
top = _percentile(tops, 0.15)
|
||
bottom = _percentile(bottoms, 0.15)
|
||
|
||
cap_x = width * MAX_MARGIN_RATIO
|
||
cap_y = height * MAX_MARGIN_RATIO
|
||
return (
|
||
min(max(left, MIN_MARGIN_PT), cap_x),
|
||
min(max(right, MIN_MARGIN_PT), cap_x),
|
||
min(max(top, MIN_MARGIN_PT), cap_y),
|
||
min(max(bottom, MIN_MARGIN_PT), cap_y),
|
||
)
|
||
|
||
|
||
def _configure_page_raster_section(section) -> None:
|
||
"""Fit one rendered PDF page on exactly one Word page."""
|
||
section.top_margin = Pt(PAGE_RASTER_MARGIN_PT)
|
||
section.bottom_margin = Pt(PAGE_RASTER_MARGIN_PT)
|
||
section.left_margin = Pt(PAGE_RASTER_MARGIN_PT)
|
||
section.right_margin = Pt(PAGE_RASTER_MARGIN_PT)
|
||
# The raster already carries the source page's header and footer.
|
||
section.header_distance = Pt(0)
|
||
section.footer_distance = Pt(0)
|
||
|
||
|
||
def _add_page_raster(document, block: Block, page, *, pdf_bytes: bytes | None = None) -> None:
|
||
"""Write a source-page raster without a caption or paragraph drift.
|
||
|
||
When *pdf_bytes* is provided, a hidden text layer is added after the image
|
||
so the page is searchable in Word (Ctrl+F) and accessible to screen
|
||
readers, even though the visible content is a picture.
|
||
"""
|
||
page_w = float(page.width or 612.0)
|
||
page_h = float(page.height or 792.0)
|
||
usable_w = max(1.0, page_w - 2 * PAGE_RASTER_MARGIN_PT)
|
||
usable_h = max(1.0, page_h - 2 * PAGE_RASTER_MARGIN_PT - PAGE_RASTER_PARAGRAPH_SAFETY_PT)
|
||
scale = min(usable_w / page_w, usable_h / page_h)
|
||
p = document.add_paragraph()
|
||
p.paragraph_format.space_before = Pt(0)
|
||
p.paragraph_format.space_after = Pt(0)
|
||
p.paragraph_format.line_spacing = 1
|
||
# ``document.add_page_break()`` creates a separate paragraph. With a
|
||
# near-full-page image that paragraph does not fit after the image, moves
|
||
# to the next page, and then breaks *again* — one blank page between every
|
||
# source page. Page-break-before belongs to the image paragraph itself.
|
||
p.paragraph_format.page_break_before = page.index > 0
|
||
p.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||
p.add_run().add_picture(
|
||
io.BytesIO(_compress_figure_bytes(block.image_png or b"", max_side=1800, quality=78)),
|
||
width=Pt(page_w * scale),
|
||
height=Pt(page_h * scale),
|
||
)
|
||
|
||
# Hidden searchable text layer — white 1pt text behind the image so the
|
||
# page is findable via Ctrl+F and accessible to screen readers.
|
||
if pdf_bytes is not None:
|
||
try:
|
||
from app.services.convert import doc_cache as _dc
|
||
|
||
page_text = _dc.page_text(pdf_bytes, page.index)
|
||
if page_text and page_text.strip():
|
||
hidden = sanitize_ooxml_text(page_text.strip())
|
||
if hidden:
|
||
run = p.add_run(hidden)
|
||
run.font.size = Pt(1)
|
||
run.font.color.rgb = RGBColor(0xFF, 0xFF, 0xFF)
|
||
except Exception:
|
||
pass # searchability is a bonus; never fail the conversion
|
||
|
||
|
||
def format_docx(doc: Document, *, pdf_bytes: bytes | None = None) -> bytes:
|
||
document = DocxDocument()
|
||
style = document.styles["Normal"]
|
||
style.font.name = "Calibri"
|
||
style.font.size = Pt(11)
|
||
|
||
page_raster_fallback = _uses_full_page_raster_fallback(doc)
|
||
header_texts: list[str] = []
|
||
footer_texts: list[str] = []
|
||
header_marks: list[tuple[bytes, float, float]] = []
|
||
if not page_raster_fallback:
|
||
for page in doc.pages:
|
||
for block in page.blocks:
|
||
if _is_masthead_mark(block):
|
||
w_in, h_in = _masthead_size_inches(block, page.width)
|
||
header_marks.append((block.image_png, w_in, h_in))
|
||
continue
|
||
t = block.plain_text().strip()
|
||
if not t:
|
||
continue
|
||
if block.type == BlockType.header and t not in header_texts:
|
||
header_texts.append(t)
|
||
if block.type == BlockType.footer and t not in footer_texts:
|
||
if looks_like_page_number(t):
|
||
# A per-page stamp: it becomes one PAGE field below, not
|
||
# forty different literal footers.
|
||
continue
|
||
footer_texts.append(t)
|
||
|
||
section = document.sections[0]
|
||
if doc.pages:
|
||
try:
|
||
# PDF points → twips (1pt = 20 twips)
|
||
section.page_width = Twips(int(doc.pages[0].width * 20))
|
||
section.page_height = Twips(int(doc.pages[0].height * 20))
|
||
except Exception as exc:
|
||
doc.warnings.append(f"page size from mediabox failed: {exc}")
|
||
if page_raster_fallback:
|
||
_configure_page_raster_section(section)
|
||
else:
|
||
margins = _margins_from_content(doc)
|
||
if margins:
|
||
left, right, top, bottom = margins
|
||
try:
|
||
section.left_margin = Pt(left)
|
||
section.right_margin = Pt(right)
|
||
section.top_margin = Pt(top)
|
||
section.bottom_margin = Pt(bottom)
|
||
# Header and footer sit inside the margin they were measured
|
||
# against; leaving Word's half-inch default would push the
|
||
# body down past text that is already accounted for.
|
||
section.header_distance = Pt(min(top * 0.5, 36.0))
|
||
section.footer_distance = Pt(min(bottom * 0.5, 36.0))
|
||
except Exception as exc:
|
||
doc.warnings.append(f"page margins from content failed: {exc}")
|
||
_fill_header_marks(section.header, header_marks, doc.warnings)
|
||
_fill_header_footer(section.header, header_texts)
|
||
_fill_header_footer(section.footer, footer_texts)
|
||
if (doc.meta or {}).get("page_number_footer"):
|
||
_add_page_number_field(section.footer)
|
||
|
||
# Decided once for the whole document: a run of list items can only be
|
||
# judged against its neighbours, not one paragraph at a time.
|
||
auto_numbered = _auto_numberable([b for pg in doc.pages for b in pg.blocks])
|
||
|
||
merge_warnings: list[str] = []
|
||
link_warnings: list[str] = []
|
||
exact_mode = _exact_layout_requested()
|
||
unpositioned = 0
|
||
unpositioned_tables = 0
|
||
flowed_pages = 0
|
||
for page in doc.pages:
|
||
if page.index > 0 and not page_raster_fallback:
|
||
document.add_page_break()
|
||
# Positioned emit is per page: a brochure cover followed by pages of
|
||
# prose should keep the cover's arrangement without freezing the prose
|
||
# into frames the reader cannot reflow.
|
||
positioned = exact_mode and docx_exact.page_is_positionable(page)
|
||
if exact_mode and not positioned:
|
||
# Exact was asked for and could not be delivered on this page —
|
||
# usually because the geometry is the plain-text fallback's, which
|
||
# looks real and describes nothing. Silence here would leave the
|
||
# customer believing the arrangement was preserved.
|
||
flowed_pages += 1
|
||
for block in sorted(page.blocks, key=lambda b: b.reading_order):
|
||
if block.type in (BlockType.header, BlockType.footer):
|
||
continue
|
||
if _is_masthead_mark(block):
|
||
continue
|
||
if block.type == BlockType.figure and block.source == PAGE_RASTER_SOURCE:
|
||
_add_page_raster(document, block, page, pdf_bytes=pdf_bytes)
|
||
continue
|
||
body_text = block.plain_text().strip()
|
||
# Speck filtering belongs to recognised text only. "@", "*" and
|
||
# ".r-^" are what a recogniser emits for a stamp or a rule; when
|
||
# they come from a PDF's own text layer they are content — a
|
||
# footnote marker, a section sign, a bullet the author typed — and
|
||
# dropping them deleted a paragraph the customer wrote.
|
||
recognised = (
|
||
getattr(block, "source", "") == "ocr"
|
||
or page.kind in (PageKind.scan, PageKind.hybrid)
|
||
)
|
||
if (
|
||
body_text
|
||
and recognised
|
||
and is_ocr_noise_line(body_text)
|
||
and block.type != BlockType.table
|
||
):
|
||
continue
|
||
place = positioned and docx_exact.has_usable_geometry(block)
|
||
if positioned and not place and block.plain_text().strip():
|
||
unpositioned += 1
|
||
if block.type == BlockType.heading:
|
||
level = min(max(block.level, 1), 3)
|
||
h = document.add_heading("", level=level)
|
||
_apply_spans(
|
||
h,
|
||
block.spans,
|
||
fallback=block.text or block.plain_text(),
|
||
warnings=link_warnings,
|
||
visual=_block_visual(block),
|
||
)
|
||
if contains_arabic(block.plain_text()):
|
||
_set_paragraph_rtl(h)
|
||
for run in h.runs:
|
||
run.font.name = "Arial"
|
||
else:
|
||
h.alignment = _ALIGN.get(block.align, WD_ALIGN_PARAGRAPH.LEFT)
|
||
if place:
|
||
docx_exact.position_paragraph(h, block, page)
|
||
elif block.type == BlockType.table and block.cells:
|
||
rows = len(block.cells)
|
||
cols = max(len(r) for r in block.cells)
|
||
table = document.add_table(rows=rows, cols=cols)
|
||
table.style = "Table Grid"
|
||
cell_visual = _block_visual(block)
|
||
for ri, row in enumerate(block.cells):
|
||
for ci in range(cols):
|
||
cell = table.rows[ri].cells[ci]
|
||
raw_cell = row[ci] if ci < len(row) else ""
|
||
cell.text = sanitize_ooxml_text(to_logical(raw_cell, visual=cell_visual))
|
||
if contains_arabic(raw_cell):
|
||
for p in cell.paragraphs:
|
||
_set_paragraph_rtl(p)
|
||
for run in p.runs:
|
||
run.font.name = "Arial"
|
||
if ri == 0:
|
||
for p in cell.paragraphs:
|
||
for run in p.runs:
|
||
run.bold = True
|
||
for r0, c0, r1, c1 in block.merges:
|
||
try:
|
||
table.rows[r0].cells[c0].merge(table.rows[r1].cells[c1])
|
||
except Exception as exc:
|
||
merge_warnings.append(f"merge failed ({r0},{c0})-({r1},{c1}): {exc}")
|
||
if positioned:
|
||
# A grid that cannot be placed stays flowing rather than
|
||
# being dropped: a table in the wrong place is still the
|
||
# customer's data; a missing table is not.
|
||
if place and docx_exact.position_table(table, block, page):
|
||
pass
|
||
else:
|
||
unpositioned_tables += 1
|
||
else:
|
||
document.add_paragraph("")
|
||
elif block.type == BlockType.figure and block.image_png:
|
||
if block.text and not place:
|
||
# In a frame the caption text would sit on top of the
|
||
# picture; positioned figures carry the image alone.
|
||
document.add_paragraph(
|
||
sanitize_ooxml_text(to_logical(block.text, visual=_block_visual(block)))
|
||
)
|
||
try:
|
||
# Skip pathological multi-MB page dumps even if they slipped through
|
||
if len(block.image_png) > 2_500_000:
|
||
doc.warnings.append("Skipped oversized figure to keep DOCX usable.")
|
||
elif place:
|
||
# Positioned: the picture goes in a frame at the block's
|
||
# own rectangle, like every other block on the page.
|
||
# Flowing it here is what made brochures come out with
|
||
# floating captions above stacked images.
|
||
payload = _compress_figure_bytes(block.image_png)
|
||
w_pt, h_pt = docx_exact.figure_size_points(block, page)
|
||
para = document.add_paragraph()
|
||
para.add_run().add_picture(
|
||
io.BytesIO(payload), width=Pt(w_pt), height=Pt(h_pt)
|
||
)
|
||
docx_exact.position_figure(para, block, page)
|
||
else:
|
||
payload = _compress_figure_bytes(block.image_png)
|
||
w_in = _figure_width_inches(block, page.width)
|
||
document.add_picture(io.BytesIO(payload), width=Inches(w_in))
|
||
except Exception as exc:
|
||
document.add_paragraph(sanitize_ooxml_text(block.plain_text()) or "(image)")
|
||
doc.warnings.append(f"figure embed failed: {exc}")
|
||
elif block.type == BlockType.list_item:
|
||
# Word renumbers a "List Number" paragraph from one and throws
|
||
# the source's marker away. That is only safe where it
|
||
# reproduces the document exactly; everywhere else the
|
||
# document's own numbering is kept as text, indented so it
|
||
# still reads as a list.
|
||
auto = block.list_style == "number" and id(block) in auto_numbered
|
||
bullet = block.list_style == "bullet"
|
||
style_name = "List Number" if auto else ("List Bullet" if bullet else None)
|
||
try:
|
||
p = (
|
||
document.add_paragraph(style=style_name)
|
||
if style_name
|
||
else document.add_paragraph()
|
||
)
|
||
except Exception:
|
||
p = document.add_paragraph()
|
||
raw = block.plain_text()
|
||
spans = block.spans
|
||
if auto or bullet:
|
||
cleaned = re.sub(_LIST_MARKER_STRIP, "", raw)
|
||
if spans and spans[0].text:
|
||
spans = [
|
||
TextSpan(
|
||
text=re.sub(_LIST_MARKER_STRIP, "", spans[0].text),
|
||
font_name=spans[0].font_name,
|
||
font_size=spans[0].font_size,
|
||
bold=spans[0].bold,
|
||
italic=spans[0].italic,
|
||
url=spans[0].url,
|
||
visual_order=getattr(spans[0], "visual_order", False),
|
||
)
|
||
] + list(spans[1:])
|
||
else:
|
||
cleaned = raw
|
||
if _list_marker(block):
|
||
# A hanging indent puts the wrapped lines under the
|
||
# text rather than under the number, which is what
|
||
# makes it read as a list without Word owning it.
|
||
p.paragraph_format.left_indent = Inches(0.25 + 0.25 * block.level)
|
||
p.paragraph_format.first_line_indent = Inches(-0.25)
|
||
_apply_spans(
|
||
p,
|
||
spans,
|
||
fallback=cleaned,
|
||
warnings=link_warnings,
|
||
visual=_block_visual(block),
|
||
)
|
||
if contains_arabic(raw):
|
||
_set_paragraph_rtl(p)
|
||
for run in p.runs:
|
||
run.font.name = "Arial"
|
||
if block.level >= 1 and (auto or bullet):
|
||
# A literal-marker item already set its own hanging indent.
|
||
p.paragraph_format.left_indent = Inches(0.25 * block.level)
|
||
if place:
|
||
docx_exact.position_paragraph(p, block, page)
|
||
else:
|
||
text = block.plain_text()
|
||
if sanitize_ooxml_text(text).strip():
|
||
p = document.add_paragraph()
|
||
_apply_spans(
|
||
p,
|
||
block.spans,
|
||
fallback=text,
|
||
warnings=link_warnings,
|
||
visual=_block_visual(block),
|
||
)
|
||
if contains_arabic(text):
|
||
_set_paragraph_rtl(p)
|
||
for run in p.runs:
|
||
run.font.name = "Arial"
|
||
else:
|
||
p.alignment = _ALIGN.get(block.align, WD_ALIGN_PARAGRAPH.LEFT)
|
||
if place:
|
||
docx_exact.position_paragraph(p, block, page)
|
||
|
||
if unpositioned:
|
||
doc.warnings.append(
|
||
f"{unpositioned} block(s) had no measurable position and were placed "
|
||
"in reading order instead."
|
||
)
|
||
if flowed_pages:
|
||
doc.warnings.append(
|
||
f"{flowed_pages} page(s) had no measurable geometry and were rebuilt as "
|
||
"flowing text rather than positioned frames."
|
||
)
|
||
if unpositioned_tables:
|
||
# Said once per document, not once per table: a page of unplaceable
|
||
# grids should not bury the warnings that matter.
|
||
doc.warnings.append(
|
||
f"{unpositioned_tables} table(s) had no measurable position and were left "
|
||
"flowing in reading order; their contents are complete."
|
||
)
|
||
if merge_warnings:
|
||
doc.warnings.extend(merge_warnings)
|
||
if link_warnings:
|
||
doc.warnings.extend(link_warnings)
|
||
|
||
buf = io.BytesIO()
|
||
document.save(buf)
|
||
return buf.getvalue()
|