187 lines
6.3 KiB
Python
187 lines
6.3 KiB
Python
"""Block typing: heading, list, paragraph."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
|
||
from app.services.convert.idm.model import BBox, Block, BlockType, TextSpan
|
||
from app.services.convert.layout.glyphs import Line
|
||
from app.services.convert.layout.styles import heading_level, is_caption
|
||
|
||
_BULLET_RE = re.compile(r"""^(\u2022|\-|\*|•|○|●|◦|·|▪|['"`])\s+""")
|
||
_NUMBER_RE = re.compile(r"^(\d+[\.\)]|\(\d+\))\s+")
|
||
_LETTER_RE = re.compile(r"^([A-Za-z][\.\)]|\([A-Za-z]\))\s+")
|
||
# OCR often turns a bullet into a lone I / l before the sentence
|
||
_OCR_BULLET_RE = re.compile(r"^[Il]\s+[A-ZÀ-ÖØ-Þ]")
|
||
_URI_RE = re.compile(r"(https?://[^\s<>]+|www\.[^\s<>]+)", re.I)
|
||
|
||
|
||
def _annotate_uris(spans: list[TextSpan]) -> list[TextSpan]:
|
||
out: list[TextSpan] = []
|
||
for s in spans:
|
||
m = _URI_RE.search(s.text or "")
|
||
if m and not s.url:
|
||
out.append(
|
||
TextSpan(
|
||
text=s.text,
|
||
font_name=s.font_name,
|
||
font_size=s.font_size,
|
||
bold=s.bold,
|
||
italic=s.italic,
|
||
x=s.x,
|
||
w=s.w,
|
||
url=m.group(1),
|
||
)
|
||
)
|
||
else:
|
||
out.append(s)
|
||
return out
|
||
|
||
|
||
def _line_is_bold(line: Line) -> bool:
|
||
"""Whether essentially the whole line is bold, weighted by character count.
|
||
|
||
A part-bold line is emphasis inside prose -- "**Note:** the rest of the
|
||
sentence" -- and must not read as a heading, so the test is share of
|
||
characters rather than presence of any bold span.
|
||
"""
|
||
total = bold = 0
|
||
for span in line.spans or []:
|
||
n = len((span.text or "").strip())
|
||
if not n:
|
||
continue
|
||
total += n
|
||
if span.bold:
|
||
bold += n
|
||
return total > 0 and (bold / total) >= 0.8
|
||
|
||
|
||
def line_to_block(line: Line, order: int, body_size: float) -> Block:
|
||
text = line.text.strip()
|
||
list_style = ""
|
||
nest = 0
|
||
# M2: prefer list detection over heading for bullet/number markers
|
||
if _BULLET_RE.match(text) or _OCR_BULLET_RE.match(text):
|
||
btype = BlockType.list_item
|
||
list_style = "bullet"
|
||
level = 0
|
||
elif _NUMBER_RE.match(text):
|
||
btype = BlockType.list_item
|
||
list_style = "number"
|
||
level = 0
|
||
elif _LETTER_RE.match(text):
|
||
btype = BlockType.list_item
|
||
list_style = "letter"
|
||
level = 0
|
||
elif is_caption(text):
|
||
btype = BlockType.paragraph
|
||
level = 0
|
||
else:
|
||
level = heading_level(text, line.font_size, body_size, bold=_line_is_bold(line))
|
||
if level:
|
||
btype = BlockType.heading
|
||
else:
|
||
btype = BlockType.paragraph
|
||
level = 0
|
||
# Indent heuristic → shallow nest
|
||
if btype == BlockType.list_item and line.x0 > 100:
|
||
nest = 1
|
||
spans = line.spans or [TextSpan(text=text, font_size=line.font_size, font_name=line.font_name)]
|
||
spans = _annotate_uris(spans)
|
||
return Block(
|
||
type=btype,
|
||
text=text,
|
||
level=level or nest,
|
||
bbox=BBox(x=line.x0, y=line.y, w=max(line.x1 - line.x0, 1), h=line.font_size),
|
||
spans=spans,
|
||
reading_order=order,
|
||
list_style=list_style,
|
||
align="left",
|
||
synthetic_geometry=bool(getattr(line, "synthetic_geometry", False)),
|
||
)
|
||
|
||
|
||
def lines_to_block(
|
||
lines: list[Line],
|
||
order: int,
|
||
body_size: float,
|
||
*,
|
||
lexical_pairs: set[tuple[str, str]] | None = None,
|
||
) -> Block:
|
||
"""Build one block from a paragraph run of wrapped lines.
|
||
|
||
Single-line runs delegate to :func:`line_to_block` so heading, caption and
|
||
list detection behave identically to the per-line path. Multi-line runs are
|
||
joined with hyphenation resolved, and carry the union bbox plus the
|
||
concatenated spans so run-level styling survives into the formatters.
|
||
"""
|
||
if not lines:
|
||
raise ValueError("lines_to_block requires at least one line")
|
||
if len(lines) == 1:
|
||
return line_to_block(lines[0], order, body_size)
|
||
|
||
from app.services.convert.layout.paragraphs import join_line_texts
|
||
|
||
text = join_line_texts([ln.text for ln in lines], lexical_pairs=lexical_pairs)
|
||
first = lines[0]
|
||
|
||
spans: list[TextSpan] = []
|
||
for idx, ln in enumerate(lines):
|
||
line_spans = ln.spans or [
|
||
TextSpan(text=ln.text, font_size=ln.font_size, font_name=ln.font_name)
|
||
]
|
||
if idx > 0 and spans and line_spans:
|
||
# Wrapped lines are separated by a space unless the previous line
|
||
# was hyphenated, which join_line_texts has already resolved.
|
||
prev_text = spans[-1].text or ""
|
||
if prev_text and not prev_text.endswith((" ", "-", "")):
|
||
spans.append(
|
||
TextSpan(
|
||
text=" ",
|
||
font_size=line_spans[0].font_size,
|
||
font_name=line_spans[0].font_name,
|
||
)
|
||
)
|
||
elif prev_text.endswith(("-", "")) and not text.count(prev_text[-1]):
|
||
# join_line_texts removed the hyphen; mirror that in the spans.
|
||
spans[-1] = TextSpan(
|
||
text=prev_text[:-1],
|
||
font_name=spans[-1].font_name,
|
||
font_size=spans[-1].font_size,
|
||
bold=spans[-1].bold,
|
||
italic=spans[-1].italic,
|
||
x=spans[-1].x,
|
||
w=spans[-1].w,
|
||
url=spans[-1].url,
|
||
color=spans[-1].color,
|
||
visual_order=spans[-1].visual_order,
|
||
)
|
||
spans.extend(line_spans)
|
||
spans = _annotate_uris(spans)
|
||
|
||
x0 = min(ln.x0 for ln in lines)
|
||
x1 = max(ln.x1 for ln in lines)
|
||
y_top = max(ln.y for ln in lines)
|
||
y_bot = min(ln.y for ln in lines)
|
||
height = max(y_top - y_bot + (first.font_size or body_size), first.font_size or body_size)
|
||
|
||
return Block(
|
||
type=BlockType.paragraph,
|
||
text=text,
|
||
level=0,
|
||
bbox=BBox(x=x0, y=y_bot, w=max(x1 - x0, 1), h=height),
|
||
spans=spans,
|
||
reading_order=order,
|
||
align="left",
|
||
synthetic_geometry=any(
|
||
getattr(ln, "synthetic_geometry", False) for ln in lines
|
||
),
|
||
)
|
||
|
||
|
||
def estimate_body_size(lines: list[Line]) -> float:
|
||
if not lines:
|
||
return 12.0
|
||
sizes = sorted(ln.font_size for ln in lines if ln.font_size > 0)
|
||
return sizes[len(sizes) // 2]
|