605 lines
26 KiB
Python
605 lines
26 KiB
Python
"""Glyph / run clustering into lines."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
from app.services.convert.idm.model import TextSpan
|
|
from app.services.convert.layout.styles import infer_font_flags, style_key
|
|
from app.services.convert.text.arabic_logical import is_rtl_dominant
|
|
|
|
# How many *consecutive* text rows a band must stay blank down before it counts
|
|
# as a gutter. Five is enough to rule out a stretched space in justified text
|
|
# and short enough that a full-width figure dropped between two columns still
|
|
# leaves a usable run of rows above and below itself.
|
|
MIN_GUTTER_ROWS = 5
|
|
|
|
# A PDF text walker can place a continuation glyph a fraction of a point on
|
|
# the far side of a detected gutter. This is common when a zero-width space
|
|
# marker and its following character are emitted with separate text matrices:
|
|
# the marker/character pair is one word, not a new page column. Keep the
|
|
# tolerance below ordinary inter-column spacing so genuine columns remain
|
|
# isolated.
|
|
GUTTER_EDGE_CONTINUATION_PT = 2.0
|
|
|
|
|
|
@dataclass
|
|
class Line:
|
|
y: float
|
|
x0: float
|
|
x1: float
|
|
text: str
|
|
font_size: float = 12.0
|
|
font_name: str = ""
|
|
spans: list[TextSpan] = field(default_factory=list)
|
|
# True when x/y were fabricated by lines_from_plain_text rather than measured
|
|
# from glyph geometry. Consumers must not draw geometric conclusions
|
|
# (columns, indents, line width) from a line carrying this flag.
|
|
synthetic_geometry: bool = False
|
|
|
|
|
|
def _span_from_glyph(g: dict, default_size: float) -> TextSpan | None:
|
|
t = str(g.get("text", ""))
|
|
if not t:
|
|
return None
|
|
w = float(g.get("w", 0) or 0)
|
|
# PDF producers commonly encode a word space as a zero-width glyph whose
|
|
# advance is carried by the following text matrix. Dropping that marker
|
|
# forces the line builder to infer every boundary from tiny x gaps; in
|
|
# tightly set fonts those gaps are indistinguishable from kerning and
|
|
# ordinary words get welded together ("Tracebased" / "Justintime").
|
|
# Keep the marker with zero width so the explicit source signal wins.
|
|
fname = str(g.get("fontName") or g.get("font_name") or "")
|
|
h = float(g.get("h", 0) or 0)
|
|
raw_size = float(g.get("fontSize") or 0)
|
|
if raw_size <= 2.0 and h > 2.0:
|
|
fsize = h
|
|
elif raw_size > 0:
|
|
fsize = raw_size
|
|
elif h > 0:
|
|
fsize = h
|
|
else:
|
|
fsize = default_size
|
|
bold = bool(g.get("bold")) if "bold" in g else False
|
|
italic = bool(g.get("italic")) if "italic" in g else False
|
|
if not bold and not italic:
|
|
ib, ii = infer_font_flags(fname)
|
|
bold = bold or ib
|
|
italic = italic or ii
|
|
return TextSpan(
|
|
text=t,
|
|
font_name=fname,
|
|
font_size=fsize,
|
|
bold=bold,
|
|
italic=italic,
|
|
x=float(g.get("x", 0)),
|
|
w=w or len(t) * 6,
|
|
color=(str(g["color"]) if g.get("color") else None),
|
|
)
|
|
|
|
|
|
def _runs_on(cur: TextSpan, nxt: TextSpan) -> bool:
|
|
"""Whether *nxt* continues *cur* without an intervening gap.
|
|
|
|
A merged span keeps the first span's ``x`` and the *sum* of the widths, so
|
|
merging across a gap yields a span that claims to start where the first one
|
|
did and to be far narrower than the ground it covers. Every consumer that
|
|
asks "which column does this text sit in?" is then told the wrong answer —
|
|
which is how a table row's six cells collapsed into its first one.
|
|
"""
|
|
if cur.x is None or nxt.x is None:
|
|
return True
|
|
gap = float(nxt.x) - (float(cur.x) + float(cur.w or 0.0))
|
|
if gap <= 0:
|
|
return True
|
|
# An inter-word space runs about a third of an em. Anything appreciably
|
|
# wider is column spacing, a tab stop or a dot leader.
|
|
return gap <= max(2.0, float(cur.font_size or 12.0) * 0.8)
|
|
|
|
|
|
def coalesce_spans(spans: list[TextSpan]) -> list[TextSpan]:
|
|
"""Merge adjacent spans that share a style *and* run on without a gap."""
|
|
if not spans:
|
|
return []
|
|
out: list[TextSpan] = []
|
|
cur = TextSpan(
|
|
text=spans[0].text,
|
|
font_name=spans[0].font_name,
|
|
font_size=spans[0].font_size,
|
|
bold=spans[0].bold,
|
|
italic=spans[0].italic,
|
|
x=spans[0].x,
|
|
w=spans[0].w,
|
|
color=spans[0].color,
|
|
)
|
|
cur_right = float(cur.x or 0) + float(cur.w or 0)
|
|
for s in spans[1:]:
|
|
sx = float(s.x or 0)
|
|
sw = float(s.w or 0)
|
|
gap = sx - cur_right
|
|
same_style = (
|
|
(
|
|
style_key(cur.font_name, cur.font_size, cur.bold, cur.italic)
|
|
== style_key(s.font_name, s.font_size, s.bold, s.italic)
|
|
or (
|
|
cur.font_name.lower().strip() == s.font_name.lower().strip()
|
|
and cur.bold == s.bold
|
|
and cur.italic == s.italic
|
|
and abs(float(cur.font_size or 0) - float(s.font_size or 0)) <= 2.0
|
|
)
|
|
)
|
|
and cur.color == s.color
|
|
)
|
|
max_gap = max(2.0, float(cur.font_size or 12.0) * 0.8)
|
|
if same_style and (gap <= max_gap or gap <= 0):
|
|
cur.text += s.text
|
|
cur_right = max(cur_right, sx + sw)
|
|
if cur.x is not None:
|
|
cur.w = cur_right - float(cur.x)
|
|
elif s.w is not None:
|
|
cur.w = s.w
|
|
else:
|
|
out.append(cur)
|
|
cur = TextSpan(
|
|
text=s.text,
|
|
font_name=s.font_name,
|
|
font_size=s.font_size,
|
|
bold=s.bold,
|
|
italic=s.italic,
|
|
x=s.x,
|
|
w=s.w,
|
|
color=s.color,
|
|
)
|
|
cur_right = sx + sw
|
|
out.append(cur)
|
|
return out
|
|
|
|
|
|
def page_gutters(
|
|
glyphs: list[dict], *, min_width: float = 12.0, row_height: float = 12.0
|
|
) -> list[tuple[float, float, float]]:
|
|
"""Vertical bands almost no row of the page writes in — its column gutters.
|
|
|
|
A projection profile over the whole page, not a per-line gap test. The
|
|
gutter of a two-column paper is often only 20pt wide, so a fixed
|
|
"gap > 36pt means two columns" rule welds both columns of every academic
|
|
paper into single lines; and a *per-line* threshold cannot tell a stretched
|
|
space in justified text from a real gutter, because on one line they look
|
|
identical. Across the page they do not: a gutter is empty on nearly every
|
|
row, a stretched space is not.
|
|
|
|
The profile is *local*, computed over a sliding window of neighbouring
|
|
rows rather than the whole page. A page is rarely one thing throughout: a
|
|
paper puts a full-width title and author block above two columns of
|
|
abstract, and a full-width figure in the middle of them. A page-wide
|
|
profile finds no gutter on such a page at all, because the title crosses
|
|
every candidate band — and then both columns of the abstract come out
|
|
interleaved line by line.
|
|
|
|
Returns ``(x, y_low, y_high)`` for each gutter band, so a caller can apply
|
|
a gutter only to the rows it actually separates.
|
|
"""
|
|
if len(glyphs) < 12:
|
|
return []
|
|
xs0 = [float(g.get("x", 0.0)) for g in glyphs]
|
|
xs1 = [x + float(g.get("w", 0.0) or 0.0) for x, g in zip(xs0, glyphs, strict=False)]
|
|
left, right = min(xs0), max(xs1)
|
|
span = right - left
|
|
if span <= 0:
|
|
return []
|
|
|
|
step = max(row_height, 1.0)
|
|
bin_pt = 2.0
|
|
bins = max(8, min(2048, int(span / bin_pt) + 1))
|
|
per_bin = span / bins
|
|
need = max(2, int(min_width / per_bin))
|
|
|
|
# Cluster glyphs into baseline rows so descenders (p, g, y) do not fall
|
|
# into a separate row and create false gutter detections.
|
|
ordered_y = sorted(glyphs, key=lambda g: -float(g.get("y", 0.0)))
|
|
row_clusters: list[list[dict]] = []
|
|
row_ys: list[float] = []
|
|
cluster_tol = max(step * 0.7, 4.0)
|
|
for g in ordered_y:
|
|
gy = float(g.get("y", 0.0))
|
|
placed = False
|
|
for idx in range(len(row_clusters) - 1, -1, -1):
|
|
if abs(gy - row_ys[idx]) <= cluster_tol:
|
|
row_clusters[idx].append(g)
|
|
placed = True
|
|
break
|
|
if row_ys[idx] - gy > cluster_tol:
|
|
break
|
|
if not placed:
|
|
row_clusters.append([g])
|
|
row_ys.append(gy)
|
|
|
|
# One occupancy bitmap per text row, top of page first.
|
|
occ: dict[int, int] = {}
|
|
for cl, ry in zip(row_clusters, row_ys, strict=False):
|
|
row_key = int(round(ry / step))
|
|
bits = 0
|
|
for g in cl:
|
|
gx0 = float(g.get("x", 0.0))
|
|
gw = float(g.get("w", 0.0) or 0.0)
|
|
i = max(0, min(bins - 1, int((gx0 - left) / span * bins)))
|
|
j = max(0, min(bins - 1, int((gx0 + gw - left) / span * bins)))
|
|
bits |= (((1 << (j - i + 1)) - 1) << i)
|
|
occ[row_key] = occ.get(row_key, 0) | bits
|
|
|
|
order = sorted(occ, reverse=True)
|
|
if len(order) < MIN_GUTTER_ROWS:
|
|
return []
|
|
|
|
full = (1 << bins) - 1
|
|
# Bins each row leaves blank, top of page first.
|
|
blank = [full ^ occ[r] for r in order]
|
|
|
|
# A column must be substantially wider than the gutter beside it. Without
|
|
# this, any persistently blank band qualifies -- and a form is full of them:
|
|
# the numbered stub down the left of an IRS 1040 leaves a real, unbroken gap
|
|
# between the item number and its label, so "24 Add lines 22 and 23" was
|
|
# split into "24" and "Add lines 22 and 23", detaching every line number
|
|
# from the line it numbers. A 10pt-wide column of two-digit numbers is a
|
|
# stub, not a text column.
|
|
min_side = max(min_width * 3.0, span * 0.12)
|
|
|
|
out: list[tuple[float, float, float]] = []
|
|
# A gutter is a band that stays blank down a *run* of consecutive rows.
|
|
#
|
|
# This used to be a window centred on each row that required unanimity,
|
|
# which inverted the intent described above: the three rows below a
|
|
# full-width title, author block or figure caption still had that full-width
|
|
# row inside their window, so the gutter was suppressed for exactly the
|
|
# first rows of the column region -- the rows where the columns begin.
|
|
# Measured on the TraceMonkey paper, the real x=305 gutter was found only
|
|
# from y=421 down while the two-column abstract starts at y=445, so its
|
|
# first rows were never split, both columns welded into single lines, and
|
|
# "...more difficult to com-" was joined to "applications such as Google
|
|
# Mail..." to yield "Zimbra Colpile". Recall cannot see that -- every word is
|
|
# still present, in the wrong order and partly re-spelled -- which is how it
|
|
# survived.
|
|
#
|
|
# Runs carry the same evidence without the boundary artefact: a band is a
|
|
# gutter at every row of any run of MIN_GUTTER_ROWS consecutive rows it
|
|
# stays blank down. A stretched space in justified text is blank on one or
|
|
# two rows and still cannot reach the run length, so the false positive the
|
|
# projection profile exists to avoid is still avoided; and a full-width
|
|
# figure between two columns now ends one run and starts another instead of
|
|
# erasing three rows on each side of itself.
|
|
for s in range(len(order) - MIN_GUTTER_ROWS + 1):
|
|
rows = order[s : s + MIN_GUTTER_ROWS]
|
|
empty = full
|
|
union = 0
|
|
for i in range(s, s + MIN_GUTTER_ROWS):
|
|
empty &= blank[i]
|
|
union |= occ[order[i]]
|
|
if not empty or not union:
|
|
continue
|
|
bits = bin(empty)[2:].zfill(bins)[::-1]
|
|
# The ground the run's own rows cover. Judged over the run rather than a
|
|
# neighbourhood, because the question is what *this* band separates: a
|
|
# row that writes straight through the band is not one of its columns.
|
|
used = bin(union)[2:].zfill(bins)[::-1]
|
|
first, last = used.find("1"), used.rfind("1")
|
|
run = 0
|
|
for k in range(bins + 1):
|
|
if k < bins and bits[k] == "1":
|
|
run += 1
|
|
continue
|
|
if run >= need:
|
|
start = k - run
|
|
# Interior only: a gutter has content on both sides of it, and
|
|
# enough of it on each side to be a column.
|
|
lo_end, hi_start = used.rfind("1", 0, start), used.find("1", k)
|
|
if lo_end >= 0 and hi_start >= 0:
|
|
left_w = (lo_end - first + 1) * per_bin
|
|
right_w = (last - hi_start + 1) * per_bin
|
|
if left_w >= min_side and right_w >= min_side:
|
|
x = left + (start + k) / 2.0 * per_bin
|
|
out.extend((x, r * step, r * step) for r in rows)
|
|
run = 0
|
|
|
|
return _merge_gutter_spans(out, step)
|
|
|
|
|
|
def _merge_gutter_spans(
|
|
spans: list[tuple[float, float, float]], step: float
|
|
) -> list[tuple[float, float, float]]:
|
|
"""Collapse per-row gutter hits into ``(x, y_low, y_high)`` bands."""
|
|
if not spans:
|
|
return []
|
|
merged: list[list[float]] = []
|
|
for x, y0, y1 in sorted(spans, key=lambda s: (round(s[0], 0), s[1])):
|
|
for band in merged:
|
|
if abs(band[0] - x) <= 4.0 and y0 <= band[2] + step * 2.5 and y1 >= band[1] - step * 2.5:
|
|
band[0] = (band[0] + x) / 2.0
|
|
band[1] = min(band[1], y0)
|
|
band[2] = max(band[2], y1)
|
|
break
|
|
else:
|
|
merged.append([x, y0, y1])
|
|
return [(b[0], b[1], b[2]) for b in merged]
|
|
|
|
|
|
def _split_group_at(
|
|
group: list[dict], gutters: list[tuple[float, float, float]], *, slack: float = 6.0
|
|
) -> list[list[dict]]:
|
|
"""Split one y-group wherever it straddles a gutter that spans its rows."""
|
|
if not gutters or len(group) < 2:
|
|
return [group]
|
|
ys = [float(g.get("y", 0.0)) for g in group]
|
|
y = sum(ys) / len(ys)
|
|
here = sorted(x for x, lo, hi in gutters if lo - slack <= y <= hi + slack)
|
|
if not here:
|
|
return [group]
|
|
group = sorted(group, key=lambda g: float(g.get("x", 0)))
|
|
parts: list[list[dict]] = []
|
|
current: list[dict] = []
|
|
idx = 0
|
|
for g in group:
|
|
x = float(g.get("x", 0))
|
|
cur_right = max(
|
|
(float(c.get("x", 0)) + float(c.get("w", 0) or 0) for c in current),
|
|
default=0.0,
|
|
)
|
|
while idx < len(here) and x >= here[idx]:
|
|
# Only split if there is an actual gap separating columns across the gutter
|
|
if current and (x - cur_right >= 4.0) and (cur_right <= here[idx] + 2.0):
|
|
parts.append(current)
|
|
current = []
|
|
idx += 1
|
|
current.append(g)
|
|
if current:
|
|
parts.append(current)
|
|
return parts or [group]
|
|
|
|
|
|
def _split_group_on_gutter(group: list[dict], *, min_gap: float = 36.0) -> list[list[dict]]:
|
|
"""Recursively split a y-clustered glyph group on large horizontal gutters (2+ columns)."""
|
|
if len(group) < 2:
|
|
return [group]
|
|
group = sorted(group, key=lambda g: float(g.get("x", 0)))
|
|
best_i = -1
|
|
best_gap = 0.0
|
|
for i in range(len(group) - 1):
|
|
x0 = float(group[i].get("x", 0))
|
|
w0 = float(group[i].get("w", 0) or 0)
|
|
x1 = float(group[i + 1].get("x", 0))
|
|
gap = x1 - (x0 + w0)
|
|
if gap > best_gap:
|
|
best_gap = gap
|
|
best_i = i
|
|
if best_i < 0 or best_gap < min_gap:
|
|
return [group]
|
|
left, right = group[: best_i + 1], group[best_i + 1 :]
|
|
if not left or not right:
|
|
return [group]
|
|
out: list[list[dict]] = []
|
|
out.extend(_split_group_on_gutter(left, min_gap=min_gap))
|
|
out.extend(_split_group_on_gutter(right, min_gap=min_gap))
|
|
return out
|
|
|
|
|
|
def _group_is_rtl(group: list[dict], *, threshold: float = 0.5) -> bool:
|
|
"""True when a line's letters are predominantly right-to-left."""
|
|
text = "".join(str(g.get("text", "")) for g in group)
|
|
return is_rtl_dominant(text, threshold=threshold)
|
|
|
|
|
|
def _sort_group_by_reading_order(group: list[dict]) -> list[dict]:
|
|
"""Order a line's glyphs in logical reading order for its dominant script."""
|
|
if _group_is_rtl(group):
|
|
# Right-to-left: the logical first token sits furthest right. Sort by
|
|
# the right edge descending so a wide glyph does not overtake a narrow
|
|
# neighbour that starts further right.
|
|
return sorted(
|
|
group,
|
|
key=lambda g: -(float(g.get("x", 0)) + float(g.get("w", 0) or 0)),
|
|
)
|
|
return sorted(group, key=lambda g: float(g.get("x", 0)))
|
|
|
|
|
|
def _column_band(
|
|
g: dict,
|
|
gutters: list[tuple[float, float, float]],
|
|
*,
|
|
slack: float = 6.0,
|
|
at_y: float | None = None,
|
|
) -> tuple[int, int]:
|
|
"""Which column *g* sits in: whether gutters are active, and its column side index."""
|
|
if not gutters:
|
|
return (0, 0)
|
|
y = at_y if at_y is not None else float(g.get("y", 0.0))
|
|
active = tuple(sorted(gx for gx, lo, hi in gutters if lo - slack <= y <= hi + slack))
|
|
if not active:
|
|
return (0, 0)
|
|
x = float(g.get("x", 0.0))
|
|
return (
|
|
1,
|
|
sum(1 for gx in active if x >= gx + GUTTER_EDGE_CONTINUATION_PT),
|
|
)
|
|
|
|
|
|
def lines_from_glyphs(glyphs: list[dict]) -> list[Line]:
|
|
"""Cluster glyph dicts {text,x,y,w,h,fontSize?,fontName?} into reading lines."""
|
|
# Keep zero-width whitespace glyphs: they are real word-boundary markers
|
|
# in PDFs that encode spacing through text-matrix advances.
|
|
clean_glyphs = [g for g in glyphs if str(g.get("text", ""))]
|
|
if not clean_glyphs:
|
|
return []
|
|
sizes = [float(g.get("fontSize") or g.get("h") or 12.0) for g in clean_glyphs]
|
|
median = sorted(sizes)[len(sizes) // 2] if sizes else 12.0
|
|
# Generous tol so descenders (g,p,y) stay on the same line as the baseline
|
|
tol = max(median * 0.7, 4.0)
|
|
# 12pt is wider than the gutter of the most densely set two-column pages --
|
|
# the Federal Register CFR pages in the corpus leave only 9pt -- so those
|
|
# still weld their columns together. Lowering the floor to about one em does
|
|
# fix them, and the two tests in ``page_gutters`` are strong enough to keep
|
|
# an inter-word space from passing at that width. It is not landed because it
|
|
# is not yet *safe*: splitting the IRS 1040 address block into more lines
|
|
# makes something downstream of table reconstruction drop about fifteen
|
|
# words, both "Last name" labels among them, and losing a label from a tax
|
|
# form is worse than leaving a known welding case unfixed. Raise this only
|
|
# together with a fix for that.
|
|
gutters = page_gutters(
|
|
clean_glyphs, min_width=max(12.0, median * 1.4), row_height=max(median * 0.9, 4.0)
|
|
)
|
|
|
|
# Baselines, descending. Two costs used to hide here: the mean of every
|
|
# bucket was recomputed for every glyph, and every bucket was tried even
|
|
# though the input is sorted. Carrying a running total and stopping once a
|
|
# bucket's baseline is out of reach turns a quadratic scan — 1.6M
|
|
# arithmetic operations on four dense pages — into a linear one.
|
|
ordered = sorted(clean_glyphs, key=lambda g: -float(g.get("y", 0)))
|
|
buckets: list[list[dict]] = []
|
|
totals: list[float] = []
|
|
# One band per bucket, so a bucket's running baseline is never an average of
|
|
# two columns. See ``_column_band`` for what goes wrong without this.
|
|
bands: list[tuple[int, int]] = []
|
|
for g in ordered:
|
|
y = float(g.get("y", 0))
|
|
placed = False
|
|
for idx in range(len(buckets) - 1, -1, -1):
|
|
by = totals[idx] / len(buckets[idx])
|
|
if abs(y - by) <= tol:
|
|
band = _column_band(g, gutters, at_y=by)
|
|
if bands[idx] != band:
|
|
# Another column's row at this height. Keep looking for one
|
|
# of *this* column's rows; do not join and do not stop, or a
|
|
# glyph would be pulled into the wrong column.
|
|
continue
|
|
buckets[idx].append(g)
|
|
totals[idx] += y
|
|
placed = True
|
|
break
|
|
if by - y > tol:
|
|
# Buckets before this one sit higher still; none can match.
|
|
break
|
|
if not placed:
|
|
buckets.append([g])
|
|
totals.append(y)
|
|
bands.append(_column_band(g, gutters, at_y=y))
|
|
|
|
lines: list[Line] = []
|
|
for group in buckets:
|
|
for band in _split_group_at(group, gutters):
|
|
for subgroup in _split_group_on_gutter(band):
|
|
# Reading order within a line follows the script, not the page.
|
|
# Sorting a right-to-left line by ascending x yields *visual* order,
|
|
# which reverses the words: "مشروع منصة هويتي" comes out as
|
|
# "هويتي منصة مشروع". Ordering by descending x recovers logical
|
|
# order at the source, which is far more reliable than trying to
|
|
# detect and undo the reversal downstream.
|
|
rtl_line = _group_is_rtl(subgroup)
|
|
subgroup = _sort_group_by_reading_order(subgroup)
|
|
spans: list[TextSpan] = []
|
|
parts: list[str] = []
|
|
prev_x1: float | None = None
|
|
for g in subgroup:
|
|
span = _span_from_glyph(g, median)
|
|
if span is None:
|
|
continue
|
|
x0 = float(g.get("x", 0))
|
|
# Insert inter-word space when engine omits space glyphs.
|
|
# Conservative: avoid splitting glued tokens like Apple10 / LeftCol1A.
|
|
if (
|
|
prev_x1 is not None
|
|
and parts
|
|
and not parts[-1].endswith((" ", "\t"))
|
|
and not (span.text or "").startswith((" ", "\t"))
|
|
):
|
|
# In a right-to-left line the next glyph lies to the
|
|
# *left*, so the inter-glyph gap runs the other way.
|
|
gap = (prev_x1 - (x0 + float(g.get("w", 0) or 0))) if rtl_line else (x0 - prev_x1)
|
|
prev_ch = parts[-1][-1]
|
|
next_ch = (span.text or " ")[0]
|
|
glued_alnum = prev_ch.isalnum() and next_ch.isalnum()
|
|
# The gap is font-relative. Where the extractor reports
|
|
# the font's own space advance for this text state, use
|
|
# it: typeset text positions words with TJ offsets
|
|
# rather than space glyphs, and a fraction-of-median
|
|
# rule welds them together — "Trace-based Just-in-Time"
|
|
# arrived as one token at 9pt. Fall back to the median
|
|
# heuristic when no space width is reported.
|
|
# A reported space width is trusted only inside the
|
|
# band real fonts occupy — roughly 0.15em to 0.7em.
|
|
# Outside it the number is in the wrong unit (text
|
|
# space rather than page space) or the font lies, and
|
|
# believing it either welds words together or, worse,
|
|
# splits every letter of a word onto its own token.
|
|
size = float(g.get("fontSize") or g.get("h") or median or 12.0)
|
|
space_w = float(g.get("spaceWidth") or 0.0)
|
|
lo, hi = size * 0.15, size * 0.70
|
|
if lo <= space_w <= hi:
|
|
min_word_gap = space_w * 0.55
|
|
else:
|
|
min_word_gap = max(3.0, median * 0.40)
|
|
if glued_alnum:
|
|
min_word_gap = max(min_word_gap, median * 0.55)
|
|
if gap > min_word_gap:
|
|
parts.append(" ")
|
|
spans.append(
|
|
TextSpan(
|
|
text=" ",
|
|
font_size=span.font_size,
|
|
font_name=span.font_name,
|
|
x=prev_x1,
|
|
)
|
|
)
|
|
parts.append(span.text)
|
|
spans.append(span)
|
|
w = float(g.get("w", 0) or 0)
|
|
# Prefer reported width; floor so under-reported w does not inflate next gap
|
|
advance = w if w > 0.5 else max(len(span.text) * median * 0.45, median * 0.35)
|
|
# The "trailing edge" in reading order is the left edge for RTL.
|
|
prev_x1 = x0 if rtl_line else x0 + advance
|
|
text = "".join(parts).strip()
|
|
if not text:
|
|
continue
|
|
spans = coalesce_spans(spans)
|
|
xs = [float(g.get("x", 0)) for g in subgroup]
|
|
ws = [float(g.get("w", 0)) for g in subgroup]
|
|
ys = [float(g.get("y", 0)) for g in subgroup]
|
|
first = spans[0] if spans else None
|
|
lines.append(
|
|
Line(
|
|
y=sum(ys) / len(ys),
|
|
x0=min(xs),
|
|
x1=max(x + w for x, w in zip(xs, ws, strict=False)),
|
|
text=text,
|
|
font_size=float(first.font_size if first else median),
|
|
font_name=str(first.font_name if first else ""),
|
|
spans=spans,
|
|
)
|
|
)
|
|
return lines
|
|
|
|
|
|
def lines_from_plain_text(text: str, page_height: float = 792.0) -> list[Line]:
|
|
"""Fallback when only plain extract_text is available (no glyph X geometry).
|
|
|
|
The x/y values here are fabricated to keep downstream code total; they carry
|
|
no information about the page. Lines are flagged ``synthetic_geometry`` so
|
|
consumers can choose text-based heuristics over geometric ones.
|
|
"""
|
|
lines: list[Line] = []
|
|
y = page_height - 72
|
|
for raw in text.splitlines():
|
|
t = raw.strip()
|
|
if not t:
|
|
y -= 14
|
|
continue
|
|
lines.append(
|
|
Line(
|
|
y=y,
|
|
x0=72,
|
|
x1=72 + len(t) * 6,
|
|
text=t,
|
|
spans=[TextSpan(text=t)],
|
|
synthetic_geometry=True,
|
|
)
|
|
)
|
|
y -= 14
|
|
return lines
|