1306 lines
50 KiB
Python
1306 lines
50 KiB
Python
"""Real page geometry from pypdf — glyph boxes and vector path operators.
|
|
|
|
The C++ engine supplies glyph geometry and a display list. When it is absent —
|
|
which is every Linux container today, and therefore production — the layout
|
|
pipeline fell back to :func:`lines_from_plain_text`, which fabricates ``x0=72``
|
|
for *every* line. Downstream that means: no columns, no indents, no cell
|
|
boundaries, no ruling lines, and a reading order taken from content-stream
|
|
order rather than from the page. Tables were structurally impossible to find.
|
|
|
|
This module recovers both signals from pypdf alone, so the engine-less path is
|
|
degraded in *fidelity* (no rasteriser, no shaped glyph boxes) rather than in
|
|
*structure*.
|
|
|
|
Two extractors:
|
|
|
|
``glyphs_from_page``
|
|
Text spans with real user-space positions, taken from the text matrix that
|
|
pypdf hands to its ``visitor_text`` callback. Advance widths are not
|
|
exposed by that API, so they are calibrated per document from the observed
|
|
spacing between adjacent spans on a baseline — self-correcting, and far
|
|
closer than a fixed 6pt-per-character guess.
|
|
|
|
``path_ops_from_page``
|
|
``re`` rectangles and ``m``/``l``/``c``/``v``/``y`` path segments, walked
|
|
out of the content stream with a real CTM stack and recursion into Form
|
|
XObjects. Emitted in the same dict shape the engine display list uses, so
|
|
``extract_rects_from_display_list`` and
|
|
``extract_vertical_rulings_from_display_list`` consume them unchanged.
|
|
|
|
Everything here is pure pypdf (BSD-3-Clause), already a pinned dependency. No
|
|
new licence surface.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import bisect
|
|
import contextlib
|
|
import math
|
|
import re
|
|
import threading
|
|
import unicodedata
|
|
from collections import OrderedDict
|
|
from itertools import pairwise
|
|
from typing import Any
|
|
|
|
from app.services.convert.layout.styles import infer_font_flags
|
|
|
|
# A page whose content stream is larger than this is not worth walking for
|
|
# path geometry: the parse cost outgrows the benefit and such streams are
|
|
# almost always image or shading data rather than table rules.
|
|
MAX_CONTENT_BYTES = 12 * 1024 * 1024
|
|
|
|
# Guard against pathological vector art (maps, CAD drawings) flooding the
|
|
# table detector with meaningless segments.
|
|
MAX_PATH_OPS = 20000
|
|
|
|
# Form XObject recursion depth. Real documents nest one or two deep.
|
|
MAX_FORM_DEPTH = 6
|
|
|
|
# Separator for scope-qualified font names when a form is spliced into the
|
|
# page stream. "@" cannot appear in a PDF name token, so it cannot collide.
|
|
_SCOPE_SEP = "@"
|
|
|
|
# Fraction of ``extract_text()`` characters the precise path must recover
|
|
# before it is trusted over the coarser visitor path.
|
|
MIN_PRECISE_COVERAGE = 0.80
|
|
|
|
# Fraction of a page's non-space characters that must be real, readable
|
|
# characters before the precise path's text is trusted. A font with no usable
|
|
# ``/ToUnicode`` CMap yields raw glyph *codes* — ``\x01\x02\x03`` where the page
|
|
# shows "Dum" — and those codes are exactly as *numerous* as the text they stand
|
|
# for, so MIN_PRECISE_COVERAGE alone cannot tell the two apart.
|
|
MIN_LEGIBLE_RATIO = 0.60
|
|
|
|
# Unicode categories that carry no legible text: control, format, unassigned,
|
|
# private-use and surrogate. Private-use is included deliberately — symbol
|
|
# fonts map into it, and while those glyphs render, they do not survive as text
|
|
# in a DOCX.
|
|
_ILLEGIBLE_CATEGORIES = frozenset({"Cc", "Cf", "Cn", "Co", "Cs"})
|
|
|
|
_IDENTITY = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
|
|
|
|
# Fallback advance width as a fraction of the font size, used until the
|
|
# document calibrates its own. 0.5em is the average for the Helvetica/Times
|
|
# families that dominate business PDFs.
|
|
DEFAULT_ADVANCE_RATIO = 0.5
|
|
|
|
# Built font sets, keyed by the identity of the font objects they came from.
|
|
# Bounded so a long-lived process converting many documents cannot grow it.
|
|
#
|
|
# Process-global, so concurrent conversions share it, so it needs a lock: the
|
|
# eviction was a check-then-``clear()``-then-write, and one conversion reaching
|
|
# the ceiling discarded font metrics another conversion was in the middle of
|
|
# using. Wrong advance widths, no exception. An ``OrderedDict`` under the lock
|
|
# also turns the all-or-nothing ``clear()`` into ordinary LRU eviction, so a
|
|
# busy process keeps the sets it is actually using.
|
|
MAX_CACHED_FONT_SETS = 256
|
|
_FONT_CACHE: OrderedDict[tuple, dict] = OrderedDict()
|
|
_FONT_CACHE_LOCK = threading.Lock()
|
|
|
|
|
|
def _num(value: Any, default: float = 0.0) -> float:
|
|
try:
|
|
return float(value)
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def _matrix(seq: Any) -> list[float]:
|
|
try:
|
|
m = [float(v) for v in seq]
|
|
except Exception:
|
|
return list(_IDENTITY)
|
|
return m if len(m) == 6 else list(_IDENTITY)
|
|
|
|
|
|
def _apply(m: list[float], x: float, y: float) -> tuple[float, float]:
|
|
"""Transform a point by a PDF matrix ``[a b c d e f]``."""
|
|
return (m[0] * x + m[2] * y + m[4], m[1] * x + m[3] * y + m[5])
|
|
|
|
|
|
def _mul(a: list[float], b: list[float]) -> list[float]:
|
|
"""Concatenate two PDF matrices — ``a`` then ``b``.
|
|
|
|
pypdf's own ``matrix_multiply`` takes nested 3x3 matrices; the six-element
|
|
form is what appears in content streams, so it gets its own product here.
|
|
"""
|
|
return [
|
|
a[0] * b[0] + a[1] * b[2],
|
|
a[0] * b[1] + a[1] * b[3],
|
|
a[2] * b[0] + a[3] * b[2],
|
|
a[2] * b[1] + a[3] * b[3],
|
|
a[4] * b[0] + a[5] * b[2] + b[4],
|
|
a[4] * b[1] + a[5] * b[3] + b[5],
|
|
]
|
|
|
|
|
|
def _scale_of(m: list[float]) -> float:
|
|
"""Uniform scale factor of a matrix, used to size text."""
|
|
sx = math.hypot(m[0], m[1])
|
|
sy = math.hypot(m[2], m[3])
|
|
if sx <= 0 and sy <= 0:
|
|
return 1.0
|
|
if sx <= 0:
|
|
return sy
|
|
if sy <= 0:
|
|
return sx
|
|
return (sx + sy) / 2.0
|
|
|
|
|
|
def page_box(page: Any) -> tuple[float, float, float, float]:
|
|
"""``(x0, y0, x1, y1)`` of the visible area — CropBox when it is set.
|
|
|
|
Producers routinely park job tickets, colour bars and revision stamps
|
|
outside the crop box. A renderer clips them; plain text extraction does
|
|
not, so that debris used to land in the converted document.
|
|
"""
|
|
box = None
|
|
try:
|
|
box = page.cropbox
|
|
except Exception:
|
|
box = None
|
|
if box is None:
|
|
box = page.mediabox
|
|
try:
|
|
x0, y0 = float(box.left), float(box.bottom)
|
|
x1, y1 = float(box.right), float(box.top)
|
|
except Exception:
|
|
return (0.0, 0.0, 612.0, 792.0)
|
|
if x1 < x0:
|
|
x0, x1 = x1, x0
|
|
if y1 < y0:
|
|
y0, y1 = y1, y0
|
|
return (x0, y0, x1, y1)
|
|
|
|
|
|
def page_rotation(page: Any) -> int:
|
|
try:
|
|
rot = int(page.get("/Rotate", 0) or 0)
|
|
except Exception:
|
|
rot = 0
|
|
return rot % 360
|
|
|
|
|
|
def page_dimensions(page: Any) -> tuple[float, float]:
|
|
"""Displayed page size, with ``/Rotate`` 90/270 swapping the axes.
|
|
|
|
The pipeline previously read ``mediabox.width``/``height`` directly, so a
|
|
landscape page stored rotated came through portrait and every downstream
|
|
ratio — column detection, header bands, image coverage — was computed
|
|
against the wrong box.
|
|
"""
|
|
x0, y0, x1, y1 = page_box(page)
|
|
w, h = x1 - x0, y1 - y0
|
|
if page_rotation(page) in (90, 270):
|
|
return (h, w)
|
|
return (w, h)
|
|
|
|
|
|
def _rotation_matrix(page: Any) -> list[float] | None:
|
|
"""Matrix mapping user space into displayed space for a rotated page."""
|
|
rot = page_rotation(page)
|
|
if rot == 0:
|
|
return None
|
|
x0, y0, x1, y1 = page_box(page)
|
|
w, h = x1 - x0, y1 - y0
|
|
if rot == 90:
|
|
# (x, y) -> (y, w - x) after translating the box to the origin
|
|
return [0.0, -1.0, 1.0, 0.0, -y0, x1]
|
|
if rot == 180:
|
|
return [-1.0, 0.0, 0.0, -1.0, x1, y1]
|
|
if rot == 270:
|
|
return [0.0, 1.0, -1.0, 0.0, y1, -x0]
|
|
_ = (w, h)
|
|
return None
|
|
|
|
|
|
def _content_operations(page: Any, reader: Any) -> list:
|
|
"""Parsed operator list for a page's content stream, or ``[]``."""
|
|
from pypdf.generic import ContentStream
|
|
|
|
try:
|
|
contents = page.get_contents()
|
|
except Exception:
|
|
return []
|
|
if contents is None:
|
|
return []
|
|
try:
|
|
raw = contents.get_data()
|
|
if raw is not None and len(raw) > MAX_CONTENT_BYTES:
|
|
return []
|
|
except Exception:
|
|
pass
|
|
try:
|
|
pdf = reader if reader is not None else getattr(page, "pdf", None)
|
|
return ContentStream(contents, pdf, "bytes").operations
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _resource_font_key(obj: Any) -> tuple | None:
|
|
"""Identity of a resource dictionary's font set, for caching.
|
|
|
|
Two pages that reference the same font objects need the same ``Font``
|
|
instances, and building one means parsing its ToUnicode CMap.
|
|
"""
|
|
try:
|
|
resources = obj.get("/Resources")
|
|
if resources is None:
|
|
return None
|
|
table = resources.get_object().get("/Font")
|
|
if table is None:
|
|
return None
|
|
raw = table.raw_get if hasattr(table, "raw_get") else None
|
|
table = table.get_object()
|
|
key = []
|
|
for name in sorted(table.keys()):
|
|
ref = raw(name) if raw is not None else table[name]
|
|
idnum = getattr(ref, "idnum", None)
|
|
key.append((str(name), idnum if idnum is not None else id(table[name])))
|
|
return tuple(key)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _page_fonts(page: Any) -> dict:
|
|
"""``_layout_mode_fonts`` for a page, built once per distinct font set.
|
|
|
|
pypdf rebuilds every font's character map on each call, and a document
|
|
whose pages share one resource dictionary paid that cost per page — 4.9 of
|
|
23 seconds on a 20-page journal article, parsing the same CMaps twenty
|
|
times. ``Font`` is immutable once built, so the instances are shared.
|
|
"""
|
|
key = _resource_font_key(page)
|
|
if key is not None:
|
|
with _FONT_CACHE_LOCK:
|
|
cached = _FONT_CACHE.get(key)
|
|
if cached is not None:
|
|
_FONT_CACHE.move_to_end(key)
|
|
return cached
|
|
fonts: dict = {}
|
|
with contextlib.suppress(Exception):
|
|
fonts.update(page._layout_mode_fonts())
|
|
if key is not None:
|
|
with _FONT_CACHE_LOCK:
|
|
_FONT_CACHE[key] = fonts
|
|
_FONT_CACHE.move_to_end(key)
|
|
while len(_FONT_CACHE) > MAX_CACHED_FONT_SETS:
|
|
_FONT_CACHE.popitem(last=False)
|
|
return fonts
|
|
|
|
|
|
def _register_form_fonts(form: Any, prefix: str, fonts: dict) -> None:
|
|
"""Add a Form XObject's fonts to *fonts* under scope-qualified names.
|
|
|
|
A form carries its own ``/Resources``, so ``/F1`` inside it need not be the
|
|
``/F1`` of the page. Qualifying the name keeps the two apart after the
|
|
form's operators are spliced into the page stream.
|
|
"""
|
|
from pypdf._cmap import build_char_map_from_dict
|
|
from pypdf._text_extraction._layout_mode._font import Font
|
|
from pypdf.generic import ArrayObject, IndirectObject
|
|
|
|
try:
|
|
resources = form.get("/Resources")
|
|
if resources is None:
|
|
return
|
|
table = resources.get_object().get("/Font")
|
|
if table is None:
|
|
return
|
|
table = table.get_object()
|
|
except Exception:
|
|
return
|
|
|
|
for name in list(table.keys()):
|
|
key = f"{name}{_SCOPE_SEP}{prefix}"
|
|
if key in fonts:
|
|
continue
|
|
try:
|
|
ft = table[name].get_object()
|
|
cmap = build_char_map_from_dict(200.0, ft)
|
|
font_dict = {
|
|
k: (
|
|
v.get_object()
|
|
if isinstance(v, IndirectObject)
|
|
else [item.get_object() for item in v]
|
|
if isinstance(v, ArrayObject)
|
|
else v
|
|
)
|
|
for k, v in ft.items()
|
|
}
|
|
fonts[key] = Font(*cmap, font_dict)
|
|
except Exception:
|
|
continue
|
|
|
|
|
|
def _flatten_page(page: Any, reader: Any) -> tuple[list, dict]:
|
|
"""``(operations, fonts)`` for a page with Form XObjects spliced inline.
|
|
|
|
Real-world PDFs — every IRS form in the corpus, for one — draw most of
|
|
their content inside a Form XObject and leave the page stream almost
|
|
empty. pypdf's layout machinery does not follow ``Do``, so reading the
|
|
page stream alone recovered 13% of the text on ``i1040gi.pdf``. Splicing
|
|
each form's operators in, wrapped in ``q``/``cm``/``Q`` exactly as a
|
|
renderer would, makes the form's text and its ruling lines visible to both
|
|
extractors at their true page positions.
|
|
"""
|
|
from pypdf.generic import FloatObject, NameObject
|
|
|
|
fonts: dict = dict(_page_fonts(page))
|
|
|
|
out: list = []
|
|
scope = [0]
|
|
|
|
def walk(operations: list, resources: Any, prefix: str, depth: int, seen: frozenset) -> None:
|
|
# Page-level `cm` operators are rewritten into a single accumulated
|
|
# transform held inside one synthetic `q`. pypdf's text state manager
|
|
# keeps the CTM as a chain of maps and recomputes the effective matrix
|
|
# by multiplying the whole chain for *every* text-showing operation; a
|
|
# `cm` outside any `q` is never popped, so on a page with 5,000 of them
|
|
# the chain reached 4,000 entries and one page cost 2.5 million matrix
|
|
# multiplications. Collapsing them keeps the chain two deep and leaves
|
|
# the effective transform identical at every point in the stream.
|
|
q_depth = 0
|
|
bt_depth = 0
|
|
total = list(_IDENTITY)
|
|
collapsed = False
|
|
|
|
for operands, op in operations:
|
|
if op == b"q":
|
|
q_depth += 1
|
|
elif op == b"Q":
|
|
q_depth = max(0, q_depth - 1)
|
|
elif op == b"BT":
|
|
bt_depth += 1
|
|
elif op == b"ET":
|
|
bt_depth = max(0, bt_depth - 1)
|
|
elif (
|
|
op == b"cm"
|
|
and q_depth == 0
|
|
and bt_depth == 0
|
|
and len(operands) == 6
|
|
):
|
|
total = _mul(_matrix(operands), total)
|
|
if collapsed:
|
|
out.append(([], b"Q"))
|
|
out.append(([], b"q"))
|
|
out.append(([FloatObject(v) for v in total], b"cm"))
|
|
collapsed = True
|
|
continue
|
|
|
|
if op == b"Tf" and prefix and operands:
|
|
scoped = NameObject(f"{operands[0]}{_SCOPE_SEP}{prefix}")
|
|
if scoped in fonts:
|
|
out.append(([scoped, *list(operands[1:])], op))
|
|
continue
|
|
out.append((operands, op))
|
|
elif op == b"Do" and depth < MAX_FORM_DEPTH and operands:
|
|
if not _inline_form(operands[0], resources, prefix, depth, seen, walk):
|
|
out.append((operands, op))
|
|
else:
|
|
out.append((operands, op))
|
|
|
|
if collapsed:
|
|
out.append(([], b"Q"))
|
|
|
|
def _inline_form(
|
|
name: Any, resources: Any, prefix: str, depth: int, seen: frozenset, recurse
|
|
) -> bool:
|
|
try:
|
|
table = resources.get("/XObject") if resources else None
|
|
if table is None:
|
|
return False
|
|
form = table.get_object().get(name)
|
|
if form is None:
|
|
return False
|
|
ref = form.idnum if hasattr(form, "idnum") else id(form)
|
|
if ref in seen:
|
|
return False
|
|
form = form.get_object()
|
|
if str(form.get("/Subtype")) != "/Form":
|
|
return False
|
|
scope[0] += 1
|
|
child = f"{scope[0]}"
|
|
_register_form_fonts(form, child, fonts)
|
|
matrix = _matrix(form.get("/Matrix", _IDENTITY))
|
|
inner = _content_operations_of(form, reader)
|
|
if not inner:
|
|
return False
|
|
out.append(([], b"q"))
|
|
out.append(([FloatObject(v) for v in matrix], b"cm"))
|
|
recurse(inner, form.get("/Resources"), child, depth + 1, seen | {ref})
|
|
out.append(([], b"Q"))
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
walk(_content_operations(page, reader), page.get("/Resources"), "", 0, frozenset())
|
|
return out, fonts
|
|
|
|
|
|
def _content_operations_of(obj: Any, reader: Any) -> list:
|
|
from pypdf.generic import ContentStream
|
|
|
|
try:
|
|
return ContentStream(obj, reader, "bytes").operations
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Text spans
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def _strip_subset_tag(name: str) -> str:
|
|
"""``/ABCDEF+Helvetica`` -> ``Helvetica``.
|
|
|
|
Subset prefixes are random per-file noise and would defeat style grouping:
|
|
the same typeface embedded twice gets two different tags, so two runs that
|
|
should share a style would land in different style buckets.
|
|
"""
|
|
if "+" in name[:8]:
|
|
name = name.split("+", 1)[1]
|
|
return name.lstrip("/")
|
|
|
|
|
|
def _font_name(font_dict: Any) -> str:
|
|
if not font_dict:
|
|
return ""
|
|
for key in ("/BaseFont", "/Name"):
|
|
try:
|
|
value = font_dict.get(key)
|
|
except Exception:
|
|
value = None
|
|
if value:
|
|
return _strip_subset_tag(str(value))
|
|
return ""
|
|
|
|
|
|
# ``/FontWeight`` is a number in 100..900 per PDF 32000-1 table 122, but pypdf
|
|
# also fills ``FontDescriptor.weight`` from a TrueType subfamily name, and
|
|
# defaults it to the string "Unknown". So it arrives as 700, "Bold", "Medium"
|
|
# or "Unknown" depending on the producer, and a bare ``weight >= 600`` raises
|
|
# TypeError on perfectly ordinary files.
|
|
_BOLD_WEIGHT_NAMES = ("bold", "black", "heavy", "ultra", "extrab", "semib", "demi")
|
|
|
|
|
|
def _weight_is_bold(value: Any) -> bool:
|
|
"""Whether a ``FontDescriptor.weight`` positively declares bold.
|
|
|
|
Only ever used to *add* boldness, never to remove it — plenty of producers
|
|
embed a bold face with the weight left at its 400 default, and the name
|
|
("...-Bold") is then the only true signal.
|
|
"""
|
|
if value is None or isinstance(value, bool):
|
|
return False
|
|
if isinstance(value, (int, float)):
|
|
return float(value) >= 600.0
|
|
text = str(value).strip().lower()
|
|
if not text or text == "unknown":
|
|
return False
|
|
try:
|
|
return float(text) >= 600.0
|
|
except ValueError:
|
|
pass
|
|
return any(tag in text for tag in _BOLD_WEIGHT_NAMES)
|
|
|
|
|
|
def _font_identity(font: Any) -> tuple[str, bool, bool]:
|
|
"""``(name, bold, italic)`` for a pypdf ``Font`` from the content stream.
|
|
|
|
Sources, in order of trust:
|
|
|
|
1. ``font_dictionary`` — the raw ``/Font`` dict. Present on pypdf 5.x.
|
|
On pypdf 6.x the attribute does not exist at all: ``Font`` became a
|
|
dataclass of typed fields. Reading only that made every glyph on the
|
|
precise path anonymous (``fontName=''``, ``bold=False``), so headings
|
|
and bold runs flattened into body text and adjacent paragraphs merged
|
|
into one. The failure was silent in the worst way — the coarse visitor
|
|
path *is* handed a real dict by pypdf, so it kept reporting fonts
|
|
correctly and only the high-fidelity path lost them.
|
|
2. ``font.name`` — the BaseFont, subset tag and all.
|
|
3. ``font_descriptor.name`` / ``.family`` — last resort for a font whose
|
|
BaseFont is missing.
|
|
|
|
``/FontDescriptor`` also carries the producer's own ``/FontWeight`` and
|
|
``/ItalicAngle``, which beat sniffing the name: a face called plain
|
|
"Calibri" with ``weight=700`` really is bold, and ``infer_font_flags``
|
|
would have missed it. Both are applied additively, so a descriptor that
|
|
under-reports can never undo a name that says "-Bold".
|
|
"""
|
|
name = _font_name(getattr(font, "font_dictionary", None))
|
|
if not name:
|
|
raw = getattr(font, "name", None)
|
|
if isinstance(raw, str) and raw.strip():
|
|
name = _strip_subset_tag(raw.strip())
|
|
desc = getattr(font, "font_descriptor", None)
|
|
if not name and desc is not None:
|
|
for attr in ("name", "family"):
|
|
raw = getattr(desc, attr, None)
|
|
if isinstance(raw, str) and raw.strip() and raw.strip().lower() != "unknown":
|
|
name = _strip_subset_tag(raw.strip())
|
|
break
|
|
|
|
bold, italic = infer_font_flags(name)
|
|
if desc is not None:
|
|
if _weight_is_bold(getattr(desc, "weight", None)):
|
|
bold = True
|
|
try:
|
|
angle = float(getattr(desc, "italic_angle", 0) or 0)
|
|
except (TypeError, ValueError):
|
|
angle = 0.0
|
|
try:
|
|
flags = int(getattr(desc, "flags", 0) or 0)
|
|
except (TypeError, ValueError):
|
|
flags = 0
|
|
# Bit 7 (value 64) is the Italic flag, PDF 32000-1 table 123.
|
|
if abs(angle) > 0.5 or flags & 64:
|
|
italic = True
|
|
return name, bold, italic
|
|
|
|
|
|
def _decode(value: Any, font: Any) -> str:
|
|
"""Bytes from a show operator to text, through the font's encoding + cmap."""
|
|
if not isinstance(value, bytes):
|
|
return str(value)
|
|
encoding = getattr(font, "encoding", None)
|
|
try:
|
|
if isinstance(encoding, str):
|
|
text = value.decode(encoding, "surrogatepass")
|
|
elif isinstance(encoding, dict):
|
|
text = "".join(
|
|
encoding[b] if b in encoding else bytes((b,)).decode() for b in value
|
|
)
|
|
else:
|
|
text = value.decode("utf-8", "replace")
|
|
except (UnicodeEncodeError, UnicodeDecodeError):
|
|
text = value.decode("utf-8", "replace")
|
|
char_map = getattr(font, "char_map", None) or {}
|
|
return "".join(char_map.get(ch, ch) for ch in text)
|
|
|
|
|
|
# Sentinel for "this operator sets a colour I decline to interpret". Distinct
|
|
# from None, which means black.
|
|
_KEEP = object()
|
|
# Colour-space names whose single component is a grey level rather than a tint.
|
|
_GRAY_CS_RE = re.compile(r"(devicegray|calgray|^/?g$)", re.I)
|
|
|
|
|
|
def _hex_from_rgb(r: float, g: float, b: float) -> str | None:
|
|
"""``RRGGBB`` for a non-black colour, or ``None`` for black and near-black.
|
|
|
|
``None`` rather than ``"000000"`` on purpose. Word's default run colour is
|
|
"automatic", which is what a reader wants for body text: it flips to white
|
|
when the document is printed in a dark theme, and it is what an author who
|
|
never set a colour would see. Writing an explicit black onto every run would
|
|
both freeze that behaviour and put a ``<w:color>`` element on every span in
|
|
the document. The threshold is not exactly zero because a producer that means
|
|
black often writes ``0.0039`` (1/255) or a CMYK rich black that converts a
|
|
shade off.
|
|
"""
|
|
vals = []
|
|
for v in (r, g, b):
|
|
v = 0.0 if v < 0.0 else (1.0 if v > 1.0 else v)
|
|
vals.append(int(round(v * 255.0)))
|
|
if max(vals) <= 8:
|
|
return None
|
|
return "".join(f"{v:02X}" for v in vals)
|
|
|
|
|
|
def _fill_from_operands(op: bytes, operands: list, cs_name: str) -> str | None | object:
|
|
"""Fill colour for a colour-setting operator, in the PDF's own model.
|
|
|
|
Three return values are needed, not two. ``None`` is a real colour — black —
|
|
and ``_KEEP`` means "this operator carries a colour this function declines to
|
|
interpret, so leave the current one alone". Collapsing the last case into
|
|
``None`` would silently repaint text black, which is worse than not reading
|
|
the colour at all.
|
|
|
|
Only fill operators are read. Text is filled by default (``Tr 0``); the
|
|
stroke colour set by ``RG``/``K``/``G`` paints outlines, and a document using
|
|
``Tr 1`` to draw text as outlines only is rare enough that guessing from the
|
|
stroke colour would be wrong more often than right.
|
|
|
|
``sc``/``scn`` is where care is needed: its operands are components in
|
|
whatever space ``cs`` selected, and the count is the only clue available
|
|
without resolving the resource dictionary. Three or four components are
|
|
unambiguously RGB or CMYK. A *single* component is not: in DeviceGray 1.0 is
|
|
white, while in a Separation space 1.0 is full colorant -- usually the
|
|
darkest value. Reading a tint as a grey would invert it, so one component is
|
|
only accepted when ``cs`` named a grey space outright.
|
|
"""
|
|
try:
|
|
nums = [_num(v) for v in operands]
|
|
except (TypeError, ValueError):
|
|
return _KEEP
|
|
if op == b"g" and len(nums) == 1:
|
|
return _hex_from_rgb(nums[0], nums[0], nums[0])
|
|
if op == b"rg" and len(nums) == 3:
|
|
return _hex_from_rgb(*nums)
|
|
if op == b"k" and len(nums) == 4:
|
|
c, m, y, kk = nums
|
|
return _hex_from_rgb(
|
|
(1.0 - c) * (1.0 - kk), (1.0 - m) * (1.0 - kk), (1.0 - y) * (1.0 - kk)
|
|
)
|
|
if op in (b"sc", b"scn"):
|
|
if len(nums) == 3:
|
|
return _hex_from_rgb(*nums)
|
|
if len(nums) == 4:
|
|
c, m, y, kk = nums
|
|
return _hex_from_rgb(
|
|
(1.0 - c) * (1.0 - kk), (1.0 - m) * (1.0 - kk), (1.0 - y) * (1.0 - kk)
|
|
)
|
|
if len(nums) == 1 and _GRAY_CS_RE.search(cs_name):
|
|
return _hex_from_rgb(nums[0], nums[0], nums[0])
|
|
return _KEEP
|
|
|
|
|
|
def _text_state_ops(page: Any, reader: Any, ops: list | None, fonts: dict | None) -> list[Any]:
|
|
"""Per-show-operation text state for a page: exact x, y, width and font.
|
|
|
|
A direct pass over the flattened operator list, maintaining the text
|
|
matrix, the text line matrix and the CTM stack exactly as §9.4.4 of the
|
|
PDF specification describes, and handing each shown string to pypdf's
|
|
``TextStateParams`` — which owns the font metrics, the displacement
|
|
arithmetic and the rotation handling that are genuinely worth reusing.
|
|
|
|
Driving pypdf's own ``TextStateManager`` instead was the obvious route and
|
|
is what this did first. That class models the current transform as a chain
|
|
of matrices and recomputes the effective one by multiplying the *whole*
|
|
chain for every string shown; a ``TJ`` array of n elements pushes n
|
|
matrices, so the cost is quadratic in the array's length. On one page of
|
|
the IRS tax tables the chain reached 4,000 entries and the page cost 2.5
|
|
million matrix multiplications. Concatenating the matrix directly is O(1)
|
|
per element and gives the same answer.
|
|
|
|
Advancing after *every* shown string, rather than only when a kerning
|
|
number follows, also fixes the placement of two adjacent strings inside one
|
|
``TJ`` array — the second used to be reported at the first one's position.
|
|
"""
|
|
from pypdf._text_extraction._layout_mode._text_state_params import TextStateParams
|
|
|
|
if fonts is None or ops is None:
|
|
ops, fonts = _flatten_page(page, reader)
|
|
|
|
collected: list[Any] = []
|
|
ctm = list(_IDENTITY)
|
|
# ``q``/``Q`` save and restore the whole graphics state, not just the
|
|
# transform, so fill colour and colour space ride on the stack with the CTM.
|
|
# Keeping colour outside it would leak a heading's colour into the body text
|
|
# that follows, because a producer sets the colour inside the q/Q pair that
|
|
# draws the heading and relies on the restore to undo it.
|
|
stack: list[tuple[list[float], str | None, str]] = []
|
|
tm = list(_IDENTITY)
|
|
tlm = list(_IDENTITY)
|
|
font = None
|
|
size = 0.0
|
|
tc = tw = ts = 0.0
|
|
tz = 100.0
|
|
tl = 0.0
|
|
fill: str | None = None
|
|
cs_name = ""
|
|
|
|
def show(raw: Any) -> None:
|
|
nonlocal tm
|
|
if font is None:
|
|
return
|
|
text = _decode(raw, font)
|
|
if not text:
|
|
return
|
|
params = TextStateParams(text, font, size, tc, tw, tz, tl, ts, _mul(tm, ctm))
|
|
# TextStateParams is pypdf's, and has no colour concept; it is a plain
|
|
# mutable dataclass with no __slots__, so the fill travels as an extra
|
|
# attribute rather than forcing a parallel list that could fall out of
|
|
# step with ``collected``.
|
|
params.dq_fill = fill
|
|
collected.append(params)
|
|
tm = _mul([1.0, 0.0, 0.0, 1.0, params.word_tx(text), 0.0], tm)
|
|
|
|
def next_line(tx: float, ty: float) -> None:
|
|
nonlocal tm, tlm
|
|
tlm = _mul([1.0, 0.0, 0.0, 1.0, tx, ty], tlm)
|
|
tm = list(tlm)
|
|
|
|
for operands, operator in ops:
|
|
op = operator
|
|
try:
|
|
if op == b"q":
|
|
stack.append((list(ctm), fill, cs_name))
|
|
elif op == b"Q":
|
|
if stack:
|
|
ctm, fill, cs_name = stack.pop()
|
|
ctm = list(ctm)
|
|
else:
|
|
ctm = list(_IDENTITY)
|
|
elif op in (b"g", b"rg", b"k", b"sc", b"scn"):
|
|
got = _fill_from_operands(op, list(operands), cs_name)
|
|
if got is not _KEEP:
|
|
fill = got # type: ignore[assignment]
|
|
elif op == b"cs":
|
|
cs_name = str(operands[0]) if operands else ""
|
|
# A colour space change resets the colour to that space's
|
|
# initial value, which is black in every device space.
|
|
fill = None
|
|
elif op == b"cm":
|
|
if len(operands) == 6:
|
|
ctm = _mul(_matrix(operands), ctm)
|
|
elif op == b"BT":
|
|
tm = list(_IDENTITY)
|
|
tlm = list(_IDENTITY)
|
|
elif op == b"Tf":
|
|
candidate = fonts.get(operands[0]) if operands else None
|
|
if candidate is not None:
|
|
font = candidate
|
|
size = _num(operands[1], 0.0)
|
|
elif op == b"Td":
|
|
next_line(_num(operands[0]), _num(operands[1]))
|
|
elif op == b"TD":
|
|
tl = -_num(operands[1])
|
|
next_line(_num(operands[0]), _num(operands[1]))
|
|
elif op == b"Tm":
|
|
if len(operands) == 6:
|
|
tlm = _matrix(operands)
|
|
tm = list(tlm)
|
|
elif op == b"T*":
|
|
next_line(0.0, -tl)
|
|
elif op == b"Tj":
|
|
show(operands[0])
|
|
elif op == b"TJ":
|
|
th = tz / 100.0
|
|
for item in operands[0]:
|
|
if isinstance(item, bytes):
|
|
show(item)
|
|
else:
|
|
shift = -_num(item) / 1000.0 * size * th
|
|
tm = _mul([1.0, 0.0, 0.0, 1.0, shift, 0.0], tm)
|
|
elif op == b"'":
|
|
next_line(0.0, -tl)
|
|
show(operands[0])
|
|
elif op == b'"':
|
|
tw = _num(operands[0])
|
|
tc = _num(operands[1])
|
|
next_line(0.0, -tl)
|
|
show(operands[2])
|
|
elif op == b"TL":
|
|
tl = _num(operands[0])
|
|
elif op == b"Tc":
|
|
tc = _num(operands[0])
|
|
elif op == b"Tw":
|
|
tw = _num(operands[0])
|
|
elif op == b"Tz":
|
|
tz = _num(operands[0], 100.0) or 100.0
|
|
elif op == b"Ts":
|
|
ts = _num(operands[0])
|
|
except (IndexError, TypeError, ValueError, AttributeError):
|
|
continue
|
|
|
|
return collected
|
|
|
|
|
|
def _decode_with_font(text: str, font: Any) -> str:
|
|
"""Map raw glyph codes through the font's own ``/ToUnicode`` CMap.
|
|
|
|
``TextStateParams.text`` is *not* decoded: it carries the bytes from the
|
|
content stream. For a standard-encoded font those bytes already are the
|
|
text, which is why this was easy to miss. For an embedded subset font they
|
|
are arbitrary code points:
|
|
|
|
* ``WBSBDL+TimesNewRomanPSMT`` maps ``$`` to ``A``, ``%`` to ``B``, ... —
|
|
a uniform 29-place offset, so "sediment" reaches the DOCX as
|
|
``VHGLPHQW``: printable, plausible-looking, and completely wrong.
|
|
* ``AAAAAA+ArialMT`` maps ``\\x01``..``\\x05`` to Arabic letters, so an
|
|
entire word arrives as control characters and vanishes.
|
|
|
|
pypdf resolves both on ``Font.character_map``, keyed by exactly these raw
|
|
codes. Applying it to every character is safe rather than merely expedient:
|
|
across this corpus every well-behaved font's map is either empty (the
|
|
standard 14, nothing to map) or the identity over ASCII, so the substitution
|
|
is a no-op precisely where the text is already correct. A map that is *not*
|
|
the identity is proof the raw bytes were never the text.
|
|
|
|
Characters absent from the map are passed through untouched, so a partial
|
|
CMap degrades to today's behaviour instead of dropping content.
|
|
"""
|
|
cmap = getattr(font, "character_map", None)
|
|
if not isinstance(cmap, dict) or not cmap:
|
|
return text
|
|
out: list[str] = []
|
|
changed = False
|
|
for ch in text:
|
|
mapped = cmap.get(ch)
|
|
if isinstance(mapped, str) and mapped and mapped != ch:
|
|
out.append(mapped)
|
|
changed = True
|
|
else:
|
|
out.append(ch)
|
|
return "".join(out) if changed else text
|
|
|
|
|
|
def _tj_text(tj: Any) -> str:
|
|
"""Decoded text of a ``TextStateParams``, across pypdf versions.
|
|
|
|
pypdf renamed this field from ``txt`` to ``text``. Reading only ``txt``
|
|
makes every show operation look empty on newer pypdf, so
|
|
``_glyphs_from_text_state`` returns nothing, ``glyphs_from_page`` silently
|
|
falls back to the coarse visitor, and every width becomes an estimate --
|
|
which is how adjacent table cells get welded into fabricated numbers. The
|
|
failure is invisible: no exception, and the quality score stays high.
|
|
|
|
``value`` is the constructor's first positional argument, which
|
|
``_text_state_ops`` already passes as decoded text, so it is a safe last
|
|
resort.
|
|
"""
|
|
for attr in ("txt", "text", "value"):
|
|
candidate = getattr(tj, attr, None)
|
|
if isinstance(candidate, str) and candidate:
|
|
return candidate
|
|
return ""
|
|
|
|
|
|
def _glyphs_from_text_state(
|
|
page: Any, reader: Any, ops: list | None, fonts: dict | None, rot: list[float] | None
|
|
) -> tuple[list[dict], int]:
|
|
"""``(glyphs, chars_seen)`` — the second value counts text *before* the
|
|
crop-box filter, so a caller measuring coverage is not penalised for the
|
|
off-page job tickets this extractor deliberately drops."""
|
|
try:
|
|
tjs = _text_state_ops(page, reader, ops, fonts)
|
|
except Exception:
|
|
return [], 0
|
|
if not tjs:
|
|
return [], 0
|
|
|
|
x0, y0, x1, y1 = page_box(page)
|
|
bleed = 6.0
|
|
seen = 0
|
|
glyphs: list[dict] = []
|
|
for tj in tjs:
|
|
# Whitespace-only operations are kept: at this granularity a single
|
|
# show operation is often a fragment of a word, and dropping the spans
|
|
# that carry the spaces welds "This is an example" into "Thisisan".
|
|
text = _decode_with_font(_tj_text(tj), getattr(tj, "font", None))
|
|
if not text:
|
|
continue
|
|
seen += len(text)
|
|
try:
|
|
tx = float(tj.tx)
|
|
ty = float(tj.ty)
|
|
dtx = float(tj.displaced_tx)
|
|
size = abs(float(tj.font_height)) or 12.0
|
|
except Exception:
|
|
continue
|
|
if not (x0 - bleed <= tx <= x1 + bleed and y0 - bleed <= ty <= y1 + bleed):
|
|
continue
|
|
width = abs(dtx - tx)
|
|
# What a space would advance in this exact text state — font, size,
|
|
# Tc, Tw and Tz all folded in. Producers of typeset text position
|
|
# words with TJ offsets rather than space glyphs, so this is the only
|
|
# reliable threshold for "is that gap a word break?".
|
|
#
|
|
# ``space_tx`` is computed from the *Tf* size, in unscaled text space,
|
|
# while every coordinate here is in page space. A producer that writes
|
|
# ``/F1 1 Tf`` and scales through the text matrix — Word's PDF export
|
|
# does exactly this — reports a space eleven times too small, and then
|
|
# a 0.13pt kern between two letters reads as a word break: "Low Vision"
|
|
# arrives as "L ow Vis io n". Scale it the same way the size was.
|
|
try:
|
|
space_w = abs(float(tj.space_tx))
|
|
tf_size = abs(float(getattr(tj, "font_size", 0.0) or 0.0))
|
|
if tf_size > 1e-6 and size > 0.0:
|
|
space_w *= size / tf_size
|
|
except Exception:
|
|
space_w = 0.0
|
|
name, bold, italic = _font_identity(getattr(tj, "font", None))
|
|
if rot is not None:
|
|
tx, ty = _apply(rot, tx, ty)
|
|
if not (0.5 < size < 400.0):
|
|
size = 12.0
|
|
glyphs.append(
|
|
{
|
|
"text": text,
|
|
"x": tx,
|
|
"y": ty,
|
|
"w": width,
|
|
"h": size,
|
|
"fontSize": size,
|
|
"fontName": name,
|
|
"spaceWidth": space_w,
|
|
"bold": bold,
|
|
"italic": italic,
|
|
"color": getattr(tj, "dq_fill", None),
|
|
}
|
|
)
|
|
return glyphs, seen
|
|
|
|
|
|
def _calibrate_advance_ratio(raw: list[dict]) -> float:
|
|
"""Advance width per character, as a fraction of font size.
|
|
|
|
Derived from the gaps between consecutive spans that share a baseline: if
|
|
a 10pt span of 8 characters is followed 44pt to its right by another span,
|
|
the font advances about 0.55em per character. Taking the median over the
|
|
page rejects the outliers produced by tab-like jumps between columns.
|
|
"""
|
|
by_line: dict[tuple[int, int], list[dict]] = {}
|
|
for span in raw:
|
|
size = span["size"] or 1.0
|
|
key = (int(round(span["y"] / max(size * 0.6, 1.0))), int(round(size)))
|
|
by_line.setdefault(key, []).append(span)
|
|
|
|
ratios: list[float] = []
|
|
for spans in by_line.values():
|
|
if len(spans) < 2:
|
|
continue
|
|
spans.sort(key=lambda s: s["x"])
|
|
for cur, nxt in pairwise(spans):
|
|
n = len(cur["text"])
|
|
size = cur["size"]
|
|
if n < 3 or size <= 0:
|
|
continue
|
|
advance = nxt["x"] - cur["x"]
|
|
if advance <= 0:
|
|
continue
|
|
ratio = advance / (n * size)
|
|
# Below 0.2em/char the spans overlap; above 1.1em/char there is a
|
|
# gap between them rather than a continuous run.
|
|
if 0.2 <= ratio <= 1.1:
|
|
ratios.append(ratio)
|
|
|
|
if len(ratios) < 4:
|
|
return DEFAULT_ADVANCE_RATIO
|
|
ratios.sort()
|
|
return ratios[len(ratios) // 2]
|
|
|
|
|
|
def _legible_ratio(glyphs: list[dict]) -> float:
|
|
"""Fraction of a glyph run's non-space characters that are readable text.
|
|
|
|
``1.0`` for ordinary pages. Near ``0.0`` when a font's ``/ToUnicode`` CMap is
|
|
missing or unusable and the walker has handed back raw glyph codes, which
|
|
look like text to every length-based check but render as ``\\x01\\x02`` in a
|
|
DOCX. Whitespace is ignored rather than counted either way: it is legitimate
|
|
in both cases and would otherwise dilute the signal on sparse pages.
|
|
|
|
Returns ``1.0`` for an empty run so callers never reject on no evidence.
|
|
"""
|
|
legible = counted = 0
|
|
for glyph in glyphs:
|
|
for ch in glyph.get("text") or "":
|
|
if ch.isspace():
|
|
continue
|
|
counted += 1
|
|
if unicodedata.category(ch) not in _ILLEGIBLE_CATEGORIES:
|
|
legible += 1
|
|
return 1.0 if not counted else legible / counted
|
|
|
|
|
|
def glyphs_from_page(
|
|
page: Any,
|
|
reader: Any = None,
|
|
ops: list | None = None,
|
|
fonts: dict | None = None,
|
|
*,
|
|
plain_chars: int = 0,
|
|
) -> list[dict]:
|
|
"""Text spans as ``lines_from_glyphs`` glyph dicts, in displayed space.
|
|
|
|
Returns ``[]`` when the page carries no extractable text layer, which lets
|
|
the caller fall through to OCR or to plain text exactly as before.
|
|
|
|
``plain_chars`` — the number of **non-whitespace** characters
|
|
``extract_text()`` found on the same page, which the pipeline has already
|
|
computed — is used as a coverage check: if the precise path recovers
|
|
materially less text than plain extraction did, the page uses something
|
|
this walker does not model and the coarser visitor path is the safer
|
|
answer. Structure is worth having; text is worth more.
|
|
|
|
Non-whitespace on both sides of that comparison is the whole point, and it
|
|
used to be the raw ``len()``. ``seen`` counts the characters this walker
|
|
rendered, and the walker emits one span per text-showing operator with no
|
|
inter-word spaces at all -- ``'Figur'``, ``'e'``, ``'12.'``. ``extract_text()``
|
|
meanwhile *inserts* the spaces and newlines it infers from glyph positions.
|
|
So the ratio was measuring pypdf's injected whitespace, not coverage, and it
|
|
lands near the threshold for ordinary prose: across the 14 pages of the
|
|
Mozilla PDF-spec excerpt it ran 0.754 to 0.858 against a bar of 0.80, so
|
|
four pages fell through to the visitor for no reason but their space count.
|
|
Stripping whitespace from both sides, the same walker covers 0.879 to 1.000
|
|
of those pages -- exactly 1.000 on nine of the fourteen -- and the four that
|
|
were being rejected score 0.942, 0.948, 0.888 and 0.879.
|
|
|
|
That mattered because the visitor does not measure widths, it *estimates*
|
|
them (see ``_glyphs_from_visitor``), and on a two-column page the estimate
|
|
runs long enough to cover the gutter, so the two columns weld together and
|
|
interleave. The visitor also duplicates text on precisely those four pages,
|
|
reaching 1.058 to 1.121 of the non-space characters that actually exist.
|
|
"""
|
|
rot = _rotation_matrix(page)
|
|
precise, seen = _glyphs_from_text_state(page, reader, ops, fonts, rot)
|
|
if precise and (plain_chars <= 0 or seen >= plain_chars * MIN_PRECISE_COVERAGE):
|
|
if _legible_ratio(precise) >= MIN_LEGIBLE_RATIO:
|
|
return precise
|
|
# Right geometry, unusable text. pypdf's own visitor applies decoding
|
|
# this walker does not, so let it try before giving up on the page.
|
|
coarse = _glyphs_from_visitor(page, rot)
|
|
if not precise:
|
|
return coarse
|
|
# Prefer whichever path produced *readable* text; only then the longer one.
|
|
# Without this, a page of undecodable codes beats real words on raw length.
|
|
if _legible_ratio(coarse) > _legible_ratio(precise):
|
|
return coarse
|
|
if sum(len(g["text"]) for g in coarse) > sum(len(g["text"]) for g in precise):
|
|
return coarse
|
|
return precise
|
|
|
|
|
|
def _glyphs_from_visitor(page: Any, rot: list[float] | None) -> list[dict]:
|
|
"""Coarser fallback: one span per positioning operator, widths estimated.
|
|
|
|
Used when the precise per-operation path is unavailable — an encrypted or
|
|
malformed content stream, a font pypdf cannot interpret, or a pypdf whose
|
|
internals have moved.
|
|
"""
|
|
raw: list[dict] = []
|
|
x0, y0, x1, y1 = page_box(page)
|
|
# Producers place trim marks a few points outside the box; a small bleed
|
|
# keeps legitimate edge content without readmitting off-page job tickets.
|
|
bleed = 6.0
|
|
|
|
def visitor(text: Any, cm: Any, tm: Any, font_dict: Any, font_size: Any) -> None:
|
|
s = str(text or "")
|
|
if not s.strip():
|
|
return
|
|
# pypdf flushes accumulated text at positioning operators and marks an
|
|
# internal line break with "\n". Only the first line sits at the
|
|
# reported matrix, so the rest is joined with a space rather than
|
|
# given a position it does not have. Under 0.5% of spans in practice.
|
|
s = " ".join(part for part in s.split("\n") if part.strip())
|
|
user = _mul(_matrix(tm), _matrix(cm))
|
|
x, y = user[4], user[5]
|
|
if not (x0 - bleed <= x <= x1 + bleed and y0 - bleed <= y <= y1 + bleed):
|
|
return
|
|
size = abs(_num(font_size, 12.0)) * _scale_of(user)
|
|
if not (0.5 < size < 400.0):
|
|
size = 12.0
|
|
if rot is not None:
|
|
x, y = _apply(rot, x, y)
|
|
name = _font_name(font_dict)
|
|
bold, italic = infer_font_flags(name)
|
|
raw.append(
|
|
{
|
|
"text": s,
|
|
"x": x,
|
|
"y": y,
|
|
"size": size,
|
|
"fontName": name,
|
|
"bold": bold,
|
|
"italic": italic,
|
|
}
|
|
)
|
|
|
|
try:
|
|
page.extract_text(visitor_text=visitor)
|
|
except Exception:
|
|
return []
|
|
|
|
if not raw:
|
|
return []
|
|
|
|
ratio = _calibrate_advance_ratio(raw)
|
|
|
|
glyphs: list[dict] = []
|
|
for span in raw:
|
|
size = span["size"]
|
|
width = len(span["text"]) * size * ratio
|
|
glyphs.append(
|
|
{
|
|
"text": span["text"],
|
|
"x": span["x"],
|
|
"y": span["y"],
|
|
"w": width,
|
|
"h": size,
|
|
"fontSize": size,
|
|
"fontName": span["fontName"],
|
|
"bold": span["bold"],
|
|
"italic": span["italic"],
|
|
}
|
|
)
|
|
|
|
# A span may not overrun the one that follows it on the same baseline:
|
|
# the calibrated ratio is a document average, and a wide-tracked heading
|
|
# would otherwise swallow its neighbour and destroy the column gap that
|
|
# table detection reads.
|
|
#
|
|
# "Same baseline" needs a tolerance. Keying rows by ``int(round(y))``
|
|
# compares baselines for exact equality, and the columns of a real document
|
|
# are almost never set to the same grid: on page 13 of the TraceMonkey paper
|
|
# the left column's rows sit at y=188.0 and the right column's at y=183.3.
|
|
# Different keys, so no clipping happened, so the left column's span kept the
|
|
# width this function *estimated* for it -- ``len(text) * size * ratio``, a
|
|
# monospace guess that ran to 611pt for a 60-character line and covered the
|
|
# whole page. Every line then straddled the gutter, ``page_gutters`` could
|
|
# not find a column gap that nothing left blank, and the two columns welded
|
|
# together and interleaved. That is the visible corruption in the output:
|
|
# "...industrial sponsor Sun" and "Microsystems under Project No. 07-127."
|
|
# arrived spliced through the middle of a left-column sentence.
|
|
#
|
|
# Only the width is a guess here -- ``x`` comes from the text matrix and is
|
|
# exact -- so the neighbouring span's ``x`` is the one hard bound available,
|
|
# and it is worth using wherever two spans share a row visually rather than
|
|
# numerically. Half the font size is comfortably tighter than the leading, so
|
|
# it cannot pull in a span from the row above or below.
|
|
order = sorted(glyphs, key=lambda g: g["y"])
|
|
ys = [g["y"] for g in order]
|
|
for cur in glyphs:
|
|
tol = max(float(cur.get("h") or 0.0) * 0.6, 2.0)
|
|
lo = bisect.bisect_left(ys, cur["y"] - tol)
|
|
hi = bisect.bisect_right(ys, cur["y"] + tol)
|
|
room = min(
|
|
(
|
|
other["x"] - cur["x"]
|
|
for other in order[lo:hi]
|
|
if other is not cur and other["x"] > cur["x"]
|
|
),
|
|
default=None,
|
|
)
|
|
if room is not None and room > 0 and cur["w"] > room:
|
|
cur["w"] = room
|
|
return glyphs
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Vector paths
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def _emit_rect(out: list[dict], m: list[float], x: float, y: float, w: float, h: float) -> None:
|
|
corners = [
|
|
_apply(m, x, y),
|
|
_apply(m, x + w, y),
|
|
_apply(m, x, y + h),
|
|
_apply(m, x + w, y + h),
|
|
]
|
|
xs = [c[0] for c in corners]
|
|
ys = [c[1] for c in corners]
|
|
rx, ry = min(xs), min(ys)
|
|
rw, rh = max(xs) - rx, max(ys) - ry
|
|
if rw < 0.05 and rh < 0.05:
|
|
return
|
|
out.append({"type": "rect", "x": rx, "y": ry, "w": rw, "h": rh})
|
|
# A table rule is almost always drawn as a filled rectangle a fraction of
|
|
# a point thick, not as a stroked line. Publishing it as a segment too is
|
|
# what lets ruling detection see the grid at all.
|
|
if rw <= 2.5 or rh <= 2.5:
|
|
if rw <= rh:
|
|
cx = rx + rw / 2.0
|
|
out.append({"type": "line", "x0": cx, "y0": ry, "x1": cx, "y1": ry + rh})
|
|
else:
|
|
cy = ry + rh / 2.0
|
|
out.append({"type": "line", "x0": rx, "y0": cy, "x1": rx + rw, "y1": cy})
|
|
|
|
|
|
def _emit_segments(out: list[dict], m: list[float], subpath: list[tuple[float, float]]) -> None:
|
|
for (ax, ay), (bx, by) in pairwise(subpath):
|
|
p0 = _apply(m, ax, ay)
|
|
p1 = _apply(m, bx, by)
|
|
if abs(p1[0] - p0[0]) < 0.05 and abs(p1[1] - p0[1]) < 0.05:
|
|
continue
|
|
out.append({"type": "line", "x0": p0[0], "y0": p0[1], "x1": p1[0], "y1": p1[1]})
|
|
|
|
|
|
def _walk_content(
|
|
operations: list,
|
|
reader: Any,
|
|
resources: Any,
|
|
ctm: list[float],
|
|
out: list[dict],
|
|
depth: int,
|
|
) -> None:
|
|
stack: list[list[float]] = []
|
|
m = list(ctm)
|
|
cur: list[tuple[float, float]] = []
|
|
start: tuple[float, float] | None = None
|
|
|
|
for operands, operator in operations:
|
|
if len(out) >= MAX_PATH_OPS:
|
|
return
|
|
op = operator.decode("latin-1") if isinstance(operator, bytes) else str(operator)
|
|
|
|
if op == "q":
|
|
stack.append(list(m))
|
|
elif op == "Q":
|
|
m = stack.pop() if stack else list(ctm)
|
|
elif op == "cm":
|
|
if len(operands) == 6:
|
|
m = _mul(_matrix(operands), m)
|
|
elif op == "re":
|
|
if len(operands) == 4:
|
|
_emit_rect(
|
|
out,
|
|
m,
|
|
_num(operands[0]),
|
|
_num(operands[1]),
|
|
_num(operands[2]),
|
|
_num(operands[3]),
|
|
)
|
|
elif op == "m":
|
|
if len(operands) == 2:
|
|
if len(cur) > 1:
|
|
_emit_segments(out, m, cur)
|
|
start = (_num(operands[0]), _num(operands[1]))
|
|
cur = [start]
|
|
elif op == "l":
|
|
if len(operands) == 2 and cur:
|
|
cur.append((_num(operands[0]), _num(operands[1])))
|
|
elif op in ("c", "v", "y"):
|
|
# Curves matter only as connectivity here; the endpoint is enough
|
|
# to keep a subpath's straight neighbours joined up.
|
|
if operands and cur:
|
|
cur.append((_num(operands[-2]), _num(operands[-1])))
|
|
elif op == "h":
|
|
if start and cur:
|
|
cur.append(start)
|
|
elif op in ("S", "s", "f", "F", "f*", "B", "B*", "b", "b*", "n"):
|
|
if op in ("s", "b", "b*") and start and cur:
|
|
cur.append(start)
|
|
if len(cur) > 1:
|
|
_emit_segments(out, m, cur)
|
|
cur = []
|
|
start = None
|
|
if len(cur) > 1:
|
|
_emit_segments(out, m, cur)
|
|
|
|
|
|
def path_ops_from_page(page: Any, reader: Any = None, ops: list | None = None) -> list[dict]:
|
|
"""Display-list-shaped path operators for one page, in displayed space."""
|
|
if ops is None:
|
|
ops, _fonts = _flatten_page(page, reader)
|
|
if not ops:
|
|
return []
|
|
|
|
base = _rotation_matrix(page) or list(_IDENTITY)
|
|
out: list[dict] = []
|
|
try:
|
|
_walk_content(
|
|
ops,
|
|
reader if reader is not None else getattr(page, "pdf", None),
|
|
page.get("/Resources"),
|
|
base,
|
|
out,
|
|
0,
|
|
)
|
|
except Exception:
|
|
return out
|
|
return out
|
|
|
|
|
|
def page_geometry(
|
|
page: Any, reader: Any = None, *, plain_chars: int = 0
|
|
) -> tuple[list[dict], list[dict]]:
|
|
"""``(glyphs, path_ops)`` for one page, parsing the content stream once."""
|
|
ops, fonts = _flatten_page(page, reader)
|
|
glyphs = glyphs_from_page(page, reader, ops, fonts, plain_chars=plain_chars)
|
|
paths = path_ops_from_page(page, reader, ops)
|
|
return glyphs, paths
|