877 lines
36 KiB
Python
877 lines
36 KiB
Python
"""Header/footer band detection across pages."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
import threading
|
||
from difflib import SequenceMatcher
|
||
from functools import lru_cache
|
||
|
||
from app.services.convert.idm.model import BlockType, Document, Page, PageKind, TextSpan
|
||
|
||
# Punctuation, whitespace and digits are exactly what varies between repeats of
|
||
# the same running header, so they are removed before comparing.
|
||
_SIGNATURE_STRIP_RE = re.compile(r"[^a-z\u0600-\u06ff]+")
|
||
# Two signatures this similar are treated as the same banner.
|
||
SIGNATURE_SIMILARITY = 0.88
|
||
# Two texts sharing this share of their words are the same banner even when
|
||
# the word order differs. Set high enough that a body sentence quoting the
|
||
# agency name does not match the banner it mentions.
|
||
TOKEN_OVERLAP_SIMILARITY = 0.72
|
||
# Threshold after confusable folding. Higher than the raw one, because folding
|
||
# has already removed the differences that were only glyph shape — what is
|
||
# left must genuinely be the same line.
|
||
FOLDED_SIMILARITY = 0.86
|
||
# Containment only applies between texts this short — a running header is a
|
||
# line, not a sentence — so body prose quoting the agency name cannot match.
|
||
BANNER_MAX_WORDS = 12
|
||
# ...and only for a signature this long, so a common short phrase inside a
|
||
# longer banner is not treated as the banner itself.
|
||
MIN_CONTAINMENT_CHARS = 16
|
||
# Below this length a signature is too generic to identify a header.
|
||
MIN_SIGNATURE_CHARS = 4
|
||
# Members of a banner family kept as comparison probes for single linkage.
|
||
MAX_LINKAGE_PROBES = 12
|
||
|
||
|
||
def _norm(text: str) -> str:
|
||
return " ".join(text.lower().split())
|
||
|
||
|
||
@lru_cache(maxsize=8192)
|
||
def _signature(text: str) -> str:
|
||
"""Comparison key robust to OCR noise and per-page variation.
|
||
|
||
Running headers rarely repeat byte-for-byte across a scanned document:
|
||
OCR renders the same banner as "FEDERAL AUTHORITY FOR IDENTITY," on one
|
||
page and "FEDERALAUTHORITY FOR IDENTITY." on the next, and footers carry a
|
||
page number that changes by design. Folding away punctuation, spacing and
|
||
digits leaves the part that actually repeats.
|
||
"""
|
||
lowered = (text or "").lower()
|
||
return _SIGNATURE_STRIP_RE.sub("", lowered)
|
||
|
||
|
||
# Glyph shapes a recogniser confuses, folded to one representative before any
|
||
# comparison. These are shape collisions, not language facts, so the same table
|
||
# works for a Latin letterhead, a Cyrillic one or a serial number.
|
||
_CONFUSABLE_MAP = str.maketrans(
|
||
{
|
||
"0": "o", "1": "l", "5": "s", "8": "b", "6": "b", "9": "g", "2": "z",
|
||
"i": "l", "j": "l", "t": "l",
|
||
"u": "v", "n": "m", "r": "m",
|
||
"e": "o", "a": "o", "c": "o", "q": "o", "d": "o",
|
||
"f": "t", "y": "v", "w": "v",
|
||
# Arabic shape collisions the recogniser makes on the same printed line.
|
||
"ص": "س", "ض": "د", "ط": "ت", "ظ": "ز",
|
||
"ة": "ه", "ى": "ي", "أ": "ا", "إ": "ا", "آ": "ا", "ؤ": "و", "ئ": "ي",
|
||
}
|
||
)
|
||
|
||
|
||
@lru_cache(maxsize=8192)
|
||
def _fold_confusables(signature: str) -> str:
|
||
"""Collapse visually confusable glyphs so OCR drift compares as itself.
|
||
|
||
``Urted Areb Emlrates`` and ``United Arab Emirates`` are the same printed
|
||
line read twice. Character-for-character they are only 80% alike, which is
|
||
below any threshold safe enough to use on raw text — but the differences
|
||
are all shape collisions a recogniser makes, and folding those first lets
|
||
the pair be recognised without loosening the threshold for genuinely
|
||
different lines.
|
||
"""
|
||
return signature.translate(_CONFUSABLE_MAP)
|
||
|
||
|
||
# Vowels, and the glyphs OCR emits when it fails. A candidate spelling with
|
||
# more of the first and fewer of the second is the better reading.
|
||
_VOWELS = frozenset("aeiou")
|
||
_JUNK_GLYPHS = frozenset("0123456789|\\/~^`{}[]<>@#*_")
|
||
|
||
|
||
def _reading_quality(text: str) -> float:
|
||
"""How *undamaged* a string looks. A tiebreak, not a spell-checker.
|
||
|
||
This ranks a reading with digits and symbols stuck through it below a
|
||
reading made of letters — ``U1t3d Ar@b Em1r@t3s`` below ``United Arab
|
||
Emirates``. It deliberately does not try to choose between two plausible
|
||
letter strings: without a dictionary, ``Urted Areb Emlrates`` and the
|
||
correct spelling are equally word-shaped, and any character statistic
|
||
claiming otherwise is inventing a signal.
|
||
|
||
Choosing between plausible readings is the majority vote's job in
|
||
:func:`_canonical_variant`; OCR errors vary from page to page while the
|
||
correct reading repeats, so the mode is the real evidence. This only breaks
|
||
ties the vote leaves open.
|
||
"""
|
||
stripped = (text or "").strip()
|
||
if not stripped:
|
||
return 0.0
|
||
letters = [ch for ch in stripped.lower() if ch.isalpha()]
|
||
if not letters:
|
||
return 0.0
|
||
junk_share = sum(1 for ch in stripped if ch in _JUNK_GLYPHS) / len(stripped)
|
||
alpha_share = len(letters) / len(stripped)
|
||
# A run of letters with no vowel at all is a failed reading in every
|
||
# alphabet that has vowels; the check is skipped where there are none.
|
||
has_vowel = any(ch in _VOWELS for ch in letters)
|
||
vowel_floor = 1.0 if (has_vowel or not letters) else 0.0
|
||
return round(0.45 * alpha_share + 0.35 * (1.0 - junk_share) + 0.20 * vowel_floor, 4)
|
||
|
||
|
||
# One matcher per right-hand string, per thread. ``SequenceMatcher`` indexes
|
||
# its second sequence on construction, and banner matching compares hundreds of
|
||
# candidate lines against the same handful of variants — so that index was
|
||
# rebuilt for every comparison. The cache is thread-local because a matcher
|
||
# carries the current left-hand sequence as state and conversions run
|
||
# concurrently in a thread pool.
|
||
_MATCHERS = threading.local()
|
||
MAX_CACHED_MATCHERS = 512
|
||
|
||
|
||
def _matcher_for(b: str) -> SequenceMatcher:
|
||
cache = getattr(_MATCHERS, "by_seq2", None)
|
||
if cache is None:
|
||
cache = {}
|
||
_MATCHERS.by_seq2 = cache
|
||
matcher = cache.get(b)
|
||
if matcher is None:
|
||
if len(cache) >= MAX_CACHED_MATCHERS:
|
||
cache.clear()
|
||
matcher = SequenceMatcher(None, "", b)
|
||
cache[b] = matcher
|
||
return matcher
|
||
|
||
|
||
def _similar(a: str, b: str) -> float:
|
||
if not a or not b:
|
||
return 0.0
|
||
return SequenceMatcher(None, a, b).ratio()
|
||
|
||
|
||
def _similar_at_least(a: str, b: str, floor: float) -> bool:
|
||
"""``_similar(a, b) >= floor``, without paying for the full comparison.
|
||
|
||
Banner grouping compares every candidate line against every variant
|
||
collected so far, and ``SequenceMatcher.ratio`` is quadratic in the string
|
||
length. On a 126-page form that came to 67,000 comparisons and 83% of the
|
||
conversion's wall-clock.
|
||
|
||
``real_quick_ratio`` and ``quick_ratio`` are difflib's own *upper bounds*
|
||
on ``ratio`` — one from the lengths alone, one from the multiset of
|
||
characters — so rejecting on them cannot change any verdict. Most pairs
|
||
are nothing alike and die on the first, which costs two calls to ``len``.
|
||
"""
|
||
if not a or not b:
|
||
return False
|
||
matcher = _matcher_for(b)
|
||
matcher.set_seq1(a)
|
||
if matcher.real_quick_ratio() < floor:
|
||
return False
|
||
if matcher.quick_ratio() < floor:
|
||
return False
|
||
return matcher.ratio() >= floor
|
||
|
||
|
||
def _tokens(text: str) -> frozenset[str]:
|
||
"""Word set of a banner, folded the same way as its signature."""
|
||
folded = (_SIGNATURE_STRIP_RE.sub("", w) for w in (text or "").lower().split())
|
||
return frozenset(t for t in folded if len(t) >= 3)
|
||
|
||
|
||
def _token_overlap(a: str, b: str) -> float:
|
||
"""Order-independent similarity between two banner texts.
|
||
|
||
A bilingual running header comes back from OCR with its halves in
|
||
different orders on different pages — "FEDERAL AUTHORITY FOR IDENTITY,
|
||
CITIZENSHIP…" on one and "CITIZENSHIP, CUSTOMS & PORT SECURITY FEDERAL
|
||
AUTHORITY FOR IDENTITY," on the next. Those share every word and almost no
|
||
character *sequence*, so ``SequenceMatcher`` scores them far apart and the
|
||
same banner splits into variants that each look too rare to be furniture.
|
||
"""
|
||
ta, tb = _tokens(a), _tokens(b)
|
||
if not ta or not tb:
|
||
return 0.0
|
||
return len(ta & tb) / len(ta | tb)
|
||
|
||
|
||
# Short letterhead stamps (a country name, a three-word agency line) drift
|
||
# further than a long banner: almost every glyph can be a near-miss, so the
|
||
# long-line folded threshold would split each page into its own family.
|
||
SHORT_BANNER_WORDS = 5
|
||
SHORT_FOLDED_SIMILARITY = 0.64
|
||
# A stamp glued into a longer cell is recognised when a window of the cell
|
||
# matches the stamp this closely after folding.
|
||
STAMP_WINDOW_SIMILARITY = 0.78
|
||
|
||
|
||
_DIGITS_RE = re.compile(r"\d+")
|
||
|
||
# An identifier is a token this many letters long or more. Below it the token
|
||
# is a folio marker — "p", "no", "pg" — not content.
|
||
IDENTIFIER_LETTERS = 4
|
||
|
||
|
||
def _varies_by_an_identifier(a: str, b: str) -> bool:
|
||
"""Whether two lines differ only in digits that belong to an identifier.
|
||
|
||
A running header may legitimately change from page to page — "Page 1 of
|
||
126" — and the signature drops digits so those pages group together. But a
|
||
*per-page identifier* also survives that stripping: "ScanTokenpack1P000"
|
||
and "ScanTokenpack1P001" have identical signatures, so a scanned document
|
||
stamped with a document, case or invoice number had all its numbers read as
|
||
one running banner, hoisted into the Word header, and every page's but the
|
||
first deleted from the body.
|
||
|
||
The difference is where the digits sit. A folio is a bare number, or a
|
||
number after a short marker word. Digits embedded in a long alphanumeric
|
||
token are an identifier, and an identifier is content.
|
||
"""
|
||
if a == b:
|
||
return False
|
||
if _DIGITS_RE.sub("", a) != _DIGITS_RE.sub("", b):
|
||
return False
|
||
ta, tb = a.split(), b.split()
|
||
if len(ta) != len(tb):
|
||
return False
|
||
for left, right in zip(ta, tb, strict=False):
|
||
if left == right:
|
||
continue
|
||
letters = sum(1 for ch in left if ch.isalpha())
|
||
if letters >= IDENTIFIER_LETTERS:
|
||
return True
|
||
return False
|
||
|
||
|
||
def _same_banner(a: str, b: str, *, allow_containment: bool = True) -> bool:
|
||
"""Whether two texts are the same running banner.
|
||
|
||
Three tests, because OCR breaks the banner in three different ways across
|
||
a scanned document: it misspells it (sequence similarity), it reorders the
|
||
halves of a bilingual line (token overlap), and it splits the banner over
|
||
two blocks on one page while gluing its words together on another
|
||
(containment).
|
||
|
||
The containment test compares signatures, which have spacing stripped, so
|
||
``FEDERALAUTHORITY FOR IDENTITY,`` is recognised inside the full
|
||
``FEDERAL AUTHORITY FOR IDENTITY, CITIZENSHIP, CUSTOMS & PORT SECURITY``.
|
||
It is applied only between two banner-length texts, so a body sentence
|
||
that quotes the agency is never swallowed by it.
|
||
"""
|
||
if _varies_by_an_identifier(a, b):
|
||
return False
|
||
sig_a, sig_b = _signature(a), _signature(b)
|
||
if _similar_at_least(sig_a, sig_b, SIGNATURE_SIMILARITY):
|
||
return True
|
||
# Heavy drift: fold the glyph shapes a recogniser confuses and compare
|
||
# again. "Urted Areb Emlrates" is 0.80 against "United Arab Emirates" raw
|
||
# — below any threshold that is safe on unfolded text — and comfortably
|
||
# above it once o/0, l/1/i and the rest stop counting as differences.
|
||
fold_a, fold_b = _fold_confusables(sig_a), _fold_confusables(sig_b)
|
||
if _similar_at_least(fold_a, fold_b, FOLDED_SIMILARITY):
|
||
return True
|
||
wa, wb = len(a.split()), len(b.split())
|
||
if (
|
||
2 <= wa <= SHORT_BANNER_WORDS
|
||
and 2 <= wb <= SHORT_BANNER_WORDS
|
||
and abs(wa - wb) <= 1
|
||
and _similar_at_least(fold_a, fold_b, SHORT_FOLDED_SIMILARITY)
|
||
):
|
||
return True
|
||
if _token_overlap(a, b) >= TOKEN_OVERLAP_SIMILARITY:
|
||
return True
|
||
if not allow_containment:
|
||
# Containment is the one relation that legitimately joins *different*
|
||
# banner lines: a page that read the whole two-line banner as one block
|
||
# contains each line. Chaining families through it would merge the
|
||
# lines into a single family and lose one of them, so grouping asks for
|
||
# the strict relations only.
|
||
return False
|
||
if len(a.split()) > BANNER_MAX_WORDS or len(b.split()) > BANNER_MAX_WORDS:
|
||
return False
|
||
short, long_ = sorted((sig_a, sig_b), key=len)
|
||
if not (len(short) >= MIN_CONTAINMENT_CHARS and short in long_):
|
||
return False
|
||
# Containment is for a banner split vs glued, not for a journal title that
|
||
# sits inside a longer issue line ("Perspectives" inside "Number 4, Winter…").
|
||
if _containment_has_distinct_payload(a, b):
|
||
return False
|
||
return True
|
||
|
||
|
||
_ISSUE_LINE_RE = re.compile(
|
||
r"\b(?:vol(?:ume)?|no\.?|number|issue|pp\.?)\b|\d{2,}",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
|
||
def _containment_has_distinct_payload(a: str, b: str) -> bool:
|
||
"""True when the longer string carries a different line, not a fuller OCR."""
|
||
wa, wb = a.split(), b.split()
|
||
if abs(len(wa) - len(wb)) <= 2:
|
||
return False
|
||
short_w, long_w = (wa, wb) if len(wa) <= len(wb) else (wb, wa)
|
||
short_set = {w.lower() for w in short_w}
|
||
extra = [w for w in long_w if w.lower() not in short_set]
|
||
extra_joined = " ".join(extra)
|
||
return bool(_ISSUE_LINE_RE.search(extra_joined))
|
||
|
||
|
||
# A signature must appear on at least this share of pages to count as running
|
||
# furniture. Two pages is never enough on a long document, and requiring every
|
||
# page is too strict for scans where OCR loses the banner occasionally.
|
||
MIN_BAND_PAGE_SHARE = 0.25
|
||
# The band is never shallower than this many text lines, so a two- or
|
||
# three-line banner is not cut in half on a sparse page.
|
||
BAND_MIN_LINES = 3.0
|
||
|
||
|
||
def _page_bands(page, body_blocks: list, band_ratio: float) -> tuple[list, list]:
|
||
"""Top and bottom band blocks, in whatever coordinate system the page uses.
|
||
|
||
Bands are measured against the *observed* extent of the page's own blocks
|
||
rather than ``page.height``. OCR-rebuilt pages carry pixel-derived
|
||
coordinates (negative y, hundreds of units tall) that share no origin or
|
||
scale with the PDF point height, so comparing them against ``page.height``
|
||
classified every block on every scanned page as "bottom band" and never
|
||
found the banner at all.
|
||
"""
|
||
tops = [b.bbox.y + b.bbox.h for b in body_blocks]
|
||
bots = [b.bbox.y for b in body_blocks]
|
||
hi, lo = max(tops), min(bots)
|
||
span = hi - lo
|
||
if span <= 0:
|
||
# Degenerate geometry — fall back to reading order.
|
||
ordered = sorted(body_blocks, key=lambda b: b.reading_order)
|
||
return ordered[:1], ordered[-1:]
|
||
|
||
# A running header is a couple of lines, so the band is at least that deep
|
||
# however short the page is. Using the span fraction alone made the band
|
||
# narrower than the banner itself on sparse pages — a title page with six
|
||
# blocks — and caught only the banner's first line.
|
||
heights = sorted(b.bbox.h for b in body_blocks if b.bbox.h > 0)
|
||
line_h = heights[len(heights) // 2] if heights else 0.0
|
||
depth = max(span * band_ratio, line_h * BAND_MIN_LINES)
|
||
|
||
top_band = [b for b in body_blocks if b.bbox.y + b.bbox.h >= hi - depth]
|
||
bot_band = [b for b in body_blocks if b.bbox.y <= lo + depth]
|
||
return top_band, bot_band
|
||
|
||
|
||
def _repeating_band_variants(per_page: list[list[str]], page_count: int) -> list[str]:
|
||
"""Representative texts of every banner that repeats across enough pages.
|
||
|
||
Counted once per page, so a banner split into two blocks on one page does
|
||
not out-vote a banner that appears on twenty. The return value is a list of
|
||
*representatives*, not an exhaustive set of exact strings: callers compare
|
||
against them with :func:`_same_banner`, so a page whose spelling drifted
|
||
beyond anything already collected is still recognised.
|
||
"""
|
||
pages_with_text = [texts for texts in per_page if texts]
|
||
if len(pages_with_text) < 2:
|
||
return []
|
||
|
||
groups: list[dict] = []
|
||
# A running header usually repeats byte-for-byte. Resolving each distinct
|
||
# string once and remembering where it landed turns 126 pages of
|
||
# comparisons into one, which is most of the cost of this pass on a long
|
||
# document.
|
||
resolved: dict[str, dict] = {}
|
||
for page_i, texts in enumerate(pages_with_text):
|
||
for text in dict.fromkeys(texts):
|
||
if len(_signature(text)) < MIN_SIGNATURE_CHARS:
|
||
continue
|
||
known = resolved.get(text)
|
||
if known is not None:
|
||
known["pages"].add(page_i)
|
||
continue
|
||
# Single linkage: a variant joins a family if it matches *any*
|
||
# member, not just the first one seen. OCR drift forms a chain —
|
||
# a badly damaged reading can be far from the clean spelling while
|
||
# sitting close to a middling one — and comparing only against a
|
||
# fixed representative split one banner into several families, each
|
||
# too rare on its own to clear the page-share threshold.
|
||
group = next(
|
||
(
|
||
g
|
||
for g in groups
|
||
if any(
|
||
_same_banner(v, text, allow_containment=False)
|
||
for v in g["probes"]
|
||
)
|
||
or _same_banner(g["text"], text)
|
||
),
|
||
None,
|
||
)
|
||
if group is None:
|
||
group = {
|
||
"text": text,
|
||
"pages": {page_i},
|
||
"variants": [text],
|
||
"probes": [text],
|
||
}
|
||
groups.append(group)
|
||
else:
|
||
group["pages"].add(page_i)
|
||
group["variants"].append(text)
|
||
# Chaining needs several members to link through, but not all
|
||
# of them: past a handful the extra probes are near-duplicates
|
||
# of ones the family already holds and only cost comparisons.
|
||
if len(group["probes"]) < MAX_LINKAGE_PROBES:
|
||
group["probes"].append(text)
|
||
resolved[text] = group
|
||
|
||
threshold = max(2, round(len(pages_with_text) * MIN_BAND_PAGE_SHARE))
|
||
return [
|
||
_canonical_variant(g["variants"]) for g in groups if len(g["pages"]) >= threshold
|
||
]
|
||
|
||
|
||
def _canonical_variant(variants: list[str]) -> str:
|
||
"""The cleanest spelling of a banner that OCR read differently each page.
|
||
|
||
A recogniser reading the same printed line on twenty pages returns twenty
|
||
near-misses — a zero for an O, an l for an I, a stray full stop. Taking
|
||
whichever one happened to come first put page one's mistakes into the Word
|
||
header of the whole document. Language-like spellings beat letter-salad
|
||
even when the salad repeats; among equally clean readings the majority
|
||
(then the shorter running title) wins.
|
||
"""
|
||
if not variants:
|
||
return ""
|
||
counts: dict[str, int] = {}
|
||
for text in variants:
|
||
counts[text] = counts.get(text, 0) + 1
|
||
|
||
from app.services.convert.layout.text_quality import ocr_text_quality_score
|
||
|
||
return max(
|
||
counts,
|
||
key=lambda t: (
|
||
ocr_text_quality_score([t]),
|
||
counts[t],
|
||
-len(t.split()),
|
||
-len(t),
|
||
),
|
||
)
|
||
|
||
|
||
# A bare number, "Page 4", "4 of 12", "- 4 -". Page numbers differ on every
|
||
# page by definition, so they never group as a repeating banner and were left
|
||
# behind in the body as orphans, one stray digit per page.
|
||
_PAGE_NUMBER_RE = re.compile(
|
||
r"^(?:[-–—(\[]?\s*)"
|
||
r"(?:(?:page|pg|p\.?|seite|página|صفحة)\s*)?"
|
||
r"\d{1,4}"
|
||
r"(?:\s*(?:/|of|de|von|من)\s*\d{1,4})?"
|
||
r"(?:\s*[-–—)\]]?)$",
|
||
re.I,
|
||
)
|
||
# Longer than this and it is a sentence that happens to start with a number.
|
||
PAGE_NUMBER_MAX_CHARS = 24
|
||
|
||
|
||
def looks_like_page_number(text: str) -> bool:
|
||
"""Whether a line is a page-number stamp rather than content.
|
||
|
||
Deliberately shape-based: a short line that is essentially a number, with
|
||
an optional label and an optional "of N". The few label words listed are a
|
||
convenience, not the test — a bare "7" is recognised in any language.
|
||
"""
|
||
stripped = (text or "").strip()
|
||
if not stripped or len(stripped) > PAGE_NUMBER_MAX_CHARS:
|
||
return False
|
||
if not any(ch.isdigit() for ch in stripped):
|
||
return False
|
||
return bool(_PAGE_NUMBER_RE.match(stripped))
|
||
|
||
|
||
def tag_headers_footers(
|
||
document: Document,
|
||
*,
|
||
band_ratio: float = 0.12,
|
||
header_bboxes: list | None = None,
|
||
footer_bboxes: list | None = None,
|
||
) -> None:
|
||
"""
|
||
Detect repeating top/bottom text across ≥2 pages and retag those blocks
|
||
as header/footer. Prefer high/low bbox Y when available.
|
||
Optional ML header/footer bboxes boost single-page retag.
|
||
"""
|
||
# ML hints: retag blocks whose centers fall in hint boxes (works on 1-page docs too)
|
||
if header_bboxes or footer_bboxes:
|
||
for page in document.pages:
|
||
for b in page.blocks:
|
||
if not b.plain_text().strip() or b.type in (BlockType.table, BlockType.figure):
|
||
continue
|
||
cx = b.bbox.x + b.bbox.w / 2
|
||
cy = b.bbox.y + b.bbox.h / 2
|
||
for hb in header_bboxes or []:
|
||
if hb.x <= cx <= hb.x + hb.w and hb.y <= cy <= hb.y + hb.h:
|
||
b.type = BlockType.header
|
||
break
|
||
for fb in footer_bboxes or []:
|
||
if fb.x <= cx <= fb.x + fb.w and fb.y <= cy <= fb.y + fb.h:
|
||
if b.type != BlockType.header:
|
||
b.type = BlockType.footer
|
||
break
|
||
|
||
if document.page_count < 2:
|
||
return
|
||
|
||
top_texts: list[list[str]] = []
|
||
bottom_texts: list[list[str]] = []
|
||
bands: dict[int, tuple[list, list]] = {}
|
||
for page in document.pages:
|
||
body_blocks = [b for b in page.blocks if b.plain_text().strip()]
|
||
if not body_blocks:
|
||
continue
|
||
top_band, bot_band = _page_bands(page, body_blocks, band_ratio)
|
||
bands[page.index] = (top_band, bot_band)
|
||
# Every block in the band is a candidate, not just the outermost one.
|
||
# A running banner is a *region*: this document repeats an agency line
|
||
# and "United Arab Emirates" beneath it, and taking one block per page
|
||
# left the second line in the body on all 26 pages.
|
||
top_texts.append([_norm(b.plain_text()) for b in top_band])
|
||
bottom_texts.append([_norm(b.plain_text()) for b in bot_band])
|
||
|
||
header_variants = _repeating_band_variants(top_texts, document.page_count)
|
||
footer_variants = _repeating_band_variants(bottom_texts, document.page_count)
|
||
# A doc can repeat the same line top and bottom; prefer header.
|
||
footer_variants = [
|
||
f for f in footer_variants if not any(_same_banner(f, h) for h in header_variants)
|
||
]
|
||
|
||
numbered = False
|
||
for page in document.pages:
|
||
top_band, bot_band = bands.get(page.index, ([], []))
|
||
_retag_page(page, header_variants, footer_variants, top_band, bot_band)
|
||
numbered |= _tag_page_numbers(bot_band)
|
||
|
||
# Once a family is established, strip it wherever it appears — the cover
|
||
# page, a mid-document divider, a cell. A variant that shows up on only two
|
||
# pages never clears the page-share threshold on its own, but it is still
|
||
# the same letterhead, and leaving it in the body is how a country line
|
||
# ended up as the document's opening paragraph.
|
||
_retag_stragglers(document, header_variants, footer_variants)
|
||
furniture = header_variants + footer_variants
|
||
# Stamp trimming searches every cell and paragraph for a fuzzy substring of
|
||
# the banner. It exists because a recogniser reads the letterhead and the
|
||
# first clause as one line; a digital text layer never does that — the
|
||
# banner is its own text object at its own position, and retagging has
|
||
# already moved it. Running the search anyway cost 4s of the 8s spent on
|
||
# four pages of a tax table, comparing "1,234" against the running head
|
||
# thousands of times.
|
||
if _has_recognised_text(document):
|
||
_strip_banners_from_tables(document, furniture)
|
||
_strip_stamps_from_prose(document, furniture)
|
||
|
||
if numbered:
|
||
# The writer emits one PAGE field instead of the literal numbers, which
|
||
# is both correct in Word and the only way a per-page stamp can live in
|
||
# a single header definition.
|
||
meta = document.meta if document.meta is not None else {}
|
||
meta["page_number_footer"] = True
|
||
document.meta = meta
|
||
|
||
|
||
# A straggler is only retagged if it is short enough to be furniture. Body
|
||
# prose that quotes the organisation is longer than a banner line.
|
||
STRAGGLER_MAX_WORDS = 12
|
||
# ...and it must sit within this share of the page's own vertical extent,
|
||
# measured from whichever end the family belongs to.
|
||
STRAGGLER_BAND_SHARE = 0.30
|
||
|
||
|
||
def _retag_stragglers(
|
||
document: Document, header_variants: list[str], footer_variants: list[str]
|
||
) -> None:
|
||
"""Retag banner-family lines that fell outside a detected band.
|
||
|
||
The band vote answers "does this repeat enough to be furniture". Once it
|
||
has answered yes, every other appearance of the same family is furniture
|
||
too, wherever it sits: a cover page puts the letterhead lower than a body
|
||
page does, and a divider page puts it alone in the middle. Those copies
|
||
never joined the vote and so stayed in the body — the first paragraph of
|
||
the document being a misread country line is exactly this.
|
||
"""
|
||
if not header_variants and not footer_variants:
|
||
return
|
||
for page in document.pages:
|
||
candidates = [
|
||
b
|
||
for b in page.blocks
|
||
if b.plain_text().strip()
|
||
and b.type not in (BlockType.table, BlockType.figure)
|
||
and not b.image_png
|
||
]
|
||
if not candidates:
|
||
continue
|
||
tops = [b.bbox.y + b.bbox.h for b in candidates]
|
||
lo, hi = min(b.bbox.y for b in candidates), max(tops)
|
||
span = max(hi - lo, 1.0)
|
||
|
||
for block in candidates:
|
||
if block.type in (BlockType.header, BlockType.footer):
|
||
continue
|
||
text = block.plain_text().strip()
|
||
if len(text.split()) > STRAGGLER_MAX_WORDS:
|
||
continue
|
||
# Position decides as much as wording. A line that repeats the
|
||
# banner but sits in the middle of the page is a mention — the
|
||
# document referring to its own author — while the same words at
|
||
# the top of a cover page are the letterhead that the band vote
|
||
# simply did not see enough times to count.
|
||
near_top = (block.bbox.y + block.bbox.h - lo) / span >= 1.0 - STRAGGLER_BAND_SHARE
|
||
near_bottom = (block.bbox.y - lo) / span <= STRAGGLER_BAND_SHARE
|
||
# OCR pages often have unusable Y, so a letterhead line in the
|
||
# "middle" of the reconstructed page is still furniture.
|
||
scanish = page.kind in (PageKind.scan, PageKind.hybrid) or any(
|
||
getattr(b, "source", "") == "ocr" for b in page.blocks
|
||
)
|
||
if (scanish or near_top) and any(_same_banner(text, h) for h in header_variants):
|
||
block.type = BlockType.header
|
||
block.level = 0
|
||
elif near_bottom and any(_same_banner(text, f) for f in footer_variants):
|
||
block.type = BlockType.footer
|
||
block.level = 0
|
||
|
||
|
||
def _tag_page_numbers(bot_band: list) -> bool:
|
||
"""Move page-number stamps out of the body and into the footer."""
|
||
found = False
|
||
for block in bot_band or []:
|
||
if block.type in (BlockType.table, BlockType.figure, BlockType.header):
|
||
continue
|
||
if looks_like_page_number(block.plain_text()):
|
||
block.type = BlockType.footer
|
||
block.level = 0
|
||
found = True
|
||
return found
|
||
|
||
|
||
def _shared_character_budget(needle: str, haystack: str) -> int:
|
||
"""Size of the multiset intersection of two strings.
|
||
|
||
No window of *haystack* can match *needle* more closely than the share of
|
||
needle's characters that appear in haystack at all — a bound that costs one
|
||
pass over each string, against the hundreds of full comparisons the sliding
|
||
search would otherwise run.
|
||
"""
|
||
counts: dict[str, int] = {}
|
||
for ch in needle:
|
||
counts[ch] = counts.get(ch, 0) + 1
|
||
shared = 0
|
||
for ch in haystack:
|
||
left = counts.get(ch, 0)
|
||
if left:
|
||
counts[ch] = left - 1
|
||
shared += 1
|
||
return shared
|
||
|
||
|
||
@lru_cache(maxsize=4096)
|
||
def _folded_window_match(needle: str, haystack: str) -> bool:
|
||
"""Whether ``needle`` appears inside ``haystack`` after OCR-shape folding.
|
||
|
||
Used to find a short running stamp that was concatenated onto a real cell
|
||
("Capability Area" + a misspelt country line). Whole-string equality fails
|
||
because of the extra words; an un-folded substring fails because of the
|
||
misspelling.
|
||
|
||
The sliding search is the most expensive comparison in the package — six
|
||
window widths across every offset, each a fresh ``SequenceMatcher`` — so it
|
||
is entered only after a character-budget check rules out the overwhelming
|
||
majority of pairs, and the matcher's index over the needle is built once
|
||
and reused across every window.
|
||
"""
|
||
n = _fold_confusables(_signature(needle))
|
||
h = _fold_confusables(_signature(haystack))
|
||
if len(n) < 10 or not h:
|
||
return False
|
||
if n in h:
|
||
return True
|
||
if _shared_character_budget(n, h) < STAMP_WINDOW_SIMILARITY * len(n):
|
||
return False
|
||
lo = max(8, len(n) - 2)
|
||
hi = min(len(h), len(n) + 3)
|
||
matcher = SequenceMatcher(None, "", n)
|
||
for width in range(lo, hi + 1):
|
||
if width > len(h):
|
||
break
|
||
for i in range(0, len(h) - width + 1):
|
||
matcher.set_seq1(h[i : i + width])
|
||
if matcher.real_quick_ratio() < STAMP_WINDOW_SIMILARITY:
|
||
continue
|
||
if matcher.quick_ratio() < STAMP_WINDOW_SIMILARITY:
|
||
continue
|
||
if matcher.ratio() >= STAMP_WINDOW_SIMILARITY:
|
||
return True
|
||
return False
|
||
|
||
|
||
def _trim_stamp_from_text(text: str, variants: list[str]) -> str:
|
||
"""Drop a running stamp glued onto a cell or a body line; keep the rest."""
|
||
raw = (text or "").strip()
|
||
if not raw:
|
||
return raw
|
||
words = raw.split()
|
||
if len(words) <= SHORT_BANNER_WORDS and any(_same_banner(raw, v) for v in variants):
|
||
return ""
|
||
bullets = {"·", "•", "-", "—", "*", "|"}
|
||
for v in variants:
|
||
vn = len(v.split())
|
||
if vn < 2:
|
||
continue
|
||
for n in (vn, vn + 1, vn - 1):
|
||
if n < 2 or n > len(words):
|
||
continue
|
||
for i in range(0, len(words) - n + 1):
|
||
window = " ".join(words[i : i + n])
|
||
if not (_same_banner(window, v) or _folded_window_match(v, window)):
|
||
continue
|
||
at_edge = i == 0 or i + n == len(words)
|
||
next_to_mark = (i > 0 and words[i - 1] in bullets) or (
|
||
i + n < len(words) and words[i + n] in bullets
|
||
)
|
||
if not (at_edge or next_to_mark):
|
||
continue
|
||
kept = words[:i] + words[i + n :]
|
||
return " ".join(kept).strip()
|
||
return raw
|
||
|
||
|
||
def _has_recognised_text(document: Document) -> bool:
|
||
"""Whether any of this document's text came from OCR rather than the PDF."""
|
||
for page in document.pages:
|
||
if page.kind in (PageKind.scan, PageKind.hybrid):
|
||
return True
|
||
for block in page.blocks:
|
||
if getattr(block, "source", "") == "ocr":
|
||
return True
|
||
return False
|
||
|
||
|
||
def _strip_banners_from_tables(document: Document, variants: list[str]) -> None:
|
||
"""Remove running-banner rows that were swallowed into a grid.
|
||
|
||
On a scan the banner and the top of a table sit close together, and grid
|
||
assembly sometimes takes the banner in as a first row. Retagging skips
|
||
tables, so the banner then appeared in the Word header *and* as the table's
|
||
header row — the same text twice, one of them wrong.
|
||
"""
|
||
if not variants:
|
||
return
|
||
banner_only: list = []
|
||
for page in document.pages:
|
||
for block in page.blocks:
|
||
if block.type != BlockType.table or not block.cells:
|
||
continue
|
||
rows = []
|
||
for row in block.cells:
|
||
cleaned = [_trim_stamp_from_text(c or "", variants) for c in row]
|
||
joined = " ".join(c for c in cleaned if (c or "").strip()).strip()
|
||
if not joined:
|
||
continue
|
||
if any(_same_banner(joined, v) for v in variants):
|
||
continue
|
||
rows.append(cleaned)
|
||
if rows == block.cells:
|
||
continue
|
||
if rows:
|
||
block.cells = rows
|
||
continue
|
||
# Every row was banner. This is not a table with a banner in it —
|
||
# it is the banner, mistaken for a table, and emitting it as a
|
||
# one-row grid puts the letterhead in the document twice, once as
|
||
# furniture and once as a box around nothing.
|
||
block.cells = []
|
||
block.type = BlockType.paragraph
|
||
block.text = ""
|
||
block.spans = []
|
||
banner_only.append(block)
|
||
|
||
for page in document.pages:
|
||
page.blocks = [b for b in page.blocks if id(b) not in {id(x) for x in banner_only}]
|
||
|
||
|
||
def _strip_stamps_from_prose(document: Document, variants: list[str]) -> None:
|
||
"""Trim a running stamp that was concatenated onto a body paragraph.
|
||
|
||
Tables are handled separately. Requirement lines often carry the letterhead
|
||
in the same OCR line as the clause; retagging cannot move that line into
|
||
the header without losing the clause.
|
||
"""
|
||
if not variants:
|
||
return
|
||
for page in document.pages:
|
||
for block in page.blocks:
|
||
if block.type in (BlockType.table, BlockType.figure, BlockType.header, BlockType.footer):
|
||
continue
|
||
raw = block.plain_text().strip()
|
||
if not raw:
|
||
continue
|
||
cleaned = _trim_stamp_from_text(raw, variants)
|
||
if cleaned == raw:
|
||
continue
|
||
block.text = cleaned
|
||
if block.spans:
|
||
sample = block.spans[0]
|
||
block.spans = [
|
||
TextSpan(
|
||
text=cleaned,
|
||
font_name=sample.font_name,
|
||
font_size=sample.font_size,
|
||
bold=sample.bold,
|
||
italic=sample.italic,
|
||
)
|
||
]
|
||
|
||
|
||
def _retag_page(
|
||
page: Page,
|
||
header_variants: list[str],
|
||
footer_variants: list[str],
|
||
top_band: list | None = None,
|
||
bot_band: list | None = None,
|
||
) -> None:
|
||
"""Retag banner blocks inside the page's top/bottom band.
|
||
|
||
Matching within the band rather than walking a leading run matters for two
|
||
real cases in scanned documents:
|
||
|
||
* a masthead logo sits above the banner, and a run-walk stops dead at the
|
||
figure — leaving every banner line in the body of page one;
|
||
* on a dense page the banner is not the first block in reading order, so a
|
||
run starting at index 0 never reaches it.
|
||
|
||
The band already confines the search to the top or bottom of the page, and
|
||
only text matching a document-wide banner is retagged, so body prose that
|
||
merely mentions the agency keeps its place.
|
||
"""
|
||
if top_band is None or bot_band is None:
|
||
ordered = sorted(
|
||
[b for b in page.blocks if b.plain_text().strip()],
|
||
key=lambda b: b.reading_order,
|
||
)
|
||
top_band = top_band if top_band is not None else ordered[:1]
|
||
bot_band = bot_band if bot_band is not None else ordered[-1:]
|
||
|
||
skip = (BlockType.table, BlockType.figure)
|
||
tagged: set[int] = set()
|
||
|
||
for block in top_band:
|
||
if block.type in skip or not header_variants:
|
||
continue
|
||
if any(_same_banner(block.plain_text(), h) for h in header_variants):
|
||
block.type = BlockType.header
|
||
block.level = 0
|
||
tagged.add(id(block))
|
||
|
||
for block in bot_band:
|
||
if block.type in skip or id(block) in tagged or not footer_variants:
|
||
continue
|
||
if any(_same_banner(block.plain_text(), f) for f in footer_variants):
|
||
block.type = BlockType.footer
|
||
block.level = 0
|