109 lines
3.7 KiB
Python
109 lines
3.7 KiB
Python
"""MD/HTML/TXT postprocess — hyphenation, TOC leaders, page numbers, drop-caps.
|
|
|
|
Every pass here deletes or rewrites extracted content, so each one is scoped as
|
|
narrowly as the signal allows. A pass that removes a running page number must
|
|
not also remove a standalone figure in a financial table, and a pass that
|
|
merges a drop cap must not merge the "A." of a lettered list.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
_HYPHEN_BREAK_RE = re.compile(r"(\w)[\-‐‑\x02]\n+(\w)")
|
|
_DOT_LEADER_RE = re.compile(r"(\S)\s*\.{2,}\s*(\S)")
|
|
# A page-number line: an optional "page" word, digits, optional dash decoration.
|
|
# Roman numerals are included because front matter uses them.
|
|
_PAGE_NUM_LINE_RE = re.compile(
|
|
r"^\s*(?:page\s+)?\d{1,4}\s*$"
|
|
r"|^\s*[-–—]\s*\d{1,4}\s*[-–—]\s*$"
|
|
r"|^\s*(?:page\s+)?\d{1,4}\s*(?:of|/)\s*\d{1,4}\s*$"
|
|
r"|^\s*[ivxlcdm]{1,7}\s*$",
|
|
re.I,
|
|
)
|
|
# A drop cap is a lone capital followed by lowercase continuation text. Require
|
|
# the continuation to look like prose so "A." / "B." list markers are untouched.
|
|
_DROP_CAP_RE = re.compile(r"(?m)^([A-ZÀ-ÖØ-Þ])\n+([a-zà-öø-ÿ][a-zà-öø-ÿ ,;]{8,})")
|
|
|
|
# Do not treat a lone number as a page number when it carries a decimal point,
|
|
# a thousands separator, a sign or a currency symbol — those are data.
|
|
_DATA_NUMBER_RE = re.compile(r"[.,%$€£¥₹+\-]")
|
|
|
|
|
|
def rejoin_hyphenated_breaks(text: str) -> str:
|
|
"""Join end-of-line hyphenation: ``end-\\nword`` → ``endword``."""
|
|
if not text:
|
|
return text
|
|
prev = None
|
|
out = text
|
|
while prev != out:
|
|
prev = out
|
|
out = _HYPHEN_BREAK_RE.sub(r"\1\2", out)
|
|
return out
|
|
|
|
|
|
def collapse_dot_leaders(text: str) -> str:
|
|
"""Collapse TOC-style dot leaders to ``' ... '``."""
|
|
if not text:
|
|
return text
|
|
return _DOT_LEADER_RE.sub(r"\1 ... \2", text)
|
|
|
|
|
|
def _looks_like_page_number(line: str) -> bool:
|
|
stripped = line.strip()
|
|
if not _PAGE_NUM_LINE_RE.match(line):
|
|
return False
|
|
# "1,234" or "12.5" or "-40" are values, not page numbers.
|
|
return not _DATA_NUMBER_RE.search(stripped)
|
|
|
|
|
|
def filter_page_number_lines(text: str, *, min_repeats: int = 2) -> str:
|
|
"""Drop running page-number lines.
|
|
|
|
A single standalone number is ambiguous — it may be a page number or a
|
|
value from a table that lost its row. The line is removed only when the
|
|
document shows the *pattern* repeatedly (at least ``min_repeats`` such
|
|
lines), which is what a running folio looks like and what an isolated data
|
|
point does not.
|
|
"""
|
|
if not text:
|
|
return text
|
|
lines = text.splitlines()
|
|
if len(lines) < 2:
|
|
return text
|
|
candidates = [i for i, ln in enumerate(lines) if _looks_like_page_number(ln)]
|
|
if len(candidates) < min_repeats:
|
|
return text
|
|
drop = set(candidates)
|
|
return "\n".join(ln for i, ln in enumerate(lines) if i not in drop)
|
|
|
|
|
|
def merge_drop_caps(text: str) -> str:
|
|
"""Merge single-letter drop-cap lines into the following paragraph."""
|
|
if not text:
|
|
return text
|
|
return _DROP_CAP_RE.sub(r"\1\2", text)
|
|
|
|
|
|
def postprocess_inline(text: str) -> str:
|
|
"""Safe per-block polish (no page-number stripping)."""
|
|
if not text:
|
|
return text
|
|
out = rejoin_hyphenated_breaks(text)
|
|
out = collapse_dot_leaders(out)
|
|
return out
|
|
|
|
|
|
def postprocess_plain_text(text: str) -> str:
|
|
"""Apply all MD/TXT document-level polish passes."""
|
|
if not text:
|
|
return text
|
|
out = rejoin_hyphenated_breaks(text)
|
|
out = collapse_dot_leaders(out)
|
|
out = filter_page_number_lines(out)
|
|
out = merge_drop_caps(out)
|
|
out = re.sub(r"\n{3,}", "\n\n", out)
|
|
if text.endswith("\n"):
|
|
return out.strip() + "\n"
|
|
return out.strip()
|