678 lines
27 KiB
Python
678 lines
27 KiB
Python
"""Runtime glyph-position compare + U+0000 coverage audit (no engine code changes).
|
|
|
|
Uses real resume PDF only. Produces per-glyph table for \"Professional Experience\".
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import struct
|
|
import sys
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "gateway"))
|
|
sys.path.insert(0, r"C:\Users\Maskan\pdfeng-build\win-local\lib")
|
|
import pdfengine # type: ignore
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from forensic_extract import compute_layout, extract_flat_runs, build_reflow_data # type: ignore
|
|
|
|
PDF = Path(r"C:\Users\Maskan\Downloads\Saqib_Ali_Mir_Resume.pdf")
|
|
OUT = Path(__file__).resolve().parent / "forensic_real"
|
|
FID = "Arial-BoldMT_TrueType_32"
|
|
TARGET = "Professional Experience"
|
|
TOL = 0.05 # PDF points — first position differ threshold
|
|
|
|
|
|
# --- TJ / content-stream helpers -------------------------------------------------
|
|
|
|
def decompress_streams(pdf_bytes: bytes) -> list[str]:
|
|
out = []
|
|
for m in re.finditer(rb"stream\r?\n(.*?)\r?\nendstream", pdf_bytes, re.DOTALL):
|
|
raw = m.group(1)
|
|
try:
|
|
out.append(zlib.decompress(raw).decode("latin-1", "replace"))
|
|
except Exception:
|
|
try:
|
|
out.append(raw.decode("latin-1", "replace"))
|
|
except Exception:
|
|
pass
|
|
return out
|
|
|
|
|
|
def parse_tj_kerning(tj_body: str) -> list[dict]:
|
|
"""Parse a TJ array body into char runs + kerning (thousandths of em).
|
|
|
|
Example: [(Pr)-6(o)7(fe)-6(s)...]
|
|
Returns list of {chars, kern_before} where kern_before is the TJ number
|
|
preceding that string fragment (0 for the first).
|
|
"""
|
|
# Strip outer brackets if present
|
|
body = tj_body.strip()
|
|
if body.startswith("["):
|
|
body = body[1:]
|
|
if body.endswith("]"):
|
|
body = body[:-1]
|
|
items: list[dict] = []
|
|
pos = 0
|
|
pending_kern = 0.0
|
|
while pos < len(body):
|
|
while pos < len(body) and body[pos].isspace():
|
|
pos += 1
|
|
if pos >= len(body):
|
|
break
|
|
if body[pos] == "(":
|
|
# literal string with escape handling
|
|
pos += 1
|
|
chars = []
|
|
while pos < len(body) and body[pos] != ")":
|
|
if body[pos] == "\\" and pos + 1 < len(body):
|
|
chars.append(body[pos + 1])
|
|
pos += 2
|
|
else:
|
|
chars.append(body[pos])
|
|
pos += 1
|
|
pos += 1 # )
|
|
s = "".join(chars)
|
|
items.append({"chars": s, "kern_before": pending_kern})
|
|
pending_kern = 0.0
|
|
elif body[pos] == "<":
|
|
end = body.find(">", pos)
|
|
hexpart = body[pos + 1 : end]
|
|
s = bytes.fromhex(hexpart).decode("latin-1", "replace")
|
|
items.append({"chars": s, "kern_before": pending_kern})
|
|
pending_kern = 0.0
|
|
pos = end + 1
|
|
else:
|
|
# number (kerning)
|
|
m = re.match(r"[+-]?\d+(?:\.\d+)?", body[pos:])
|
|
if not m:
|
|
pos += 1
|
|
continue
|
|
pending_kern = float(m.group(0))
|
|
pos += len(m.group(0))
|
|
return items
|
|
|
|
|
|
def expand_tj_to_glyphs(tj_items: list[dict], font_size: float, tm: list[float]) -> list[dict]:
|
|
"""Expand TJ fragments to per-char records with cumulative X from Tm + kerning.
|
|
|
|
Note: absolute X needs glyph advances; here we only attach kerning adjustment
|
|
(PDF units = kern/1000 * fontSize) and the text matrix at the run start.
|
|
Per-char X from stream alone is incomplete without widths — pair with extraction.
|
|
"""
|
|
rows = []
|
|
for frag in tj_items:
|
|
kern_pdf = (frag["kern_before"] / 1000.0) * font_size
|
|
for i, ch in enumerate(frag["chars"]):
|
|
rows.append({
|
|
"char": ch,
|
|
"kern_before_thousandths": frag["kern_before"] if i == 0 else 0.0,
|
|
"kern_adj_pdf": kern_pdf if i == 0 else 0.0,
|
|
"text_matrix": tm[:],
|
|
})
|
|
return rows
|
|
|
|
|
|
def find_original_heading_tj(pdf_bytes: bytes) -> tuple[list[float], float, list[dict]]:
|
|
"""Locate Professional Experience TJ near y=597.45."""
|
|
for s in decompress_streams(pdf_bytes):
|
|
if "597.45" not in s or "Professional" not in s and "Pr)-6(o)" not in s:
|
|
# still check for the known TJ pattern
|
|
if "597.45" not in s:
|
|
continue
|
|
lines = s.splitlines()
|
|
for i, ln in enumerate(lines):
|
|
if "597.45" in ln and "Tm" in ln:
|
|
# look forward for Tf + TJ
|
|
tm = [float(x) for x in re.findall(r"[+-]?\d+(?:\.\d+)?", ln)[:6]]
|
|
font_size = 12.0
|
|
tj_body = None
|
|
for j in range(i, min(i + 10, len(lines))):
|
|
if " Tf" in lines[j]:
|
|
nums = re.findall(r"[+-]?\d+(?:\.\d+)?", lines[j])
|
|
if nums:
|
|
font_size = float(nums[-1])
|
|
if "TJ" in lines[j] and "[" in lines[j]:
|
|
tj_body = lines[j]
|
|
# may be `... ] TJ` on same line
|
|
m = re.search(r"\[(.*)\]\s*TJ", lines[j])
|
|
if m:
|
|
tj_body = m.group(1)
|
|
break
|
|
if tj_body is None:
|
|
continue
|
|
# Prefer the heading that contains Pr
|
|
if "Pr" not in tj_body and "Professional" not in tj_body:
|
|
continue
|
|
items = parse_tj_kerning(tj_body)
|
|
return tm, font_size, expand_tj_to_glyphs(items, font_size, tm)
|
|
raise RuntimeError("original TJ for Professional Experience not found")
|
|
|
|
|
|
def find_preview_heading_glyphs(pdf_bytes: bytes) -> list[dict]:
|
|
"""Parse per-char Tm + TJ hex CIDs for FXF3 at y≈597.45."""
|
|
rows = []
|
|
for s in decompress_streams(pdf_bytes):
|
|
if "FXF3" not in s or "597.45" not in s:
|
|
continue
|
|
# Match blocks: Tm ... Tf ... [<HHHH>] TJ
|
|
for m in re.finditer(
|
|
r"1 0 0 1 ([0-9.+\-]+) ([0-9.+\-]+) Tm\s+/FXF3 ([0-9.]+) Tf.*?\[<([0-9A-Fa-f]+)>\]\s*TJ",
|
|
s,
|
|
re.DOTALL,
|
|
):
|
|
x, y, size, cid_hex = m.group(1), m.group(2), m.group(3), m.group(4)
|
|
rows.append({
|
|
"x": float(x),
|
|
"y": float(y),
|
|
"font_size": float(size),
|
|
"cid_or_gid": int(cid_hex, 16),
|
|
"cid_hex": cid_hex.upper(),
|
|
"text_matrix": [1.0, 0.0, 0.0, 1.0, float(x), float(y)],
|
|
"kern_adj_pdf": 0.0, # Identity-H per-char emit has no TJ kerning numbers
|
|
})
|
|
# Sort by x ascending (stream is reverse insert order)
|
|
rows.sort(key=lambda r: r["x"])
|
|
return rows
|
|
|
|
|
|
# --- Extraction helpers ---------------------------------------------------------
|
|
|
|
def collect_para_glyphs(para) -> list[dict]:
|
|
rows = []
|
|
for ln in para.lines:
|
|
for r in ln.runs:
|
|
text = r.text or ""
|
|
glyphs = list(r.glyphs)
|
|
for i, g in enumerate(glyphs):
|
|
ch = g.text if getattr(g, "text", None) is not None else (text[i] if i < len(text) else "?")
|
|
# advance = delta to next origin, else bbox_w
|
|
if i + 1 < len(glyphs):
|
|
adv = glyphs[i + 1].origin_x - g.origin_x
|
|
else:
|
|
adv = g.bbox_w if g.bbox_w > 0 else (r.font_size or 12) * 0.5
|
|
row = {
|
|
"char": ch,
|
|
"unicode": ord(ch) if len(ch) == 1 else None,
|
|
"origin_x": g.origin_x,
|
|
"origin_y": g.origin_y,
|
|
"advance": adv,
|
|
"bbox_w": g.bbox_w,
|
|
"bbox_h": g.bbox_h,
|
|
"font_size": g.font_size,
|
|
"font_name": g.font_name,
|
|
"fid": r.internal_font_id,
|
|
}
|
|
for attr in ("glyph_id", "gid", "charcode", "unicode_value", "font_glyph_id"):
|
|
if hasattr(g, attr):
|
|
row["glyph_id"] = getattr(g, attr)
|
|
break
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def find_para(doc):
|
|
page = doc.get_page(0)
|
|
model = page.extract_document_model()
|
|
for idx, para in enumerate(model.paragraphs):
|
|
text = "".join(r.text or "" for ln in para.lines for r in ln.runs)
|
|
if TARGET in text or TARGET.replace(" ", "") in text.replace(" ", ""):
|
|
# Prefer exact heading paragraph
|
|
flat = "".join(r.text or "" for ln in para.lines for r in ln.runs)
|
|
if TARGET in flat or flat.strip().startswith("Professional"):
|
|
return idx, para, flat
|
|
raise RuntimeError("paragraph not found")
|
|
|
|
|
|
# --- U+0000 coverage audit (mirrors engine, no C++ edits) -----------------------
|
|
|
|
def utf8_to_utf16le_mirror(s: str) -> list[int]:
|
|
"""Mirror pdfium_internal.cpp utf8_to_utf16le including trailing NUL."""
|
|
utf16: list[int] = []
|
|
data = s.encode("utf-8")
|
|
i = 0
|
|
while i < len(data):
|
|
c = data[i]
|
|
if c < 0x80:
|
|
cp, extra = c, 0
|
|
elif (c & 0xE0) == 0xC0:
|
|
cp, extra = c & 0x1F, 1
|
|
elif (c & 0xF0) == 0xE0:
|
|
cp, extra = c & 0x0F, 2
|
|
elif (c & 0xF8) == 0xF0:
|
|
cp, extra = c & 0x07, 3
|
|
else:
|
|
i += 1
|
|
continue
|
|
if i + extra >= len(data):
|
|
break
|
|
invalid = False
|
|
for j in range(1, extra + 1):
|
|
nxt = data[i + j]
|
|
if (nxt & 0xC0) != 0x80:
|
|
invalid = True
|
|
break
|
|
cp = (cp << 6) | (nxt & 0x3F)
|
|
if invalid:
|
|
i += 1
|
|
continue
|
|
i += 1 + extra
|
|
if cp < 0x10000:
|
|
utf16.append(cp)
|
|
else:
|
|
cp -= 0x10000
|
|
utf16.append((cp >> 10) + 0xD800)
|
|
utf16.append((cp & 0x3FF) + 0xDC00)
|
|
utf16.append(0) # <-- engine always appends NUL terminator
|
|
return utf16
|
|
|
|
|
|
def to_codepoints_mirror(s: str) -> list[int]:
|
|
"""Mirror toCodepoints lambda in pdfium_edit_reflow.cpp."""
|
|
u16 = utf8_to_utf16le_mirror(s)
|
|
cps: list[int] = []
|
|
i = 0
|
|
while i < len(u16):
|
|
cp = u16[i]
|
|
if 0xD800 <= cp <= 0xDBFF and i + 1 < len(u16):
|
|
low = u16[i + 1]
|
|
if 0xDC00 <= low <= 0xDFFF:
|
|
cp = 0x10000 + ((cp - 0xD800) << 10) + (low - 0xDC00)
|
|
i += 2
|
|
else:
|
|
i += 1
|
|
else:
|
|
i += 1
|
|
cps.append(cp)
|
|
return cps
|
|
|
|
|
|
def unicode_cmap(font_bytes: bytes) -> dict[int, int]:
|
|
"""Prefer MS/Unicode cmap (what FreeType typically uses for FT_Get_Char_Index)."""
|
|
from fontTools.ttLib import TTFont # type: ignore
|
|
import io
|
|
tt = TTFont(io.BytesIO(font_bytes))
|
|
glyph_order = tt.getGlyphOrder()
|
|
name_to_gid = {n: i for i, n in enumerate(glyph_order)}
|
|
|
|
def to_map(table) -> dict[int, int]:
|
|
out = {}
|
|
for cp, name in table.cmap.items():
|
|
out[cp] = name_to_gid.get(name, 0) if isinstance(name, str) else int(name)
|
|
return out
|
|
|
|
preferred: list[dict[int, int]] = []
|
|
fallback: list[dict[int, int]] = []
|
|
for t in tt["cmap"].tables:
|
|
mapping = to_map(t)
|
|
if t.platformID == 3 and t.platEncID in (1, 10):
|
|
preferred.append(mapping)
|
|
elif t.platformID == 0:
|
|
preferred.append(mapping)
|
|
else:
|
|
fallback.append(mapping)
|
|
merged: dict[int, int] = {}
|
|
for m in (preferred or fallback):
|
|
merged.update(m)
|
|
return merged
|
|
|
|
|
|
def cmap_has_glyph(font_bytes: bytes, codepoint: int) -> tuple[bool, int]:
|
|
"""Return (has_glyph_like_engine, gid). Engine: FT_Get_Char_Index != 0."""
|
|
try:
|
|
cmap = unicode_cmap(font_bytes)
|
|
gid = int(cmap.get(codepoint, 0) or 0)
|
|
return (gid != 0), gid
|
|
except Exception:
|
|
return False, -1
|
|
|
|
|
|
def parse_emit_log(log_text: str) -> list[dict]:
|
|
"""Parse [EMIT_FONT] lines for Professional Experience heading."""
|
|
rows = []
|
|
for m in re.finditer(
|
|
r"\[EMIT_FONT\] text='([^']*)'.*?atX=([0-9.+\-]+)\s+baselineY=([0-9.+\-]+).*?runPerChar=(\d+)",
|
|
log_text,
|
|
):
|
|
rows.append({
|
|
"char": m.group(1),
|
|
"origin_x": float(m.group(2)),
|
|
"origin_y": float(m.group(3)),
|
|
"runPerChar": int(m.group(4)),
|
|
"text_matrix": [1.0, 0.0, 0.0, 1.0, float(m.group(2)), float(m.group(3))],
|
|
"emitted": True,
|
|
})
|
|
return rows
|
|
|
|
|
|
def align_emit_to_target(target: str, emit_rows: list[dict], advances: list[float], baseline_y: float) -> list[dict]:
|
|
"""Align EMIT rows to TARGET; spaces skipped by runPerChar get emitted=False with inferred X."""
|
|
out: list[dict] = []
|
|
ei = 0
|
|
x_cursor = None
|
|
for i, ch in enumerate(target):
|
|
adv = advances[i] if i < len(advances) else 0.0
|
|
if ch == " ":
|
|
# runPerChar path: if (seg.text[c] != ' ') emitObj(...) — space NOT emitted
|
|
if x_cursor is None and out:
|
|
x_cursor = out[-1]["origin_x"] + out[-1]["advance"]
|
|
elif x_cursor is None:
|
|
x_cursor = 0.0
|
|
out.append({
|
|
"char": " ",
|
|
"origin_x": x_cursor,
|
|
"origin_y": baseline_y,
|
|
"advance": adv,
|
|
"emitted": False,
|
|
"glyph_id": None,
|
|
"kern_adj_pdf": 0.0,
|
|
"kern_before_thousandths": 0.0,
|
|
"text_matrix": None,
|
|
"note": "space skipped by runPerChar emit (x advanced only)",
|
|
})
|
|
x_cursor = x_cursor + adv
|
|
continue
|
|
if ei >= len(emit_rows):
|
|
out.append({
|
|
"char": ch, "origin_x": None, "origin_y": None, "advance": adv,
|
|
"emitted": False, "glyph_id": None, "kern_adj_pdf": 0.0,
|
|
"kern_before_thousandths": 0.0, "text_matrix": None,
|
|
"note": "missing emit",
|
|
})
|
|
continue
|
|
er = emit_rows[ei]
|
|
ei += 1
|
|
# If emit char doesn't match (shouldn't), still take position
|
|
row = {
|
|
"char": ch,
|
|
"origin_x": er["origin_x"],
|
|
"origin_y": er["origin_y"],
|
|
"advance": adv,
|
|
"emitted": True,
|
|
"glyph_id": None,
|
|
"kern_adj_pdf": 0.0,
|
|
"kern_before_thousandths": 0.0,
|
|
"text_matrix": er["text_matrix"],
|
|
"note": "" if er["char"] == ch else f"emit_char_mismatch emit={er['char']!r}",
|
|
}
|
|
out.append(row)
|
|
x_cursor = er["origin_x"] + adv
|
|
return out
|
|
|
|
|
|
def main():
|
|
import io
|
|
from contextlib import redirect_stderr, redirect_stdout
|
|
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
orig_bytes = PDF.read_bytes()
|
|
|
|
print("=== LOAD ORIGINAL ===")
|
|
doc = pdfengine.PdfDocument.load_from_file(str(PDF), "")
|
|
pidx, para, flat = find_para(doc)
|
|
print(f"paragraph idx={pidx} text={flat!r}")
|
|
|
|
orig_glyphs = collect_para_glyphs(para)
|
|
joined = "".join(g["char"] for g in orig_glyphs)
|
|
start = joined.find(TARGET)
|
|
if start < 0:
|
|
target_glyphs = [g for g in orig_glyphs if g.get("fid") == FID]
|
|
else:
|
|
target_glyphs = orig_glyphs[start : start + len(TARGET)]
|
|
|
|
print(f"orig glyph count for target={len(target_glyphs)} chars={''.join(g['char'] for g in target_glyphs)!r}")
|
|
|
|
tm, fsize, tj_rows = find_original_heading_tj(orig_bytes)
|
|
print(f"original Tm={tm} fontSize={fsize} TJ expanded chars={len(tj_rows)}")
|
|
|
|
for i, g in enumerate(target_glyphs):
|
|
if i < len(tj_rows):
|
|
g["kern_before_thousandths"] = tj_rows[i]["kern_before_thousandths"]
|
|
g["kern_adj_pdf"] = tj_rows[i]["kern_adj_pdf"]
|
|
g["text_matrix"] = tj_rows[i]["text_matrix"]
|
|
else:
|
|
g["kern_before_thousandths"] = 0.0
|
|
g["kern_adj_pdf"] = 0.0
|
|
g["text_matrix"] = tm
|
|
|
|
layout = compute_layout(para)
|
|
dominant_fid = next((r["fid"] for r in layout["seedRuns"] if r["text"].strip() and r["fid"]), FID)
|
|
dom_run = next((r for r in layout["seedRuns"] if r["fid"] == dominant_fid and r["text"].strip()), layout["seedRuns"][0])
|
|
runs = extract_flat_runs(layout["seedRuns"], dominant_fid, dom_run["size"], dom_run["color"])
|
|
data = build_reflow_data(layout, runs, layout["origLines"], f"pos-{pidx}")
|
|
op = {"version": "1.0", "operations": [{
|
|
"id": "pos", "type": "reflow_paragraph", "pageIndex": 0, "data": data,
|
|
}]}
|
|
|
|
print("=== APPLY REFLOW (edit-entry, unchanged text) ===")
|
|
# Capture spdlog on stderr/stdout if redirected; also read prior pattern from engine
|
|
log_buf = io.StringIO()
|
|
doc2 = pdfengine.PdfDocument.load_from_file(str(PDF), "")
|
|
# spdlog writes to stderr typically via our capture of the process — here we rely on
|
|
# re-parsing from a file we tee. Capture by running and also reading STAGE_5 from
|
|
# a side channel: apply then parse preview stream + reconstruct from known EMIT positions.
|
|
doc2.apply_edits(json.dumps(op))
|
|
prev_bytes = doc2.save_full()
|
|
(OUT / "preview_pos.pdf").write_bytes(prev_bytes)
|
|
|
|
# Client advances from seed (same numbers reflow uses when lengths match)
|
|
client_adv = None
|
|
for r in layout["seedRuns"]:
|
|
if r.get("text") == TARGET and r.get("advances") and len(r["advances"]) == len(TARGET):
|
|
client_adv = list(r["advances"])
|
|
break
|
|
if client_adv is None:
|
|
client_adv = [g["advance"] for g in target_glyphs]
|
|
|
|
# Preview positions: from content stream FXF3 (non-space) + inferred space
|
|
stream_prev = find_preview_heading_glyphs(prev_bytes)
|
|
print(f"preview stream FXF3 glyphs={len(stream_prev)} (spaces intentionally not emitted in runPerChar)")
|
|
|
|
# Build synthetic emit rows from stream (sorted by x) — chars from TARGET without spaces
|
|
non_space = [ch for ch in TARGET if ch != " "]
|
|
emit_rows = []
|
|
for i, ch in enumerate(non_space):
|
|
if i < len(stream_prev):
|
|
sp = stream_prev[i]
|
|
emit_rows.append({
|
|
"char": ch,
|
|
"origin_x": sp["x"],
|
|
"origin_y": sp["y"],
|
|
"runPerChar": 1,
|
|
"text_matrix": sp["text_matrix"],
|
|
"emitted": True,
|
|
"glyph_id": sp["cid_or_gid"],
|
|
})
|
|
# Also try to enrich from glyph_pos_log if present from prior tee
|
|
log_path = OUT / "glyph_pos_log.txt"
|
|
if log_path.exists():
|
|
parsed = parse_emit_log(log_path.read_text(encoding="utf-8", errors="replace"))
|
|
if len(parsed) >= len(non_space):
|
|
# Prefer live EMIT atX from log (more authoritative for this run if same)
|
|
emit_rows = []
|
|
for i, ch in enumerate(non_space):
|
|
er = parsed[i]
|
|
emit_rows.append({**er, "char": ch})
|
|
|
|
baseline_y = target_glyphs[0]["origin_y"] if target_glyphs else 597.45
|
|
prev_glyphs = align_emit_to_target(TARGET, emit_rows, client_adv, baseline_y)
|
|
# Attach GIDs from stream to emitted glyphs
|
|
si = 0
|
|
for g in prev_glyphs:
|
|
if g["emitted"] and si < len(stream_prev):
|
|
g["glyph_id"] = stream_prev[si]["cid_or_gid"]
|
|
si += 1
|
|
|
|
font_bytes = bytes(doc.get_font_data(FID) or b"")
|
|
print(f"embedded font bytes={len(font_bytes)}")
|
|
for g in target_glyphs:
|
|
if g.get("unicode") is not None and font_bytes:
|
|
has, gid = cmap_has_glyph(font_bytes, g["unicode"])
|
|
g["glyph_id"] = gid
|
|
g["cmap_has"] = has
|
|
|
|
print("\n=== PER-GLYPH COMPARISON (orig extraction vs emit atX) ===")
|
|
print(
|
|
f"{'#':>2} {'ch':>3} {'gidO':>5} {'gidP':>5} {'emit':>4} "
|
|
f"{'xO':>10} {'xP':>10} {'dx':>8} "
|
|
f"{'yO':>10} {'yP':>10} {'dy':>8} "
|
|
f"{'advO':>8} {'advP':>8} {'dAdv':>8} "
|
|
f"{'kernO':>8} {'kernP':>8} Tm tx,ty"
|
|
)
|
|
first_diff = None
|
|
rows_out = []
|
|
n = min(len(target_glyphs), len(prev_glyphs), len(TARGET))
|
|
for i in range(n):
|
|
o, p = target_glyphs[i], prev_glyphs[i]
|
|
xP = p.get("origin_x")
|
|
yP = p.get("origin_y")
|
|
if xP is None:
|
|
dx = dy = float("nan")
|
|
pos_diff = True
|
|
else:
|
|
dx = xP - o["origin_x"]
|
|
dy = yP - o["origin_y"]
|
|
pos_diff = (not p.get("emitted")) or abs(dx) > TOL or abs(dy) > TOL
|
|
dadv = (p["advance"] - o["advance"]) if p.get("advance") is not None else float("nan")
|
|
adv_diff = abs(dadv) > TOL if dadv == dadv else True
|
|
|
|
if first_diff is None and (pos_diff or adv_diff):
|
|
reason = "not_emitted" if not p.get("emitted") else ("position" if pos_diff else "advance")
|
|
first_diff = {
|
|
"index": i,
|
|
"char": o["char"],
|
|
"reason": reason,
|
|
"dx": dx, "dy": dy, "dAdv": dadv,
|
|
"xO": o["origin_x"], "xP": xP,
|
|
"yO": o["origin_y"], "yP": yP,
|
|
"advO": o["advance"], "advP": p.get("advance"),
|
|
"kernO": o.get("kern_adj_pdf", 0), "kernP": p.get("kern_adj_pdf", 0),
|
|
"gidO": o.get("glyph_id"), "gidP": p.get("glyph_id"),
|
|
"tmO": o.get("text_matrix"), "tmP": p.get("text_matrix"),
|
|
"note": p.get("note"),
|
|
}
|
|
|
|
tmO, tmP = o.get("text_matrix"), p.get("text_matrix")
|
|
tmOs = f"O[{tmO[4]:.3f},{tmO[5]:.3f}]" if tmO and len(tmO) >= 6 else "O[—]"
|
|
tmPs = f"P[{tmP[4]:.3f},{tmP[5]:.3f}]" if tmP and len(tmP) >= 6 else ("P[— skipped]" if not p.get("emitted") else "P[—]")
|
|
xPs = f"{xP:10.4f}" if xP is not None else f"{'None':>10}"
|
|
yPs = f"{yP:10.4f}" if yP is not None else f"{'None':>10}"
|
|
dxs = f"{dx:8.4f}" if dx == dx else f"{'nan':>8}"
|
|
dys = f"{dy:8.4f}" if dy == dy else f"{'nan':>8}"
|
|
print(
|
|
f"{i:2d} {o['char']:>3} {str(o.get('glyph_id')):>5} {str(p.get('glyph_id')):>5} "
|
|
f"{'Y' if p.get('emitted') else 'N':>4} "
|
|
f"{o['origin_x']:10.4f} {xPs} {dxs} "
|
|
f"{o['origin_y']:10.4f} {yPs} {dys} "
|
|
f"{o['advance']:8.4f} {p.get('advance', 0):8.4f} {dadv:8.4f} "
|
|
f"{o.get('kern_adj_pdf', 0):8.4f} {p.get('kern_adj_pdf', 0):8.4f} {tmOs} {tmPs}"
|
|
)
|
|
rows_out.append({
|
|
"i": i, "char": o["char"],
|
|
"gid_orig": o.get("glyph_id"), "gid_prev": p.get("glyph_id"),
|
|
"emitted": bool(p.get("emitted")),
|
|
"x_orig": o["origin_x"], "x_prev": xP, "dx": dx if dx == dx else None,
|
|
"y_orig": o["origin_y"], "y_prev": yP, "dy": dy if dy == dy else None,
|
|
"adv_orig": o["advance"], "adv_prev": p.get("advance"), "d_adv": dadv if dadv == dadv else None,
|
|
"kern_orig_pdf": o.get("kern_adj_pdf", 0),
|
|
"kern_orig_thousandths": o.get("kern_before_thousandths", 0),
|
|
"kern_prev_pdf": p.get("kern_adj_pdf", 0),
|
|
"tm_orig": o.get("text_matrix"),
|
|
"tm_prev": p.get("text_matrix"),
|
|
"pos_differs": pos_diff,
|
|
"adv_differs": adv_diff,
|
|
"note": p.get("note"),
|
|
})
|
|
|
|
print("\n=== FIRST GLYPH WHOSE POSITION DIFFERS ===")
|
|
print(json.dumps(first_diff, indent=2))
|
|
|
|
# Visible-glyph-only: ignore intentional space skip
|
|
first_visible = None
|
|
for row in rows_out:
|
|
if row["char"] == " ":
|
|
continue
|
|
if row["pos_differs"] or row["adv_differs"]:
|
|
first_visible = row
|
|
break
|
|
print("\n=== FIRST VISIBLE (non-space) GLYPH DIFF ===")
|
|
print(json.dumps(first_visible, indent=2))
|
|
|
|
print("\n=== U+0000 COVERAGE AUDIT ===")
|
|
run_texts = [r["text"] for r in layout["seedRuns"] if r.get("text")]
|
|
print(f"seed run texts: {run_texts!r}")
|
|
all_cps: list[int] = []
|
|
for t in run_texts:
|
|
cps = to_codepoints_mirror(t)
|
|
print(f" text={t!r}")
|
|
print(f" utf16le_mirror (incl NUL) = {[f'U+{c:04X}' for c in utf8_to_utf16le_mirror(t)]}")
|
|
print(f" toCodepoints_mirror = {[f'U+{c:04X}' for c in cps]}")
|
|
print(f" contains U+0000? {0 in cps}")
|
|
all_cps.extend(cps)
|
|
|
|
print(f"aggregated codepoints ({len(all_cps)}): {[f'U+{c:04X}' for c in all_cps]}")
|
|
print(f"U+0000 count in aggregated set: {all_cps.count(0)}")
|
|
|
|
missing_real = []
|
|
lacking = []
|
|
has0 = False
|
|
gid0 = -1
|
|
if font_bytes:
|
|
has0, gid0 = cmap_has_glyph(font_bytes, 0)
|
|
print(f"Unicode cmap U+0000: hasGlyph_engine_rule={has0} gid={gid0}")
|
|
print(" (engine hasGlyph: FT_Get_Char_Index(cp) != 0; gid==0 => MISSING)")
|
|
# Mac Roman may map NUL — note for audit
|
|
try:
|
|
from fontTools.ttLib import TTFont
|
|
tt = TTFont(io.BytesIO(font_bytes))
|
|
for t in tt["cmap"].tables:
|
|
if 0 in t.cmap:
|
|
print(f" note: platform={t.platformID} enc={t.platEncID} maps U+0000 -> {t.cmap[0]} "
|
|
f"(FreeType Unicode cmap path still typically misses this)")
|
|
except Exception:
|
|
pass
|
|
for ch in TARGET:
|
|
has, gid = cmap_has_glyph(font_bytes, ord(ch))
|
|
if not has:
|
|
missing_real.append((ch, ord(ch), gid))
|
|
print(f"missing real TARGET chars in Unicode cmap: {missing_real or 'NONE'}")
|
|
real_cps = [c for c in all_cps if c != 0]
|
|
lacking = [c for c in real_cps if not cmap_has_glyph(font_bytes, c)[0]]
|
|
print(f"coverage without U+0000: lacking={ [f'U+{c:04X}' for c in lacking] or 'NONE' }")
|
|
forces = (0 in all_cps) and (not has0) and (not lacking)
|
|
print(f"CONCLUSION: U+0000 {'DOES' if forces else 'does not alone'} force fullProvenLacking "
|
|
f"for this paragraph (runtime also logged only U+0000 as MISSING)")
|
|
|
|
report = {
|
|
"target": TARGET,
|
|
"fid": FID,
|
|
"orig_tm": tm,
|
|
"orig_font_size": fsize,
|
|
"first_diff": first_diff,
|
|
"first_visible_diff": first_visible,
|
|
"glyphs": rows_out,
|
|
"u0000_audit": {
|
|
"run_texts": run_texts,
|
|
"codepoints_per_run": [
|
|
{"text": t, "cps": [f"U+{c:04X}" for c in to_codepoints_mirror(t)]}
|
|
for t in run_texts
|
|
],
|
|
"u0000_injected_by": "utf8_to_utf16le() always push_back(0); toCodepoints iterates full vector including NUL",
|
|
"engine_hasGlyph_rule": "FT_Get_Char_Index(face, cp) != 0",
|
|
"embedded_unicode_cmap_has_u0000": has0,
|
|
"embedded_u0000_gid": gid0,
|
|
"embedded_missing_real_chars": missing_real,
|
|
"coverage_ok_if_u0000_ignored": not lacking,
|
|
"runtime_log": "FONT_COVERAGE_DEBUG full font MISSING codepoint U+0000 only",
|
|
},
|
|
}
|
|
(OUT / "glyph_pos_report.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
|
|
print(f"\nWrote {OUT / 'glyph_pos_report.json'}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|