131 lines
5.0 KiB
Python
131 lines
5.0 KiB
Python
"""Compare full-page render hashes + PDF objects across corpus files."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sys
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "gateway"))
|
|
import pdfengine # type: ignore
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from forensic_extract import compute_layout, extract_flat_runs, build_reflow_data # type: ignore
|
|
from forensic_obj_compare import apply_and_save # type: ignore
|
|
|
|
|
|
def inflate(pdf_bytes: bytes) -> list[str]:
|
|
out = []
|
|
for m in re.finditer(rb"stream\r?\n(.*?)\r?\nendstream", pdf_bytes, re.DOTALL):
|
|
try:
|
|
out.append(zlib.decompress(m.group(1)).decode("latin-1", "replace"))
|
|
except Exception:
|
|
pass
|
|
return out
|
|
|
|
|
|
def font_dicts(b: bytes) -> list[str]:
|
|
return [m.group(0).decode("latin-1", "replace") for m in re.finditer(rb"<</BaseFont/[^>]+>>", b)]
|
|
|
|
|
|
def sha_full(doc, dpi=144) -> tuple[str, int]:
|
|
img = doc.get_page(0).render(dpi)
|
|
png = bytes(img.data)
|
|
return hashlib.sha256(png).hexdigest()[:24], len(png)
|
|
|
|
|
|
def run(pdf: Path):
|
|
print(f"\n######## {pdf}")
|
|
doc = pdfengine.PdfDocument.load_from_file(str(pdf), "")
|
|
s0, n0 = sha_full(doc)
|
|
print(f"ORIG sha={s0} pngbytes={n0}")
|
|
model = doc.get_page(0).extract_document_model()
|
|
if not model.paragraphs:
|
|
print("no paragraphs")
|
|
return
|
|
para = model.paragraphs[0]
|
|
layout = compute_layout(para)
|
|
for f in doc.get_fonts(0, 0)[:6]:
|
|
print(f" FontInfo name={f.font_name} emb={f.is_embedded} type={f.type} "
|
|
f"subset={getattr(f, 'is_subset', None)} id={f.internal_font_id}")
|
|
print(" seedRuns:", [(r["fontName"], r["fid"], round(r["size"], 2),
|
|
len(r["advances"]) if r.get("advances") else 0)
|
|
for r in layout["seedRuns"][:4]])
|
|
|
|
fid = next((r["fid"] for r in layout["seedRuns"] if r["text"].strip() and r["fid"]), "")
|
|
dom = next((r for r in layout["seedRuns"] if r["text"].strip()), layout["seedRuns"][0])
|
|
flat = extract_flat_runs(layout["seedRuns"], fid, dom["size"], dom["color"])
|
|
|
|
data = build_reflow_data(layout, flat, layout["origLines"], "cmp")
|
|
op = {"version": "1.0", "operations": [{
|
|
"id": "e", "type": "reflow_paragraph", "pageIndex": 0, "data": data,
|
|
}]}
|
|
outp = Path(__file__).resolve().parent / "_tmp_entry.pdf"
|
|
apply_and_save(pdf, op, outp)
|
|
d2 = pdfengine.PdfDocument.load_from_memory(outp.read_bytes(), "")
|
|
s1, n1 = sha_full(d2)
|
|
print(f"ENTRY sha={s1} pngbytes={n1} {'IDENTICAL' if s0 == s1 else '*** DIFFERENT ***'}")
|
|
b = outp.read_bytes()
|
|
print(" entry fonts:", font_dicts(b))
|
|
for s in inflate(b):
|
|
if "Tm" in s or "TJ" in s or "Tj" in s:
|
|
one_line = s.replace("\n", " | ")
|
|
print(" stream:", one_line[:700])
|
|
tj = s.count(" TJ")
|
|
print(f" TJ count={tj}")
|
|
break
|
|
|
|
flat1 = [{**flat[0], "text": flat[0]["text"] + "x"}] + flat[1:]
|
|
data1 = build_reflow_data(layout, flat1, None, "cmp")
|
|
# also try force whole if supported
|
|
data1_force = {**build_reflow_data(layout, flat, layout["origLines"], "cmp"),
|
|
"forensicForceWholeRun": True}
|
|
op1 = {"version": "1.0", "operations": [{
|
|
"id": "o", "type": "reflow_paragraph", "pageIndex": 0, "data": data1,
|
|
}]}
|
|
out1 = Path(__file__).resolve().parent / "_tmp_one.pdf"
|
|
apply_and_save(pdf, op1, out1)
|
|
d3 = pdfengine.PdfDocument.load_from_memory(out1.read_bytes(), "")
|
|
s2, n2 = sha_full(d3)
|
|
print(f"ONE sha={s2} pngbytes={n2} vsORIG={'SAME' if s0 == s2 else 'DIFF'} vsENTRY={'SAME' if s1 == s2 else 'DIFF'}")
|
|
|
|
# force whole on unchanged
|
|
opf = {"version": "1.0", "operations": [{
|
|
"id": "f", "type": "reflow_paragraph", "pageIndex": 0, "data": data1_force,
|
|
}]}
|
|
outf = Path(__file__).resolve().parent / "_tmp_force.pdf"
|
|
apply_and_save(pdf, opf, outf)
|
|
df = pdfengine.PdfDocument.load_from_memory(outf.read_bytes(), "")
|
|
sf, nf = sha_full(df)
|
|
bf = outf.read_bytes()
|
|
tjf = sum(s.count(" TJ") for s in inflate(bf))
|
|
tje = sum(s.count(" TJ") for s in inflate(b))
|
|
print(f"FORCE sha={sf} pngbytes={nf} vsORIG={'SAME' if s0 == sf else 'DIFF'} "
|
|
f"vsENTRY={'SAME' if s1 == sf else 'DIFF'} TJ_entry={tje} TJ_force={tjf}")
|
|
if "forensicForceWholeRun" in json.dumps(opf):
|
|
if tje != tjf:
|
|
print(" => forensicForceWholeRun ACTIVE (TJ count changed)")
|
|
else:
|
|
print(" => forensicForceWholeRun NOT in binary yet (TJ unchanged) OR no-op")
|
|
|
|
|
|
def main():
|
|
for p in [
|
|
ROOT / "corpus" / "basic" / "hello_world.pdf",
|
|
ROOT / "tests" / "edits" / "forensic_multiline.pdf",
|
|
ROOT / "corpus" / "fonts" / "embedded_truetype.pdf",
|
|
ROOT / "corpus" / "fonts" / "subset_font.pdf",
|
|
]:
|
|
if p.exists():
|
|
try:
|
|
run(p)
|
|
except Exception as e:
|
|
print(f"FAILED {p}: {e}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|