278 lines
9.8 KiB
Python
278 lines
9.8 KiB
Python
<<<<<<< HEAD
|
|
import re
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
import sys
|
|
=======
|
|
import contextlib
|
|
>>>>>>> 2638dfe969874b954502b26656df48ad971e306d
|
|
import os
|
|
import sys
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
# Add both the root directory and the gateway directory to PYTHONPATH
|
|
sys.path.insert(0, os.path.abspath("."))
|
|
sys.path.insert(0, os.path.abspath(".."))
|
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
|
|
|
|
with contextlib.suppress(ImportError):
|
|
from app.main import app
|
|
|
|
client = TestClient(app)
|
|
|
|
<<<<<<< HEAD
|
|
# Regex: six uppercase ASCII letters followed by '+'
|
|
SUBSET_PREFIX_RE = re.compile(r'^[A-Z]{6}\+')
|
|
|
|
=======
|
|
>>>>>>> 2638dfe969874b954502b26656df48ad971e306d
|
|
|
|
def get_doc_id(filename: str) -> str:
|
|
"""Upload a PDF from corpus/fonts and return its document ID."""
|
|
filepath = os.path.abspath(f"../corpus/fonts/{filename}")
|
|
with open(filepath, "rb") as f:
|
|
resp = client.post("/documents", files={"file": (filename, f, "application/pdf")})
|
|
assert resp.status_code == 201, f"Failed to load {filename}: {resp.json()}"
|
|
return resp.json()["id"]
|
|
|
|
|
|
<<<<<<< HEAD
|
|
# =========================================================================
|
|
# 1. Vertical font regression
|
|
# =========================================================================
|
|
|
|
=======
|
|
>>>>>>> 2638dfe969874b954502b26656df48ad971e306d
|
|
def test_is_vertical_regression():
|
|
"""Identity-V fonts must be flagged isVertical; horizontal fonts must not."""
|
|
doc_id = get_doc_id("vertical_text.pdf")
|
|
resp = client.get(f"/documents/{doc_id}/fonts")
|
|
assert resp.status_code == 200
|
|
fonts = resp.json()
|
|
assert len(fonts) > 0
|
|
font = fonts[0]
|
|
assert font["isVertical"] is True
|
|
assert font["encoding"] == "Identity-V"
|
|
|
|
<<<<<<< HEAD
|
|
=======
|
|
# 2. Verify horizontal fonts are not falsely detected as vertical
|
|
>>>>>>> 2638dfe969874b954502b26656df48ad971e306d
|
|
doc_id_h = get_doc_id("utf-8.pdf")
|
|
resp_h = client.get(f"/documents/{doc_id_h}/fonts")
|
|
fonts_h = resp_h.json()
|
|
assert len(fonts_h) > 0
|
|
for f in fonts_h:
|
|
assert f["isVertical"] is False
|
|
|
|
|
|
<<<<<<< HEAD
|
|
# =========================================================================
|
|
# 2. Internal Font ID: no duplicate subset prefix (core regression)
|
|
# =========================================================================
|
|
|
|
def test_internal_font_id_no_duplicate_prefix():
|
|
"""
|
|
Regression: subset fonts must NOT produce "ABCDEF_ABCDEF+Arial".
|
|
The internalFontId for a subset font should be the fontName itself
|
|
(e.g. "ABCDEF+Arial"), which already encodes the subset tag.
|
|
"""
|
|
doc_id = get_doc_id("subset_font.pdf")
|
|
resp = client.get(f"/documents/{doc_id}/fonts")
|
|
assert resp.status_code == 200
|
|
fonts = resp.json()
|
|
assert len(fonts) > 0
|
|
|
|
for font in fonts:
|
|
fid = font["internalFontId"]
|
|
tag = font.get("subsetTag", "")
|
|
|
|
if font.get("isSubset") and tag:
|
|
# Must NOT start with "TAG_TAG"
|
|
assert not fid.startswith(tag + "_" + tag), (
|
|
f"Duplicate subset prefix detected: internalFontId='{fid}'"
|
|
)
|
|
# Must equal fontName directly (e.g. "ABCDEF+Arial")
|
|
assert fid == font["fontName"], (
|
|
f"Expected internalFontId==fontName for subset font, "
|
|
f"got '{fid}' vs '{font['fontName']}'"
|
|
)
|
|
|
|
|
|
def test_internal_font_id_subset_format():
|
|
"""
|
|
For any subset font the internalFontId must match the pattern
|
|
ABCDEF+BaseName — exactly the fontName reported by PDFium.
|
|
"""
|
|
=======
|
|
def test_internal_font_id_regression():
|
|
# Verify subset fonts do not duplicate subset prefixes
|
|
>>>>>>> 2638dfe969874b954502b26656df48ad971e306d
|
|
doc_id = get_doc_id("text_font.pdf")
|
|
resp = client.get(f"/documents/{doc_id}/fonts")
|
|
assert resp.status_code == 200
|
|
fonts = resp.json()
|
|
assert len(fonts) > 0
|
|
|
|
for font in fonts:
|
|
if font.get("isSubset"):
|
|
assert font["subsetTag"] in font["fontName"]
|
|
<<<<<<< HEAD
|
|
# internalFontId == fontName (e.g. "ABCDEF+Arial")
|
|
assert font["internalFontId"] == font["fontName"]
|
|
# The ID must contain exactly one '+' from the subset tag
|
|
assert font["internalFontId"].count("+") == 1
|
|
|
|
|
|
# =========================================================================
|
|
# 3. Non-subset font ID format
|
|
# =========================================================================
|
|
|
|
def test_internal_font_id_non_subset_format():
|
|
"""
|
|
Non-subset fonts must have internalFontId = fontName_type_flags.
|
|
Examples: "Helvetica_Type1_32", "Times-Roman_Type1_32".
|
|
"""
|
|
doc_id = get_doc_id("utf-8.pdf")
|
|
resp = client.get(f"/documents/{doc_id}/fonts")
|
|
assert resp.status_code == 200
|
|
fonts = resp.json()
|
|
assert len(fonts) > 0
|
|
|
|
for font in fonts:
|
|
if not font.get("isSubset"):
|
|
expected = f"{font['fontName']}_{font['type']}_{font['flags']}"
|
|
assert font["internalFontId"] == expected, (
|
|
f"Non-subset internalFontId mismatch: "
|
|
f"got '{font['internalFontId']}', expected '{expected}'"
|
|
)
|
|
# Must NOT contain a '+' (no subset prefix)
|
|
assert "+" not in font["internalFontId"]
|
|
|
|
|
|
# =========================================================================
|
|
# 4. ID stability across document-level and page-level APIs
|
|
# =========================================================================
|
|
|
|
def test_font_id_stable_across_apis():
|
|
"""
|
|
The internalFontId for the same font must be identical whether queried
|
|
from the document-level /fonts endpoint or the page-level /pages/0/fonts.
|
|
"""
|
|
for pdf in ("subset_font.pdf", "utf-8.pdf", "vertical_text.pdf"):
|
|
filepath = os.path.abspath(f"../corpus/fonts/{pdf}")
|
|
if not os.path.exists(filepath):
|
|
continue
|
|
|
|
doc_id = get_doc_id(pdf)
|
|
|
|
doc_resp = client.get(f"/documents/{doc_id}/fonts")
|
|
page_resp = client.get(f"/documents/{doc_id}/pages/0/fonts")
|
|
assert doc_resp.status_code == 200
|
|
assert page_resp.status_code == 200
|
|
|
|
doc_ids = {f["fontName"]: f["internalFontId"] for f in doc_resp.json()}
|
|
page_ids = {f["fontName"]: f["internalFontId"] for f in page_resp.json()}
|
|
|
|
for name in page_ids:
|
|
assert name in doc_ids, f"Page font '{name}' not in doc fonts for {pdf}"
|
|
assert page_ids[name] == doc_ids[name], (
|
|
f"ID mismatch for '{name}' in {pdf}: "
|
|
f"doc='{doc_ids[name]}' vs page='{page_ids[name]}'"
|
|
)
|
|
|
|
|
|
# =========================================================================
|
|
# 5. CID collection regression
|
|
# =========================================================================
|
|
=======
|
|
assert not font["internalFontId"].startswith(
|
|
font["subsetTag"] + "_" + font["subsetTag"]
|
|
)
|
|
|
|
>>>>>>> 2638dfe969874b954502b26656df48ad971e306d
|
|
|
|
def test_cid_collection_regression():
|
|
"""Adobe CID collections must use the 'Adobe-' prefix."""
|
|
doc_id = get_doc_id("vertical_text.pdf")
|
|
resp = client.get(f"/documents/{doc_id}/fonts")
|
|
fonts = resp.json()
|
|
assert len(fonts) > 0
|
|
for font in fonts:
|
|
if font.get("cidSystemInfo") and font.get("cidSystemInfo") != "None":
|
|
assert "Adobe-" in font["cidSystemInfo"]
|
|
|
|
|
|
<<<<<<< HEAD
|
|
def test_cns1_regression():
|
|
"""Verify Adobe-CNS1 (Traditional Chinese) CID fonts and text extraction."""
|
|
doc_id = get_doc_id("cns1_test.pdf")
|
|
|
|
# 1. Verify font extraction
|
|
resp = client.get(f"/documents/{doc_id}/fonts")
|
|
assert resp.status_code == 200
|
|
fonts = resp.json()
|
|
assert len(fonts) > 0
|
|
cns1_fonts = [f for f in fonts if f.get("cidSystemInfo") == "Adobe-CNS1"]
|
|
assert len(cns1_fonts) > 0, "No Adobe-CNS1 fonts detected"
|
|
|
|
# 2. Verify text extraction
|
|
resp = client.get(f"/documents/{doc_id}/pages/0/text")
|
|
assert resp.status_code == 200
|
|
text = resp.json()["text"]
|
|
assert "\u4e00\u4e2d\u4ed7" in text, "Failed to extract Traditional Chinese text"
|
|
|
|
# =========================================================================
|
|
# 6. UTF-8 corpus regression
|
|
# =========================================================================
|
|
|
|
=======
|
|
>>>>>>> 2638dfe969874b954502b26656df48ad971e306d
|
|
def test_utf8_corpus_regression():
|
|
doc_id = get_doc_id("utf-8.pdf")
|
|
resp = client.get(f"/documents/{doc_id}/pages/0/text")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert len(data["glyphs"]) > 0
|
|
assert len(data["text"]) > 0
|
|
|
|
|
|
<<<<<<< HEAD
|
|
# =========================================================================
|
|
# 7. Font size regression
|
|
# =========================================================================
|
|
|
|
=======
|
|
>>>>>>> 2638dfe969874b954502b26656df48ad971e306d
|
|
def test_font_size_regression():
|
|
doc_id = get_doc_id("utf-8.pdf")
|
|
resp = client.get(f"/documents/{doc_id}/pages/0/text")
|
|
data = resp.json()
|
|
sizes = set(g["fontSize"] for g in data["glyphs"])
|
|
assert len(sizes) >= 2 # utf-8.pdf should have multiple font sizes
|
|
<<<<<<< HEAD
|
|
|
|
|
|
# =========================================================================
|
|
# 8. ID uniqueness
|
|
# =========================================================================
|
|
|
|
def test_font_ids_unique_within_document():
|
|
"""Each font within a document must have a distinct internalFontId."""
|
|
for pdf in ("subset_font.pdf", "utf-8.pdf", "vertical_text.pdf"):
|
|
filepath = os.path.abspath(f"../corpus/fonts/{pdf}")
|
|
if not os.path.exists(filepath):
|
|
continue
|
|
|
|
doc_id = get_doc_id(pdf)
|
|
resp = client.get(f"/documents/{doc_id}/fonts")
|
|
fonts = resp.json()
|
|
|
|
ids = [f["internalFontId"] for f in fonts]
|
|
assert len(ids) == len(set(ids)), (
|
|
f"Duplicate internalFontId values in {pdf}: {ids}"
|
|
)
|
|
=======
|
|
>>>>>>> 2638dfe969874b954502b26656df48ad971e306d
|