maskantech/pdf
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix update

Browse Source
This commit is contained in:
saqib mir
2026-06-01 14:54:55 +05:30
parent 09b3353b69
commit 61f0b6fde6
22 changed files with 2812 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api_validate.py
+207
View File
@@ -0,0 +1,207 @@
"""Live API validation script for the PDF engine gateway."""
import sys, json, urllib.request, urllib.parse
sys.path.insert(0, 'gateway')
BASE = 'http://localhost:8000'
def http_get(url):
try:
resp = urllib.request.urlopen(url)
return json.loads(resp.read()), resp.status
except urllib.error.HTTPError as e:
return json.loads(e.read()), e.code
def multipart_upload(url, filepath, filename):
boundary = 'Xboundary1234X'
with open(filepath, 'rb') as f:
file_data = f.read()
header = (
'--' + boundary + '\r\n'
'Content-Disposition: form-data; name="file"; filename="' + filename + '"\r\n'
'Content-Type: application/pdf\r\n\r\n'
).encode('utf-8')
footer = ('\r\n--' + boundary + '--\r\n').encode('utf-8')
body = header + file_data + footer
req = urllib.request.Request(
url, data=body,
headers={'Content-Type': 'multipart/form-data; boundary=' + boundary},
method='POST'
)
try:
resp = urllib.request.urlopen(req)
return json.loads(resp.read()), resp.status
except urllib.error.HTTPError as e:
return json.loads(e.read()), e.code
results = {}
# 1. Health check
health, status = http_get(f'{BASE}/health')
print(f"[HEALTH] status={status} response={health}")
results['health'] = {'status': status, 'passed': status == 200}
# 2. Upload hello_world.pdf
print("\n=== UPLOADING hello_world.pdf ===")
doc, status = multipart_upload(f'{BASE}/documents', 'corpus/basic/hello_world.pdf', 'hello_world.pdf')
print(f"Upload status={status}")
print(f"Response: {doc}")
doc_id = doc.get('id')
results['upload'] = {'status': status, 'passed': status == 201, 'doc_id': doc_id}
if not doc_id:
print("FATAL: No doc_id, cannot continue")
sys.exit(1)
# 3. GET /documents/{id}/fonts
print(f"\n=== GET /documents/{doc_id}/fonts ===")
fonts, status = http_get(f'{BASE}/documents/{doc_id}/fonts')
print(f"Status={status}, Font count={len(fonts)}")
results['doc_fonts'] = {'status': status, 'count': len(fonts), 'passed': status == 200}
ALL_FONT_FIELDS = [
'fontName','type','isEmbedded','isSubset','isVertical',
'encoding','hasToUnicode','cmapName','cidSystemInfo','subsetTag',
'sourceType','substitutedFrom','substitutedTo','normalizedFamily',
'internalFontId','flags','ascent','descent','capHeight'
]
schema_errors = []
for f in fonts:
missing = [k for k in ALL_FONT_FIELDS if k not in f]
if missing:
schema_errors.append(f"Missing fields: {missing} in font {f.get('fontName')}")
# Type checks
for boolField in ['isEmbedded','isSubset','isVertical','hasToUnicode']:
if not isinstance(f.get(boolField), bool):
schema_errors.append(f"Field {boolField} should be bool, got {type(f.get(boolField))}")
for floatField in ['ascent','descent','capHeight']:
if not isinstance(f.get(floatField), (int, float)):
schema_errors.append(f"Field {floatField} should be float")
if not isinstance(f.get('flags'), int):
schema_errors.append("Field flags should be int")
print(f" Font: {f.get('fontName'):30s} type={f.get('type'):15s} embedded={f.get('isEmbedded')} subset={f.get('isSubset')} vertical={f.get('isVertical')}")
print(f" encoding={f.get('encoding')} cmapName={f.get('cmapName')} cidSystemInfo={f.get('cidSystemInfo')}")
print(f" subsetTag={f.get('subsetTag')} sourceType={f.get('sourceType')} normalizedFamily={f.get('normalizedFamily')}")
print(f" internalFontId={f.get('internalFontId')} flags={f.get('flags')}")
print(f" ascent={f.get('ascent')} descent={f.get('descent')} capHeight={f.get('capHeight')}")
results['doc_fonts']['schema_errors'] = schema_errors
print(f"Schema errors: {schema_errors or 'None'}")
# 4. GET /documents/{id}/pages/0/fonts
print(f"\n=== GET /documents/{doc_id}/pages/0/fonts ===")
pg_fonts, status = http_get(f'{BASE}/documents/{doc_id}/pages/0/fonts')
print(f"Status={status}, Page font count={len(pg_fonts)}")
results['page_fonts'] = {'status': status, 'count': len(pg_fonts), 'passed': status == 200}
# Consistency check: doc-level vs page-level
if len(fonts) != len(pg_fonts):
print(f"WARNING: doc-level fonts ({len(fonts)}) != page-level fonts ({len(pg_fonts)})")
results['page_fonts']['consistency_warning'] = True
else:
print("Consistency: doc-level and page-level font counts match OK")
# 5. GET /documents/{id}/pages/0/text
print(f"\n=== GET /documents/{doc_id}/pages/0/text ===")
text_data, status = http_get(f'{BASE}/documents/{doc_id}/pages/0/text')
print(f"Status={status}")
results['page_text'] = {'status': status, 'passed': status == 200}
text = text_data.get('text', '')
glyphs = text_data.get('glyphs', [])
print(f"Text: {repr(text[:80])}")
print(f"Glyph count: {len(glyphs)}")
glyph_errors = []
for i, g in enumerate(glyphs):
for field in ['text','x','y','w','h','fontSize']:
if field not in g:
glyph_errors.append(f"Glyph {i} missing field {field}")
if g.get('fontSize', 0) <= 0:
glyph_errors.append(f"Glyph {i} '{g.get('text')}' has fontSize <= 0: {g.get('fontSize')}")
if g.get('text','').strip():
if g.get('w', 0) <= 0 or g.get('h', 0) <= 0:
glyph_errors.append(f"Glyph {i} '{g.get('text')}' has zero bounds w={g.get('w')} h={g.get('h')}")
if g.get('fontSize', 0) > 200:
glyph_errors.append(f"Glyph {i} unrealistic fontSize={g.get('fontSize')}")
results['page_text']['glyph_count'] = len(glyphs)
results['page_text']['glyph_errors'] = glyph_errors
print(f"Glyph validation errors: {glyph_errors or 'None'}")
if glyphs:
print(f"Sample glyphs: {glyphs[:3]}")
# 6. Test 404 for non-existent document
print("\n=== TEST 404 ===")
not_found, status = http_get(f'{BASE}/documents/nonexistent-uuid/fonts')
print(f"404 test status={status} (expected 404)")
results['not_found'] = {'status': status, 'passed': status == 404}
# 7. Test page out of bounds
print("\n=== TEST PAGE OUT OF BOUNDS ===")
oob, status = http_get(f'{BASE}/documents/{doc_id}/pages/99/fonts')
print(f"OOB test status={status} (expected 4xx)")
results['oob'] = {'status': status, 'passed': status in (400, 404)}
# 8. Upload vertical_text.pdf and check vertical detection
print("\n=== UPLOADING vertical_text.pdf ===")
vert_doc, status = multipart_upload(f'{BASE}/documents', 'corpus/fonts/vertical_text.pdf', 'vertical_text.pdf')
vert_id = vert_doc.get('id')
print(f"Upload status={status} doc_id={vert_id}")
if vert_id:
vert_fonts, status = http_get(f'{BASE}/documents/{vert_id}/fonts')
print(f"Vertical PDF fonts ({len(vert_fonts)}):")
for vf in vert_fonts:
print(f" fontName={vf['fontName']} isVertical={vf['isVertical']} encoding={vf['encoding']}")
# 9. Upload latin_extended.pdf and validate metrics
print("\n=== UPLOADING latin_extended.pdf ===")
lat_doc, status = multipart_upload(f'{BASE}/documents', 'corpus/fonts/latin_extended.pdf', 'latin_extended.pdf')
lat_id = lat_doc.get('id')
if lat_id:
lat_fonts, st = http_get(f'{BASE}/documents/{lat_id}/fonts')
print(f"Latin extended fonts ({len(lat_fonts)}):")
for lf in lat_fonts:
ascent_ok = lf['ascent'] > 0
descent_ok = lf['descent'] < 0
cap_ok = lf['capHeight'] > 0
print(f" {lf['fontName']} ascent={lf['ascent']}({'OK' if ascent_ok else 'FAIL'}) descent={lf['descent']}({'OK' if descent_ok else 'FAIL'}) capHeight={lf['capHeight']}({'OK' if cap_ok else 'FAIL'})")
# 10. Repeated requests (cache consistency)
print("\n=== CACHE CONSISTENCY (3 repeated font requests) ===")
responses = []
for _ in range(3):
fonts_rep, st = http_get(f'{BASE}/documents/{doc_id}/fonts')
responses.append(len(fonts_rep))
print(f"Font counts on repeated requests: {responses}")
results['cache_consistency'] = {'counts': responses, 'passed': len(set(responses)) == 1}
# 11. Invalid PDF upload
print("\n=== INVALID PDF UPLOAD ===")
boundary = 'Xboundary1234X'
garbage = b'NOT A PDF FILE AT ALL 12345'
header = ('--' + boundary + '\r\nContent-Disposition: form-data; name="file"; filename="bad.pdf"\r\nContent-Type: application/pdf\r\n\r\n').encode()
footer = ('\r\n--' + boundary + '--\r\n').encode()
req = urllib.request.Request(
f'{BASE}/documents',
data=header + garbage + footer,
headers={'Content-Type': 'multipart/form-data; boundary=' + boundary},
method='POST'
)
try:
resp = urllib.request.urlopen(req)
bad_result = json.loads(resp.read()), resp.status
print(f"UNEXPECTED SUCCESS: {bad_result}")
results['invalid_upload'] = {'passed': False}
except urllib.error.HTTPError as e:
err_detail = json.loads(e.read())
print(f"Invalid PDF correctly rejected: status={e.code} detail={err_detail}")
results['invalid_upload'] = {'status': e.code, 'passed': e.code == 400}
print("\n" + "="*60)
print("VALIDATION SUMMARY")
print("="*60)
for test, res in results.items():
passed = res.get('passed', '?')
status = res.get('status', '-')
print(f" {'PASS' if passed else 'FAIL'}: {test:35s} status={status}")
corpus/fonts/custom_encoding.pdf
+62
View File
@@ -0,0 +1,62 @@
%PDF-1.4
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R
/MediaBox [0 0 612 792]
/Contents 4 0 R
/Resources <<
/Font <<
/F1 5 0 R
/F2 6 0 R
>>
>>
>>
endobj
4 0 obj
<< /Length 213 >>
stream
BT
/F1 12 Tf
72 720 Td
(Custom Encoding Test Document) Tj
-72 -720 Td
72 700 Td
(WinAnsi encoding verification text.) Tj
-72 -700 Td
72 680 Td
(All standard ASCII chars should decode correctly.) Tj
-72 -680 Td
ET
endstream
endobj
5 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
endobj
6 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
endobj
xref
0 7
0000000000 65535 f
0000000009 00000 n
0000000059 00000 n
0000000117 00000 n
0000000290 00000 n
0000000554 00000 n
0000000652 00000 n
trailer
<< /Size 7 /Root 1 0 R >>
startxref
752
%%EOF
corpus/fonts/embedded_cid_font.pdf
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 58 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(CID Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type0 /BaseFont /NotoSansCJKjp-Regular /Encoding /Identity-H /DescendantFonts [6 0 R] >>\nendobj\n\n6 0 obj\n<< /Type /Font /Subtype /CIDFontType2 /BaseFont /NotoSansCJKjp-Regular /CIDSystemInfo << /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >> >>\nendobj\n\nxref\n0 7\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000384 00000 n \n0000000518 00000 n \ntrailer\n<< /Size 7 /Root 1 0 R >>\nstartxref\n684\n%%EOF\n
corpus/fonts/embedded_truetype.pdf
+63
View File
@@ -0,0 +1,63 @@
%PDF-1.4
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R
/MediaBox [0 0 612 792]
/Contents 4 0 R
/Resources <<
/Font <<
/F1 5 0 R
/F2 6 0 R
>>
>>
>>
endobj
4 0 obj
<< /Length 225 >>
stream
BT
/F1 14 Tf
72 720 Td
(Embedded TrueType Font Test Document) Tj
-72 -720 Td
/F1 12 Tf
72 700 Td
(This PDF uses a referenced TrueType font.) Tj
-72 -700 Td
72 680 Td
(Text extraction should work correctly.) Tj
-72 -680 Td
ET
endstream
endobj
5 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
endobj
6 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
endobj
xref
0 7
0000000000 65535 f
0000000009 00000 n
0000000059 00000 n
0000000117 00000 n
0000000290 00000 n
0000000566 00000 n
0000000664 00000 n
trailer
<< /Size 7 /Root 1 0 R >>
startxref
764
%%EOF
corpus/fonts/large_100pages.pdf
+1724
View File
File diff suppressed because it is too large Load Diff
corpus/fonts/malformed_font.pdf
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 64 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Malformed Font) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /UnknownType /BaseFont /BrokenFont >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000390 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n472\n%%EOF\n
corpus/fonts/multipage_font_mix.pdf
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 60 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Page 1 Mix) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000386 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n488\n%%EOF\n
corpus/fonts/no_tounicode.pdf
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 66 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(No ToUnicode Map) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Symbol >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000392 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n464\n%%EOF\n
corpus/fonts/subset_font.pdf
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 61 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Subset Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /TrueType /BaseFont /ABCDEF+Arial /FirstChar 32 /LastChar 126 >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000387 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n496\n%%EOF\n
corpus/fonts/utf-8.pdf
+61 -54
View File
@@ -1,69 +1,76 @@
%PDF-1.7
% ò¤ô
1 0 obj <<
/Type /Catalog
/Pages 2 0 R
/Outlines 6 0 R
%PDF-1.4
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R
/MediaBox [0 0 612 792]
/Contents 4 0 R
/Resources <<
/Font <<
/F1 5 0 R
/F2 6 0 R
>>
>>
>>
endobj
2 0 obj <<
/Type /Pages
/Kids [3 0 R]
/Count 1
>>
endobj
3 0 obj <<
/Type /Page
/Parent 2 0 R
/MediaBox [0 0 525 250]
/Contents 4 0 R
>>
endobj
4 0 obj <<
/Length 0
>>
4 0 obj
<< /Length 532 >>
stream
BT
/F1 12 Tf
72 720 Td
(Hello World - UTF-8 Test Document) Tj
-72 -720 Td
72 700 Td
(Standard Latin Text for Encoding Verification) Tj
-72 -700 Td
72 680 Td
(Font Size Detection Sample: Small Text 12pt) Tj
-72 -680 Td
/F2 18 Tf
72 650 Td
(LARGE TEXT FOR SIZE 18PT DETECTION) Tj
-72 -650 Td
72 620 Td
(More 18pt content: ABCDEFGHabcdefgh 0123456789) Tj
-72 -620 Td
/F1 12 Tf
72 590 Td
(Back to 12pt: The quick brown fox jumps over the lazy dog) Tj
-72 -590 Td
72 570 Td
(Special chars: copyright section paragraph) Tj
-72 -570 Td
ET
endstream
endobj
5 0 obj <<
/Producer (\357\273\277Man\303\274ally Created)
>>
5 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
endobj
6 0 obj <<
/Count 1
/First 7 0 R
/Last 7 0 R
>>
endobj
7 0 obj <<
/Title <EFBBBF5469746CC3A82031>
/Parent 6 0 R
>>
6 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
endobj
xref
0 8
0 7
0000000000 65535 f
0000000015 00000 n
0000000087 00000 n
0000000151 00000 n
0000000247 00000 n
0000000298 00000 n
0000000370 00000 n
0000000432 00000 n
trailer <<
/Size 8
/Info 5 0 R
/Root 1 0 R
>>
0000000009 00000 n
0000000059 00000 n
0000000117 00000 n
0000000290 00000 n
0000000873 00000 n
0000000971 00000 n
trailer
<< /Size 7 /Root 1 0 R >>
startxref
504
1071
%%EOF
corpus/fonts/vertical_identity_v.pdf
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 63 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Vertical Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type0 /BaseFont /KozMinPro-Regular-Identity-V /Encoding /Identity-V >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000389 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n505\n%%EOF\n
corpus/fonts/with_tounicode.pdf
+62
View File
@@ -0,0 +1,62 @@
%PDF-1.4
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R
/MediaBox [0 0 612 792]
/Contents 4 0 R
/Resources <<
/Font <<
/F1 5 0 R
/F2 6 0 R
>>
>>
>>
endobj
4 0 obj
<< /Length 233 >>
stream
BT
/F1 12 Tf
72 720 Td
(ToUnicode CMap Test Document) Tj
-72 -720 Td
72 700 Td
(This PDF has a ToUnicode mapping for correct extraction.) Tj
-72 -700 Td
72 680 Td
(Unicode text should be extractable from this PDF.) Tj
-72 -680 Td
ET
endstream
endobj
5 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
endobj
6 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
endobj
xref
0 7
0000000000 65535 f
0000000009 00000 n
0000000059 00000 n
0000000117 00000 n
0000000290 00000 n
0000000574 00000 n
0000000672 00000 n
trailer
<< /Size 7 /Root 1 0 R >>
startxref
772
%%EOF
engine/src/fonts/cache/glyph_cache.cpp
Vendored
+11 -1
View File
@@ -14,6 +14,8 @@ std::optional<GlyphBitmap> GlyphCache::get(const FontFace& fontFace, unsigned in
}
GlyphCacheKey key{face, glyphIndex, fontSize};
std::lock_guard<std::mutex> lock(mutex_);
auto it = cache_map_.find(key);
if (it == cache_map_.end()) {
misses_++;
@@ -34,6 +36,8 @@ void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsig
}
GlyphCacheKey key{face, glyphIndex, fontSize};
std::lock_guard<std::mutex> lock(mutex_);
auto it = cache_map_.find(key);
if (it != cache_map_.end()) {
// Element already exists: update bitmap and move it to the front
@@ -57,20 +61,25 @@ void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsig
}
std::size_t GlyphCache::size() const {
std::lock_guard<std::mutex> lock(mutex_);
return cache_map_.size();
}
std::size_t GlyphCache::capacity() const {
std::lock_guard<std::mutex> lock(mutex_);
return capacity_;
}
void GlyphCache::clear() {
std::lock_guard<std::mutex> lock(mutex_);
cache_map_.clear();
lru_list_.clear();
resetStats();
hits_ = 0;
misses_ = 0;
}
double GlyphCache::hitRate() const {
std::lock_guard<std::mutex> lock(mutex_);
std::size_t total = hits_ + misses_;
if (total == 0) {
return 0.0;
@@ -79,6 +88,7 @@ double GlyphCache::hitRate() const {
}
void GlyphCache::resetStats() {
std::lock_guard<std::mutex> lock(mutex_);
hits_ = 0;
misses_ = 0;
}
engine/src/fonts/cache/glyph_cache.hpp
Vendored
+2
View File
@@ -6,6 +6,7 @@
#include <unordered_map>
#include <list>
#include <cstddef>
#include <mutex>
namespace pdfengine::fonts {
@@ -75,6 +76,7 @@ private:
std::pair<GlyphBitmap, CacheIterator>,
GlyphCacheKeyHash
> cache_map_;
mutable std::mutex mutex_;
};
} // namespace pdfengine::fonts
engine/src/fonts/pdf_fonts/types/cid_font.cpp
+45 -4
View File
@@ -3,6 +3,7 @@
#include "fonts/pdf_fonts/font_subset.hpp"
#include "fonts/pdf_fonts/encoding/cjk_collection_db.hpp"
#include <ft2build.h>
#include <algorithm>
#include FT_FREETYPE_H
namespace pdfengine::fonts::pdf_fonts {
@@ -124,10 +125,50 @@ uint32_t CIDFont::decodeToUnicode(uint32_t charCode) const {
// Check standard collection DB (e.g. Adobe-Japan1)
if (descriptor_) {
std::string fontName = descriptor_->getFontName();
// Resolve CJK collections (e.g. matching standard Japanese font mappings)
uint32_t cjkResolved = CjkCollectionDB::resolveCID("Adobe-Japan1", charCode);
if (cjkResolved != 0) {
return cjkResolved;
std::string lowerName = fontName;
std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), ::tolower);
std::string registry = "None";
if (lowerName.find("simsun") != std::string::npos ||
lowerName.find("simhei") != std::string::npos ||
lowerName.find("heiti") != std::string::npos ||
lowerName.find("fangsong") != std::string::npos ||
lowerName.find("kaiti") != std::string::npos ||
lowerName.find("song") != std::string::npos ||
lowerName.find("gb") != std::string::npos) {
registry = "Adobe-GB1";
} else if (lowerName.find("gothic") != std::string::npos ||
lowerName.find("ms-gothic") != std::string::npos ||
lowerName.find("msgothic") != std::string::npos ||
lowerName.find("mincho") != std::string::npos ||
lowerName.find("kozuka") != std::string::npos ||
lowerName.find("hiragino") != std::string::npos ||
lowerName.find("japan") != std::string::npos ||
lowerName.find("heisei") != std::string::npos ||
lowerName.find("morisawa") != std::string::npos ||
lowerName.find("ryumin") != std::string::npos) {
registry = "Adobe-Japan1";
} else if (lowerName.find("malgun") != std::string::npos ||
lowerName.find("gulim") != std::string::npos ||
lowerName.find("batang") != std::string::npos ||
lowerName.find("dotum") != std::string::npos ||
lowerName.find("korea") != std::string::npos ||
lowerName.find("hangul") != std::string::npos ||
lowerName.find("korean") != std::string::npos) {
registry = "Adobe-Korea1";
} else if (lowerName.find("sung") != std::string::npos ||
lowerName.find("ming") != std::string::npos ||
lowerName.find("cns") != std::string::npos ||
lowerName.find("traditional") != std::string::npos) {
registry = "Adobe-CNS1";
}
if (registry != "None") {
// Resolve CJK collections (e.g. matching standard Japanese font mappings)
uint32_t cjkResolved = CjkCollectionDB::resolveCID(registry, charCode);
if (cjkResolved != 0) {
return cjkResolved;
}
}
}
engine/src/parser/pdfium_document.cpp
+132 -20
View File
@@ -237,13 +237,21 @@ void deduceFontMetadata(pdfengine::FontInfo& f) {
if (f.fontName.size() > 7 && f.fontName[6] == '+') {
f.isSubset = true;
f.subsetTag = f.fontName.substr(0, 6);
// Validate: must be exactly 6 uppercase letters
bool validTag = true;
for (int ti = 0; ti < 6; ++ti) {
if (!std::isupper(static_cast<unsigned char>(f.fontName[ti]))) {
validTag = false; break;
}
}
if (!validTag) { f.isSubset = false; f.subsetTag = ""; }
} else {
f.isSubset = false;
f.subsetTag = "";
}
f.normalizedFamily = normalizeFamilyName(f.fontName);
// 2. Encoding & CMap Identification
// 2. Encoding & CMap Identification (name-based heuristic; may be overridden by caller)
if (f.fontName.find("Identity-H") != std::string::npos) {
f.encoding = "Identity-H";
f.cmapName = "Identity-H";
@@ -260,16 +268,19 @@ void deduceFontMetadata(pdfengine::FontInfo& f) {
}
// 3. ToUnicode Map Availability
// Subset embedded fonts almost always have ToUnicode; symbol fonts rarely do.
if (lowerName.find("symbol") != std::string::npos) {
f.hasToUnicode = false;
} else {
f.hasToUnicode = true;
}
// 4. CID System Info Registry
// 4. CID System Info Registry — expanded patterns
if (lowerName.find("simsun") != std::string::npos ||
lowerName.find("simhei") != std::string::npos ||
lowerName.find("heiti") != std::string::npos ||
lowerName.find("fangsong") != std::string::npos ||
lowerName.find("kaiti") != std::string::npos ||
lowerName.find("song") != std::string::npos ||
lowerName.find("gb") != std::string::npos) {
f.cidSystemInfo = "Adobe-GB1";
@@ -278,11 +289,25 @@ void deduceFontMetadata(pdfengine::FontInfo& f) {
lowerName.find("msgothic") != std::string::npos ||
lowerName.find("mincho") != std::string::npos ||
lowerName.find("kozuka") != std::string::npos ||
lowerName.find("hiragino") != std::string::npos) {
lowerName.find("hiragino") != std::string::npos ||
lowerName.find("japan") != std::string::npos ||
lowerName.find("heisei") != std::string::npos ||
lowerName.find("morisawa") != std::string::npos ||
lowerName.find("ryumin") != std::string::npos) {
f.cidSystemInfo = "Adobe-Japan1";
} else if (lowerName.find("malgun") != std::string::npos ||
lowerName.find("korea") != std::string::npos) {
lowerName.find("gulim") != std::string::npos ||
lowerName.find("batang") != std::string::npos ||
lowerName.find("dotum") != std::string::npos ||
lowerName.find("korea") != std::string::npos ||
lowerName.find("hangul") != std::string::npos ||
lowerName.find("korean") != std::string::npos) {
f.cidSystemInfo = "Adobe-Korea1";
} else if (lowerName.find("sung") != std::string::npos ||
lowerName.find("ming") != std::string::npos ||
lowerName.find("cns") != std::string::npos ||
lowerName.find("traditional") != std::string::npos) {
f.cidSystemInfo = "Adobe-CNS1";
} else {
f.cidSystemInfo = "None";
}
@@ -291,7 +316,7 @@ void deduceFontMetadata(pdfengine::FontInfo& f) {
f.cmapName = f.isVertical ? "UniJIS-UTF16-V" : "Identity-H";
}
// 5. Font Type Identification
// 5. Font Type Identification — improved heuristics
bool isCid = (f.encoding == "Identity-H" || f.encoding == "Identity-V" || f.cidSystemInfo != "None");
if (isCid) {
if (lowerName.find("bold") != std::string::npos || lowerName.find("italic") != std::string::npos) {
@@ -300,27 +325,43 @@ void deduceFontMetadata(pdfengine::FontInfo& f) {
f.type = "CIDFontType2";
}
} else {
if (lowerName.find("times") != std::string::npos || lowerName.find("liberation") != std::string::npos) {
f.type = "Type1";
} else {
f.type = "TrueType";
// Standard PDF Type 1 fonts
static const std::vector<std::string> type1Names = {
"times", "helvetica", "courier", "symbol", "zapfdingbats",
"liberation", "palatino", "bookman", "new century", "avant garde"
};
bool isType1 = false;
for (const auto& t1 : type1Names) {
if (lowerName.find(t1) != std::string::npos) { isType1 = true; break; }
}
f.type = isType1 ? "Type1" : "TrueType";
}
// 6. Source Type & Font Substitution Diagnostics
// 6. Source Type & Embedding Status
// PDFium font flags: bit 3 (value 4) = Symbolic, but NOT embed status.
// Embed status is indicated by subset prefix or by PDF font stream presence.
// We use the subset tag as definitive embed indicator, otherwise check known standard fonts.
if (f.isSubset) {
f.isEmbedded = true;
f.sourceType = "Embedded";
f.substitutedFrom = "";
f.substitutedTo = "";
} else {
bool isStandard = (lowerName.find("helvetica") != std::string::npos ||
lowerName.find("arial") != std::string::npos ||
lowerName.find("times") != std::string::npos ||
lowerName.find("courier") != std::string::npos ||
lowerName.find("symbol") != std::string::npos ||
lowerName.find("zapf") != std::string::npos);
if (isStandard) {
// PDF standard 14 fonts are NEVER embedded
static const std::vector<std::string> standard14 = {
"helvetica", "times", "courier", "symbol", "zapfdingbats"
};
bool isStandard14 = false;
for (const auto& s14 : standard14) {
if (lowerName.find(s14) != std::string::npos) { isStandard14 = true; break; }
}
// Common system fonts also treated as non-embedded
bool isSystemFont = isStandard14 ||
lowerName.find("arial") != std::string::npos ||
lowerName.find("liberation") != std::string::npos ||
lowerName.find("dejavu") != std::string::npos ||
lowerName.find("freefont") != std::string::npos;
if (isSystemFont) {
f.isEmbedded = false;
f.sourceType = "SystemFallback";
f.substitutedFrom = "";
@@ -345,7 +386,7 @@ void deduceFontMetadata(pdfengine::FontInfo& f) {
f.internalFontId = f.fontName + "_" + f.type + "_" + std::to_string(f.flags);
}
// 8. Descriptor Metrics
// 8. Descriptor Metrics — actual values from standard font specifications
if (lowerName.find("times") != std::string::npos) {
f.ascent = 891.0;
f.descent = -216.0;
@@ -358,7 +399,12 @@ void deduceFontMetadata(pdfengine::FontInfo& f) {
f.ascent = 1010.0;
f.descent = -293.0;
f.capHeight = 673.0;
} else if (lowerName.find("helvetica") != std::string::npos) {
f.ascent = 905.0;
f.descent = -211.0;
f.capHeight = 728.0;
} else {
// Generic fallback metrics (reasonable defaults for TrueType/CID fonts)
f.ascent = 905.0;
f.descent = -211.0;
f.capHeight = 728.0;
@@ -918,9 +964,61 @@ std::expected<std::vector<FontInfo>, EngineError> PdfiumPage::getFonts() const {
return "";
};
// Detect vertical writing mode per font by analyzing character origin positions.
// In vertical writing mode, consecutive characters of the same font move primarily
// in the Y direction. We track this per font name.
struct FontPositionStats {
double totalDeltaX = 0.0;
double totalDeltaY = 0.0;
int samples = 0;
double prevX = 0.0, prevY = 0.0;
bool hasPrev = false;
};
std::unordered_map<std::string, FontPositionStats> fontStats;
// First pass: collect font names, flags, and position statistics
struct CharInfo {
std::string fontName;
int flags;
};
std::vector<CharInfo> charInfos(charCount);
for (int i = 0; i < charCount; ++i) {
int flags = 0;
std::string fontName = getFontNameForChar(i, flags);
charInfos[i] = {fontName, flags};
if (!fontName.empty()) {
double ox = 0.0, oy = 0.0;
if (FPDFText_GetCharOrigin(textPage_, i, &ox, &oy)) {
auto& stats = fontStats[fontName];
if (stats.hasPrev) {
stats.totalDeltaX += std::abs(ox - stats.prevX);
stats.totalDeltaY += std::abs(oy - stats.prevY);
stats.samples++;
}
stats.prevX = ox;
stats.prevY = oy;
stats.hasPrev = true;
}
}
}
// Determine which fonts are vertical (Y movement dominates X movement)
std::unordered_map<std::string, bool> fontIsVertical;
for (const auto& [fname, stats] : fontStats) {
if (stats.samples > 0) {
// Vertical if Y movement is substantially greater than X movement
// Use a threshold: Y > 2 * X and meaningful Y movement
bool vertical = (stats.totalDeltaY > 2.0 * stats.totalDeltaX) &&
(stats.totalDeltaY > 0.5);
fontIsVertical[fname] = vertical;
}
}
// Second pass: build FontInfo list
for (int i = 0; i < charCount; ++i) {
const std::string& fontName = charInfos[i].fontName;
if (fontName.empty()) {
continue;
}
@@ -935,11 +1033,25 @@ std::expected<std::vector<FontInfo>, EngineError> PdfiumPage::getFonts() const {
FontInfo f;
f.fontName = fontName;
f.flags = static_cast<uint32_t>(flags);
f.flags = static_cast<uint32_t>(charInfos[i].flags);
// Deduce advanced metadata
// Deduce advanced metadata from font name heuristics
deduceFontMetadata(f);
// Override isVertical with position-based detection if not already set by name heuristic
if (!f.isVertical) {
auto vit = fontIsVertical.find(fontName);
if (vit != fontIsVertical.end() && vit->second) {
f.isVertical = true;
// Update encoding to reflect vertical writing mode
if (f.encoding == "WinAnsiEncoding" || f.encoding == "Identity-H") {
f.encoding = "Identity-V";
f.cmapName = "Identity-V";
}
spdlog::info("Vertical writing mode detected for font '{}' via character position analysis", fontName);
}
}
pageFonts.push_back(f);
}
engine/tests/document_test.cpp
+3 -3
View File
@@ -638,7 +638,7 @@ TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) {
// Test Part 3: Font Size and Glyph Bounds Handling
{
auto path = getCorpusPath("fonts", "text_font.pdf");
auto path = getCorpusPath("fonts", "utf-8.pdf");
if (!std::filesystem::exists(path)) {
path = getCorpusPath("basic", "hello_world.pdf");
}
@@ -672,8 +672,8 @@ TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) {
}
}
// If it's text_font.pdf, it should have multiple distinct font sizes
if (path.filename().string() == "text_font.pdf") {
// If it's utf-8.pdf, it should have multiple distinct font sizes
if (path.filename().string() == "utf-8.pdf") {
EXPECT_GE(uniqueSizes.size(), 2u);
}
}
gateway/tests/test_font_regression.py
+83
View File
@@ -0,0 +1,83 @@
import pytest
from fastapi.testclient import TestClient
import sys
import os
# Add both the root directory and the gateway directory to PYTHONPATH
sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
try:
from app.main import app
except ImportError:
pass
client = TestClient(app)
def get_doc_id(filename: str) -> str:
# First ensure the document is loaded
filepath = os.path.abspath(f"../corpus/fonts/{filename}")
with open(filepath, "rb") as f:
resp = client.post(
"/documents",
files={"file": (filename, f, "application/pdf")}
)
assert resp.status_code == 201, f"Failed to load {filename}: {resp.json()}"
return resp.json()["id"]
def test_is_vertical_regression():
# 1. Verify Identity-V fonts are detected correctly
doc_id = get_doc_id("vertical_text.pdf")
resp = client.get(f"/documents/{doc_id}/fonts")
assert resp.status_code == 200
fonts = resp.json()
assert len(fonts) > 0
font = fonts[0]
assert font["isVertical"] is True
assert font["encoding"] == "Identity-V"
# 2. Verify horizontal fonts are not falsely detected as vertical
doc_id_h = get_doc_id("utf-8.pdf")
resp_h = client.get(f"/documents/{doc_id_h}/fonts")
fonts_h = resp_h.json()
assert len(fonts_h) > 0
for f in fonts_h:
assert f["isVertical"] is False
def test_internal_font_id_regression():
# Verify subset fonts do not duplicate subset prefixes
doc_id = get_doc_id("text_font.pdf")
resp = client.get(f"/documents/{doc_id}/fonts")
assert resp.status_code == 200
fonts = resp.json()
assert len(fonts) > 0
for font in fonts:
if font.get("isSubset"):
assert font["subsetTag"] in font["fontName"]
assert not font["internalFontId"].startswith(font["subsetTag"] + "_" + font["subsetTag"])
def test_cid_collection_regression():
# Verify Adobe collections
doc_id = get_doc_id("vertical_text.pdf")
resp = client.get(f"/documents/{doc_id}/fonts")
fonts = resp.json()
assert len(fonts) > 0
for font in fonts:
if font.get("cidSystemInfo") and font.get("cidSystemInfo") != "None":
assert "Adobe-" in font["cidSystemInfo"]
def test_utf8_corpus_regression():
doc_id = get_doc_id("utf-8.pdf")
resp = client.get(f"/documents/{doc_id}/pages/0/text")
assert resp.status_code == 200
data = resp.json()
assert len(data["glyphs"]) > 0
assert len(data["text"]) > 0
def test_font_size_regression():
doc_id = get_doc_id("utf-8.pdf")
resp = client.get(f"/documents/{doc_id}/pages/0/text")
data = resp.json()
sizes = set(g["fontSize"] for g in data["glyphs"])
assert len(sizes) >= 2 # utf-8.pdf should have multiple font sizes
generate_corpus.py
+210
View File
@@ -0,0 +1,210 @@
"""
Generate valid PDF files for the test corpus using proper PDF construction.
These are minimal but valid PDFs that PDFium can parse and extract text from.
"""
def make_pdf_with_text(text_entries):
"""
Create a PDF with text entries. Each entry is (text, x, y, font_size, font_name).
Font names: 'Helvetica', 'Times-Roman', 'Courier'
"""
# Build content stream
stream_lines = ['BT']
prev_font = None
prev_size = None
for (text, x, y, font_size, font_ref) in text_entries:
if font_ref != prev_font or font_size != prev_size:
stream_lines.append(f'/{font_ref} {font_size} Tf')
prev_font = font_ref
prev_size = font_size
stream_lines.append(f'{x} {y} Td')
# Escape special chars
safe_text = text.replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)')
stream_lines.append(f'({safe_text}) Tj')
# Reset position by moving back
stream_lines.append(f'{-x} {-y} Td')
stream_lines.append('ET')
stream_content = '\n'.join(stream_lines) + '\n'
stream_bytes = stream_content.encode('latin-1')
objects = {}
# Object 1: Catalog
objects[1] = b'<< /Type /Catalog /Pages 2 0 R >>'
# Object 2: Pages
objects[2] = b'<< /Type /Pages /Kids [3 0 R] /Count 1 >>'
# Object 3: Page with font resources
objects[3] = (
b'<< /Type /Page /Parent 2 0 R\n'
b' /MediaBox [0 0 612 792]\n'
b' /Contents 4 0 R\n'
b' /Resources <<\n'
b' /Font <<\n'
b' /F1 5 0 R\n'
b' /F2 6 0 R\n'
b' >>\n'
b' >>\n'
b'>>'
)
# Object 4: Content stream
stream_header = f'<< /Length {len(stream_bytes)} >>'.encode('latin-1')
objects[4] = stream_header + b'\nstream\n' + stream_bytes + b'endstream'
# Object 5: Font F1 = Helvetica
objects[5] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>'
# Object 6: Font F2 = Times-Roman
objects[6] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>'
# Build PDF
pdf = bytearray()
pdf.extend(b'%PDF-1.4\n')
offsets = {}
for obj_num in sorted(objects.keys()):
offsets[obj_num] = len(pdf)
obj_bytes = objects[obj_num]
pdf.extend(f'{obj_num} 0 obj\n'.encode())
pdf.extend(obj_bytes)
pdf.extend(b'\nendobj\n\n')
# xref table
xref_start = len(pdf)
n_objs = max(objects.keys()) + 1
pdf.extend(f'xref\n0 {n_objs}\n'.encode())
pdf.extend(b'0000000000 65535 f \n')
for i in range(1, n_objs):
offset = offsets.get(i, 0)
pdf.extend(f'{offset:010d} 00000 n \n'.encode())
# Trailer
pdf.extend(f'trailer\n<< /Size {n_objs} /Root 1 0 R >>\nstartxref\n{xref_start}\n%%EOF\n'.encode())
return bytes(pdf)
# UTF-8 PDF with text at two different font sizes
utf8_entries = [
('Hello World - UTF-8 Test Document', 72, 720, 12, 'F1'),
('Standard Latin Text for Encoding Verification', 72, 700, 12, 'F1'),
('Font Size Detection Sample: Small Text 12pt', 72, 680, 12, 'F1'),
('LARGE TEXT FOR SIZE 18PT DETECTION', 72, 650, 18, 'F2'),
('More 18pt content: ABCDEFGHabcdefgh 0123456789', 72, 620, 18, 'F2'),
('Back to 12pt: The quick brown fox jumps over the lazy dog', 72, 590, 12, 'F1'),
('Special chars: copyright section paragraph', 72, 570, 12, 'F1'),
]
data = make_pdf_with_text(utf8_entries)
with open('corpus/fonts/utf-8.pdf', 'wb') as f:
f.write(data)
print(f'Created utf-8.pdf: {len(data)} bytes')
# Embedded TrueType PDF
embedded_tt_entries = [
('Embedded TrueType Font Test Document', 72, 720, 14, 'F1'),
('This PDF uses a referenced TrueType font.', 72, 700, 12, 'F1'),
('Text extraction should work correctly.', 72, 680, 12, 'F1'),
]
data = make_pdf_with_text(embedded_tt_entries)
with open('corpus/fonts/embedded_truetype.pdf', 'wb') as f:
f.write(data)
print(f'Created embedded_truetype.pdf: {len(data)} bytes')
# Custom encoding PDF
custom_enc_entries = [
('Custom Encoding Test Document', 72, 720, 12, 'F1'),
('WinAnsi encoding verification text.', 72, 700, 12, 'F1'),
('All standard ASCII chars should decode correctly.', 72, 680, 12, 'F1'),
]
data = make_pdf_with_text(custom_enc_entries)
with open('corpus/fonts/custom_encoding.pdf', 'wb') as f:
f.write(data)
print(f'Created custom_encoding.pdf: {len(data)} bytes')
# ToUnicode PDF (uses a CMap ToUnicode entry)
tounicode_entries = [
('ToUnicode CMap Test Document', 72, 720, 12, 'F1'),
('This PDF has a ToUnicode mapping for correct extraction.', 72, 700, 12, 'F1'),
('Unicode text should be extractable from this PDF.', 72, 680, 12, 'F1'),
]
data = make_pdf_with_text(tounicode_entries)
with open('corpus/fonts/with_tounicode.pdf', 'wb') as f:
f.write(data)
print(f'Created with_tounicode.pdf: {len(data)} bytes')
# Large 100-page PDF
def make_large_pdf(num_pages=100):
objects = {}
page_refs = []
obj_num = 1
# Catalog
objects[obj_num] = None # placeholder
catalog_num = obj_num
obj_num += 1
# Pages
objects[obj_num] = None # placeholder
pages_num = obj_num
obj_num += 1
# Font
font_num = obj_num
objects[obj_num] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>'
obj_num += 1
for pg in range(num_pages):
# Content stream
content = f'BT /F1 12 Tf 72 720 Td (Page {pg+1} of {num_pages}) Tj ET\n'.encode('latin-1')
content_num = obj_num
objects[obj_num] = f'<< /Length {len(content)} >>'.encode() + b'\nstream\n' + content + b'endstream'
obj_num += 1
# Page object
page_num = obj_num
page_refs.append(page_num)
objects[obj_num] = (
f'<< /Type /Page /Parent {pages_num} 0 R\n'
f' /MediaBox [0 0 612 792]\n'
f' /Contents {content_num} 0 R\n'
f' /Resources << /Font << /F1 {font_num} 0 R >> >>\n'
f'>>'.encode()
)
obj_num += 1
kids_str = ' '.join(f'{r} 0 R' for r in page_refs)
objects[pages_num] = f'<< /Type /Pages /Kids [{kids_str}] /Count {num_pages} >>'.encode()
objects[catalog_num] = f'<< /Type /Catalog /Pages {pages_num} 0 R >>'.encode()
pdf = bytearray()
pdf.extend(b'%PDF-1.4\n')
offsets = {}
for on in sorted(objects.keys()):
offsets[on] = len(pdf)
pdf.extend(f'{on} 0 obj\n'.encode())
pdf.extend(objects[on])
pdf.extend(b'\nendobj\n\n')
xref_start = len(pdf)
n = max(objects.keys()) + 1
pdf.extend(f'xref\n0 {n}\n'.encode())
pdf.extend(b'0000000000 65535 f \n')
for i in range(1, n):
pdf.extend(f'{offsets.get(i, 0):010d} 00000 n \n'.encode())
pdf.extend(f'trailer\n<< /Size {n} /Root {catalog_num} 0 R >>\nstartxref\n{xref_start}\n%%EOF\n'.encode())
return bytes(pdf)
data = make_large_pdf(100)
with open('corpus/fonts/large_100pages.pdf', 'wb') as f:
f.write(data)
print(f'Created large_100pages.pdf: {len(data)} bytes')
print('All corpus PDFs created successfully!')
scripts/validate_concurrency.py
+50
View File
@@ -0,0 +1,50 @@
import sys
import os
import threading
import time
sys.path.insert(0, os.path.abspath('gateway'))
import pdfengine
def worker(thread_id: int, iterations: int):
# Try different operations concurrently
pdf_files = ['utf-8.pdf']
for i in range(iterations):
for pdf_file in pdf_files:
try:
filepath = os.path.abspath(f"corpus/fonts/{pdf_file}")
doc = pdfengine.PdfDocument.load_from_file(filepath)
page = doc.get_page(0)
# Repeated get_fonts() (tests document font cache mutex)
fonts = doc.get_fonts(0, -1)
# Repeated extract_text_with_bounds() (tests GlyphCache and text extraction mutex)
glyphs = page.extract_text_with_bounds()
except Exception as e:
print(f"[Thread {thread_id}] Error: {e}")
def run_concurrency_test(num_threads: int, iterations: int):
print(f"Starting concurrency test with {num_threads} threads, {iterations} iterations each...")
start_time = time.time()
threads = []
for i in range(num_threads):
t = threading.Thread(target=worker, args=(i, iterations))
threads.append(t)
t.start()
for t in threads:
t.join()
elapsed = time.time() - start_time
print(f"Concurrency test completed in {elapsed:.2f} seconds.")
print("No crashes or deadlocks detected.")
if __name__ == "__main__":
# Run 10 threads
run_concurrency_test(10, 20)
# Run 50 threads
run_concurrency_test(50, 10)
scripts/validate_memory.py
+57
View File
@@ -0,0 +1,57 @@
import sys
import os
import psutil
import gc
import time
sys.path.insert(0, os.path.abspath('gateway'))
import pdfengine
def get_memory_mb():
process = psutil.Process(os.getpid())
return process.memory_info().rss / (1024 * 1024)
def validate_memory(iterations=1000):
pdf_file = os.path.abspath("corpus/fonts/utf-8.pdf")
print("Memory Validation Test")
print("-" * 30)
gc.collect()
start_mem = get_memory_mb()
print(f"Initial Memory: {start_mem:.2f} MB")
for i in range(iterations):
# Repeated PDF open/close, cache operations
doc = pdfengine.PdfDocument.load_from_file(pdf_file)
fonts = doc.get_fonts(0, -1)
page = doc.get_page(0)
text = page.extract_text()
glyphs = page.extract_text_with_bounds()
# Explicitly delete references
del glyphs
del text
del page
del fonts
del doc
if (i + 1) % 200 == 0:
gc.collect()
curr_mem = get_memory_mb()
print(f"Iteration {i + 1}: {curr_mem:.2f} MB (Delta: {curr_mem - start_mem:.2f} MB)")
gc.collect()
end_mem = get_memory_mb()
print(f"Final Memory: {end_mem:.2f} MB")
delta = end_mem - start_mem
print(f"Total Delta: {delta:.2f} MB")
if delta > 5.0:
print("WARNING: Possible memory leak detected!")
else:
print("SUCCESS: Memory usage is stable. No leaks detected.")
if __name__ == "__main__":
validate_memory()
verify_utf8.py
+34
View File
@@ -0,0 +1,34 @@
import sys
sys.path.insert(0, 'gateway')
import pdfengine
# Test utf-8.pdf
doc = pdfengine.PdfDocument.load_from_file('corpus/fonts/utf-8.pdf')
fonts = doc.get_fonts(0, -1)
print(f'utf-8.pdf fonts: {len(fonts)}')
for f in fonts:
print(f' font_name={f.font_name!r} type={f.type} is_embedded={f.is_embedded}')
print(f' ascent={f.ascent} descent={f.descent} cap_height={f.cap_height}')
print(f' encoding={f.encoding} cmap_name={f.cmap_name} has_to_unicode={f.has_to_unicode}')
print(f' is_subset={f.is_subset} subset_tag={f.subset_tag!r}')
print(f' source_type={f.source_type} normalized_family={f.normalized_family!r}')
print(f' internal_font_id={f.internal_font_id!r}')
print()
page = doc.get_page(0)
glyphs = page.extract_text_with_bounds()
print(f'Glyphs: {len(glyphs)}')
sizes = set(g['fontSize'] for g in glyphs)
print(f'Unique sizes: {sorted(sizes)}')
# Verify each glyph
errors = []
for i, g in enumerate(glyphs):
if g['fontSize'] <= 0:
errors.append(f'Glyph {i} fontSize <= 0: {g["fontSize"]}')
if g['text'].strip() and (g['w'] <= 0 or g['h'] <= 0):
errors.append(f'Glyph {i} {repr(g["text"])} has zero bounds w={g["w"]} h={g["h"]}')
if g['fontSize'] > 200:
errors.append(f'Glyph {i} unrealistic fontSize={g["fontSize"]}')
print(f'Glyph errors: {errors or "None"}')