Merge pull request 'saqib' (#26) from saqib into dev

Reviewed-on: https://gitea.maskantech.in/gitea_admin/pdf/pulls/26
This commit is contained in:
furqan
2026-06-01 09:28:46 +00:00
42 changed files with 4087 additions and 79 deletions
+207
View File
@@ -0,0 +1,207 @@
"""Live API validation script for the PDF engine gateway."""
import sys, json, urllib.request, urllib.parse
sys.path.insert(0, 'gateway')
BASE = 'http://localhost:8000'
def http_get(url):
try:
resp = urllib.request.urlopen(url)
return json.loads(resp.read()), resp.status
except urllib.error.HTTPError as e:
return json.loads(e.read()), e.code
def multipart_upload(url, filepath, filename):
boundary = 'Xboundary1234X'
with open(filepath, 'rb') as f:
file_data = f.read()
header = (
'--' + boundary + '\r\n'
'Content-Disposition: form-data; name="file"; filename="' + filename + '"\r\n'
'Content-Type: application/pdf\r\n\r\n'
).encode('utf-8')
footer = ('\r\n--' + boundary + '--\r\n').encode('utf-8')
body = header + file_data + footer
req = urllib.request.Request(
url, data=body,
headers={'Content-Type': 'multipart/form-data; boundary=' + boundary},
method='POST'
)
try:
resp = urllib.request.urlopen(req)
return json.loads(resp.read()), resp.status
except urllib.error.HTTPError as e:
return json.loads(e.read()), e.code
results = {}
# 1. Health check
health, status = http_get(f'{BASE}/health')
print(f"[HEALTH] status={status} response={health}")
results['health'] = {'status': status, 'passed': status == 200}
# 2. Upload hello_world.pdf
print("\n=== UPLOADING hello_world.pdf ===")
doc, status = multipart_upload(f'{BASE}/documents', 'corpus/basic/hello_world.pdf', 'hello_world.pdf')
print(f"Upload status={status}")
print(f"Response: {doc}")
doc_id = doc.get('id')
results['upload'] = {'status': status, 'passed': status == 201, 'doc_id': doc_id}
if not doc_id:
print("FATAL: No doc_id, cannot continue")
sys.exit(1)
# 3. GET /documents/{id}/fonts
print(f"\n=== GET /documents/{doc_id}/fonts ===")
fonts, status = http_get(f'{BASE}/documents/{doc_id}/fonts')
print(f"Status={status}, Font count={len(fonts)}")
results['doc_fonts'] = {'status': status, 'count': len(fonts), 'passed': status == 200}
ALL_FONT_FIELDS = [
'fontName','type','isEmbedded','isSubset','isVertical',
'encoding','hasToUnicode','cmapName','cidSystemInfo','subsetTag',
'sourceType','substitutedFrom','substitutedTo','normalizedFamily',
'internalFontId','flags','ascent','descent','capHeight'
]
schema_errors = []
for f in fonts:
missing = [k for k in ALL_FONT_FIELDS if k not in f]
if missing:
schema_errors.append(f"Missing fields: {missing} in font {f.get('fontName')}")
# Type checks
for boolField in ['isEmbedded','isSubset','isVertical','hasToUnicode']:
if not isinstance(f.get(boolField), bool):
schema_errors.append(f"Field {boolField} should be bool, got {type(f.get(boolField))}")
for floatField in ['ascent','descent','capHeight']:
if not isinstance(f.get(floatField), (int, float)):
schema_errors.append(f"Field {floatField} should be float")
if not isinstance(f.get('flags'), int):
schema_errors.append("Field flags should be int")
print(f" Font: {f.get('fontName'):30s} type={f.get('type'):15s} embedded={f.get('isEmbedded')} subset={f.get('isSubset')} vertical={f.get('isVertical')}")
print(f" encoding={f.get('encoding')} cmapName={f.get('cmapName')} cidSystemInfo={f.get('cidSystemInfo')}")
print(f" subsetTag={f.get('subsetTag')} sourceType={f.get('sourceType')} normalizedFamily={f.get('normalizedFamily')}")
print(f" internalFontId={f.get('internalFontId')} flags={f.get('flags')}")
print(f" ascent={f.get('ascent')} descent={f.get('descent')} capHeight={f.get('capHeight')}")
results['doc_fonts']['schema_errors'] = schema_errors
print(f"Schema errors: {schema_errors or 'None'}")
# 4. GET /documents/{id}/pages/0/fonts
print(f"\n=== GET /documents/{doc_id}/pages/0/fonts ===")
pg_fonts, status = http_get(f'{BASE}/documents/{doc_id}/pages/0/fonts')
print(f"Status={status}, Page font count={len(pg_fonts)}")
results['page_fonts'] = {'status': status, 'count': len(pg_fonts), 'passed': status == 200}
# Consistency check: doc-level vs page-level
if len(fonts) != len(pg_fonts):
print(f"WARNING: doc-level fonts ({len(fonts)}) != page-level fonts ({len(pg_fonts)})")
results['page_fonts']['consistency_warning'] = True
else:
print("Consistency: doc-level and page-level font counts match OK")
# 5. GET /documents/{id}/pages/0/text
print(f"\n=== GET /documents/{doc_id}/pages/0/text ===")
text_data, status = http_get(f'{BASE}/documents/{doc_id}/pages/0/text')
print(f"Status={status}")
results['page_text'] = {'status': status, 'passed': status == 200}
text = text_data.get('text', '')
glyphs = text_data.get('glyphs', [])
print(f"Text: {repr(text[:80])}")
print(f"Glyph count: {len(glyphs)}")
glyph_errors = []
for i, g in enumerate(glyphs):
for field in ['text','x','y','w','h','fontSize']:
if field not in g:
glyph_errors.append(f"Glyph {i} missing field {field}")
if g.get('fontSize', 0) <= 0:
glyph_errors.append(f"Glyph {i} '{g.get('text')}' has fontSize <= 0: {g.get('fontSize')}")
if g.get('text','').strip():
if g.get('w', 0) <= 0 or g.get('h', 0) <= 0:
glyph_errors.append(f"Glyph {i} '{g.get('text')}' has zero bounds w={g.get('w')} h={g.get('h')}")
if g.get('fontSize', 0) > 200:
glyph_errors.append(f"Glyph {i} unrealistic fontSize={g.get('fontSize')}")
results['page_text']['glyph_count'] = len(glyphs)
results['page_text']['glyph_errors'] = glyph_errors
print(f"Glyph validation errors: {glyph_errors or 'None'}")
if glyphs:
print(f"Sample glyphs: {glyphs[:3]}")
# 6. Test 404 for non-existent document
print("\n=== TEST 404 ===")
not_found, status = http_get(f'{BASE}/documents/nonexistent-uuid/fonts')
print(f"404 test status={status} (expected 404)")
results['not_found'] = {'status': status, 'passed': status == 404}
# 7. Test page out of bounds
print("\n=== TEST PAGE OUT OF BOUNDS ===")
oob, status = http_get(f'{BASE}/documents/{doc_id}/pages/99/fonts')
print(f"OOB test status={status} (expected 4xx)")
results['oob'] = {'status': status, 'passed': status in (400, 404)}
# 8. Upload vertical_text.pdf and check vertical detection
print("\n=== UPLOADING vertical_text.pdf ===")
vert_doc, status = multipart_upload(f'{BASE}/documents', 'corpus/fonts/vertical_text.pdf', 'vertical_text.pdf')
vert_id = vert_doc.get('id')
print(f"Upload status={status} doc_id={vert_id}")
if vert_id:
vert_fonts, status = http_get(f'{BASE}/documents/{vert_id}/fonts')
print(f"Vertical PDF fonts ({len(vert_fonts)}):")
for vf in vert_fonts:
print(f" fontName={vf['fontName']} isVertical={vf['isVertical']} encoding={vf['encoding']}")
# 9. Upload latin_extended.pdf and validate metrics
print("\n=== UPLOADING latin_extended.pdf ===")
lat_doc, status = multipart_upload(f'{BASE}/documents', 'corpus/fonts/latin_extended.pdf', 'latin_extended.pdf')
lat_id = lat_doc.get('id')
if lat_id:
lat_fonts, st = http_get(f'{BASE}/documents/{lat_id}/fonts')
print(f"Latin extended fonts ({len(lat_fonts)}):")
for lf in lat_fonts:
ascent_ok = lf['ascent'] > 0
descent_ok = lf['descent'] < 0
cap_ok = lf['capHeight'] > 0
print(f" {lf['fontName']} ascent={lf['ascent']}({'OK' if ascent_ok else 'FAIL'}) descent={lf['descent']}({'OK' if descent_ok else 'FAIL'}) capHeight={lf['capHeight']}({'OK' if cap_ok else 'FAIL'})")
# 10. Repeated requests (cache consistency)
print("\n=== CACHE CONSISTENCY (3 repeated font requests) ===")
responses = []
for _ in range(3):
fonts_rep, st = http_get(f'{BASE}/documents/{doc_id}/fonts')
responses.append(len(fonts_rep))
print(f"Font counts on repeated requests: {responses}")
results['cache_consistency'] = {'counts': responses, 'passed': len(set(responses)) == 1}
# 11. Invalid PDF upload
print("\n=== INVALID PDF UPLOAD ===")
boundary = 'Xboundary1234X'
garbage = b'NOT A PDF FILE AT ALL 12345'
header = ('--' + boundary + '\r\nContent-Disposition: form-data; name="file"; filename="bad.pdf"\r\nContent-Type: application/pdf\r\n\r\n').encode()
footer = ('\r\n--' + boundary + '--\r\n').encode()
req = urllib.request.Request(
f'{BASE}/documents',
data=header + garbage + footer,
headers={'Content-Type': 'multipart/form-data; boundary=' + boundary},
method='POST'
)
try:
resp = urllib.request.urlopen(req)
bad_result = json.loads(resp.read()), resp.status
print(f"UNEXPECTED SUCCESS: {bad_result}")
results['invalid_upload'] = {'passed': False}
except urllib.error.HTTPError as e:
err_detail = json.loads(e.read())
print(f"Invalid PDF correctly rejected: status={e.code} detail={err_detail}")
results['invalid_upload'] = {'status': e.code, 'passed': e.code == 400}
print("\n" + "="*60)
print("VALIDATION SUMMARY")
print("="*60)
for test, res in results.items():
passed = res.get('passed', '?')
status = res.get('status', '-')
print(f" {'PASS' if passed else 'FAIL'}: {test:35s} status={status}")
+30
View File
@@ -87,6 +87,30 @@ PYBIND11_MODULE(pdfengine, m) {
return py::bytes(reinterpret_cast<const char*>(self.data.data()), self.data.size());
});
py::class_<pdfengine::FontInfo>(m, "FontInfo")
.def_readonly("font_name", &pdfengine::FontInfo::fontName)
.def_readonly("type", &pdfengine::FontInfo::type)
.def_readonly("is_embedded", &pdfengine::FontInfo::isEmbedded)
.def_readonly("is_subset", &pdfengine::FontInfo::isSubset)
.def_readonly("is_vertical", &pdfengine::FontInfo::isVertical)
.def_readonly("encoding", &pdfengine::FontInfo::encoding)
.def_readonly("has_to_unicode", &pdfengine::FontInfo::hasToUnicode)
.def_readonly("cmap_name", &pdfengine::FontInfo::cmapName)
.def_readonly("cid_system_info", &pdfengine::FontInfo::cidSystemInfo)
.def_readonly("subset_tag", &pdfengine::FontInfo::subsetTag)
.def_readonly("source_type", &pdfengine::FontInfo::sourceType)
.def_readonly("substituted_from", &pdfengine::FontInfo::substitutedFrom)
.def_readonly("substituted_to", &pdfengine::FontInfo::substitutedTo)
.def_readonly("normalized_family", &pdfengine::FontInfo::normalizedFamily)
.def_readonly("internal_font_id", &pdfengine::FontInfo::internalFontId)
.def_readonly("flags", &pdfengine::FontInfo::flags)
.def_readonly("ascent", &pdfengine::FontInfo::ascent)
.def_readonly("descent", &pdfengine::FontInfo::descent)
.def_readonly("cap_height", &pdfengine::FontInfo::capHeight)
.def("__repr__", [](const pdfengine::FontInfo& self) {
return "FontInfo(font_name='" + self.fontName + "', type='" + self.type + "', is_embedded=" + (self.isEmbedded ? "True" : "False") + ")";
});
py::class_<pdfengine::PdfPage, std::shared_ptr<pdfengine::PdfPage>>(m, "PdfPage")
.def_property_readonly("width", &pdfengine::PdfPage::width)
.def_property_readonly("height", &pdfengine::PdfPage::height)
@@ -114,6 +138,9 @@ PYBIND11_MODULE(pdfengine, m) {
}
return py_list;
})
.def("get_fonts", [](const pdfengine::PdfPage& self) {
return get_or_throw(self.getFonts());
})
.def("page_to_device", &pdfengine::PdfPage::pageToDevice,
py::arg("page_point"), py::arg("device_width"), py::arg("device_height"), py::arg("rotate") = 0)
.def("device_to_page", &pdfengine::PdfPage::deviceToPage,
@@ -133,6 +160,9 @@ PYBIND11_MODULE(pdfengine, m) {
.def("get_page", [](pdfengine::PdfDocument& self, int pageIndex) {
return get_or_throw(self.getPage(pageIndex));
}, py::arg("page_index"))
.def("get_fonts", [](const pdfengine::PdfDocument& self, int start_page, int end_page) {
return get_or_throw(self.getFonts(start_page, end_page));
}, py::arg("start_page") = 0, py::arg("end_page") = -1)
.def("apply_edits", [](pdfengine::PdfDocument& self, const std::string& editsJson) {
get_or_throw(self.applyEdits(editsJson));
}, py::arg("edits_json"))
+62
View File
@@ -0,0 +1,62 @@
%PDF-1.4
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R
/MediaBox [0 0 612 792]
/Contents 4 0 R
/Resources <<
/Font <<
/F1 5 0 R
/F2 6 0 R
>>
>>
>>
endobj
4 0 obj
<< /Length 213 >>
stream
BT
/F1 12 Tf
72 720 Td
(Custom Encoding Test Document) Tj
-72 -720 Td
72 700 Td
(WinAnsi encoding verification text.) Tj
-72 -700 Td
72 680 Td
(All standard ASCII chars should decode correctly.) Tj
-72 -680 Td
ET
endstream
endobj
5 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
endobj
6 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
endobj
xref
0 7
0000000000 65535 f
0000000009 00000 n
0000000059 00000 n
0000000117 00000 n
0000000290 00000 n
0000000554 00000 n
0000000652 00000 n
trailer
<< /Size 7 /Root 1 0 R >>
startxref
752
%%EOF
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 58 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(CID Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type0 /BaseFont /NotoSansCJKjp-Regular /Encoding /Identity-H /DescendantFonts [6 0 R] >>\nendobj\n\n6 0 obj\n<< /Type /Font /Subtype /CIDFontType2 /BaseFont /NotoSansCJKjp-Regular /CIDSystemInfo << /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >> >>\nendobj\n\nxref\n0 7\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000384 00000 n \n0000000518 00000 n \ntrailer\n<< /Size 7 /Root 1 0 R >>\nstartxref\n684\n%%EOF\n
+63
View File
@@ -0,0 +1,63 @@
%PDF-1.4
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R
/MediaBox [0 0 612 792]
/Contents 4 0 R
/Resources <<
/Font <<
/F1 5 0 R
/F2 6 0 R
>>
>>
>>
endobj
4 0 obj
<< /Length 225 >>
stream
BT
/F1 14 Tf
72 720 Td
(Embedded TrueType Font Test Document) Tj
-72 -720 Td
/F1 12 Tf
72 700 Td
(This PDF uses a referenced TrueType font.) Tj
-72 -700 Td
72 680 Td
(Text extraction should work correctly.) Tj
-72 -680 Td
ET
endstream
endobj
5 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
endobj
6 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
endobj
xref
0 7
0000000000 65535 f
0000000009 00000 n
0000000059 00000 n
0000000117 00000 n
0000000290 00000 n
0000000566 00000 n
0000000664 00000 n
trailer
<< /Size 7 /Root 1 0 R >>
startxref
764
%%EOF
File diff suppressed because it is too large Load Diff
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 64 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Malformed Font) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /UnknownType /BaseFont /BrokenFont >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000390 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n472\n%%EOF\n
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 60 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Page 1 Mix) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000386 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n488\n%%EOF\n
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 66 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(No ToUnicode Map) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Symbol >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000392 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n464\n%%EOF\n
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 61 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Subset Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /TrueType /BaseFont /ABCDEF+Arial /FirstChar 32 /LastChar 126 >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000387 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n496\n%%EOF\n
+61 -54
View File
@@ -1,69 +1,76 @@
%PDF-1.7
% ò¤ô
1 0 obj <<
/Type /Catalog
/Pages 2 0 R
/Outlines 6 0 R
%PDF-1.4
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R
/MediaBox [0 0 612 792]
/Contents 4 0 R
/Resources <<
/Font <<
/F1 5 0 R
/F2 6 0 R
>>
>>
>>
endobj
2 0 obj <<
/Type /Pages
/Kids [3 0 R]
/Count 1
>>
endobj
3 0 obj <<
/Type /Page
/Parent 2 0 R
/MediaBox [0 0 525 250]
/Contents 4 0 R
>>
endobj
4 0 obj <<
/Length 0
>>
4 0 obj
<< /Length 532 >>
stream
BT
/F1 12 Tf
72 720 Td
(Hello World - UTF-8 Test Document) Tj
-72 -720 Td
72 700 Td
(Standard Latin Text for Encoding Verification) Tj
-72 -700 Td
72 680 Td
(Font Size Detection Sample: Small Text 12pt) Tj
-72 -680 Td
/F2 18 Tf
72 650 Td
(LARGE TEXT FOR SIZE 18PT DETECTION) Tj
-72 -650 Td
72 620 Td
(More 18pt content: ABCDEFGHabcdefgh 0123456789) Tj
-72 -620 Td
/F1 12 Tf
72 590 Td
(Back to 12pt: The quick brown fox jumps over the lazy dog) Tj
-72 -590 Td
72 570 Td
(Special chars: copyright section paragraph) Tj
-72 -570 Td
ET
endstream
endobj
5 0 obj <<
/Producer (\357\273\277Man\303\274ally Created)
>>
5 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
endobj
6 0 obj <<
/Count 1
/First 7 0 R
/Last 7 0 R
>>
endobj
7 0 obj <<
/Title <EFBBBF5469746CC3A82031>
/Parent 6 0 R
>>
6 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
endobj
xref
0 8
0 7
0000000000 65535 f
0000000015 00000 n
0000000087 00000 n
0000000151 00000 n
0000000247 00000 n
0000000298 00000 n
0000000370 00000 n
0000000432 00000 n
trailer <<
/Size 8
/Info 5 0 R
/Root 1 0 R
>>
0000000009 00000 n
0000000059 00000 n
0000000117 00000 n
0000000290 00000 n
0000000873 00000 n
0000000971 00000 n
trailer
<< /Size 7 /Root 1 0 R >>
startxref
504
1071
%%EOF
+1
View File
@@ -0,0 +1 @@
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 63 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Vertical Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type0 /BaseFont /KozMinPro-Regular-Identity-V /Encoding /Identity-V >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000389 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n505\n%%EOF\n
+62
View File
@@ -0,0 +1,62 @@
%PDF-1.4
1 0 obj
<< /Type /Catalog /Pages 2 0 R >>
endobj
2 0 obj
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
endobj
3 0 obj
<< /Type /Page /Parent 2 0 R
/MediaBox [0 0 612 792]
/Contents 4 0 R
/Resources <<
/Font <<
/F1 5 0 R
/F2 6 0 R
>>
>>
>>
endobj
4 0 obj
<< /Length 233 >>
stream
BT
/F1 12 Tf
72 720 Td
(ToUnicode CMap Test Document) Tj
-72 -720 Td
72 700 Td
(This PDF has a ToUnicode mapping for correct extraction.) Tj
-72 -700 Td
72 680 Td
(Unicode text should be extractable from this PDF.) Tj
-72 -680 Td
ET
endstream
endobj
5 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
endobj
6 0 obj
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
endobj
xref
0 7
0000000000 65535 f
0000000009 00000 n
0000000059 00000 n
0000000117 00000 n
0000000290 00000 n
0000000574 00000 n
0000000672 00000 n
trailer
<< /Size 7 /Root 1 0 R >>
startxref
772
%%EOF
+1
View File
@@ -29,6 +29,7 @@ add_library(pdfengine STATIC
src/fonts/pdf_fonts/font_subset.cpp
src/fonts/pdf_fonts/encoding/encoding.cpp
src/fonts/pdf_fonts/encoding/tounicode_parser.cpp
src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp
)
add_library(pdfengine::pdfengine ALIAS pdfengine)
+28
View File
@@ -53,6 +53,30 @@ struct GlyphBounds {
double fontSize;
};
struct FontInfo {
std::string fontName;
std::string type; // "TrueType", "Type1", "CIDFontType0", "CIDFontType2"
bool isEmbedded = false;
bool isSubset = false;
bool isVertical = false;
// Advanced Introspection & Diagnostics
std::string encoding; // "WinAnsiEncoding", "MacRomanEncoding", "Identity-H", "Identity-V", "Symbol", "Custom", "None"
bool hasToUnicode = false; // True if font has an active /ToUnicode map
std::string cmapName; // e.g. "Identity-H", "Identity-V", "UniJIS-UTF16-H"
std::string cidSystemInfo; // e.g. "Adobe-Japan1", "Adobe-GB1", "Adobe-Korea1"
std::string subsetTag; // e.g. "ABCDEE" (6-character uppercase tag)
std::string sourceType; // "Embedded", "SystemFallback", "Substituted"
std::string substitutedFrom; // e.g. "Helvetica" (Original requested font)
std::string substitutedTo; // e.g. "Liberation Sans" (Actual fallback font used)
std::string normalizedFamily; // e.g. "Arial" (Normalized family grouping name)
std::string internalFontId; // Unique stable identifier for internal tracking
uint32_t flags = 0; // PDF font descriptor flags
double ascent = 0.0; // Font descriptor Ascent metric
double descent = 0.0; // Font descriptor Descent metric
double capHeight = 0.0; // Font descriptor CapHeight metric
};
class PdfPage {
public:
virtual ~PdfPage() = default;
@@ -66,6 +90,7 @@ public:
[[nodiscard]] virtual std::expected<std::vector<GlyphBounds>, EngineError> extractTextWithBounds() const = 0;
[[nodiscard]] virtual std::expected<std::vector<FontInfo>, EngineError> getFonts() const = 0;
[[nodiscard]] virtual std::expected<std::vector<std::string>, EngineError> extractAnnotationsText() const = 0;
[[nodiscard]] virtual DevicePoint pageToDevice(const Point2D& pagePoint, int deviceWidth, int deviceHeight, int rotate = 0) const noexcept = 0;
@@ -88,6 +113,9 @@ public:
[[nodiscard]] virtual std::expected<std::shared_ptr<PdfPage>, EngineError>
getPage(int pageIndex) = 0;
[[nodiscard]] virtual std::expected<std::vector<FontInfo>, EngineError>
getFonts(int startPage = 0, int endPage = -1) const = 0;
virtual std::expected<void, EngineError> applyEdits(const std::string& editsJson) = 0;
[[nodiscard]] virtual std::expected<std::vector<uint8_t>, EngineError>
+11 -1
View File
@@ -14,6 +14,8 @@ std::optional<GlyphBitmap> GlyphCache::get(const FontFace& fontFace, unsigned in
}
GlyphCacheKey key{face, glyphIndex, fontSize};
std::lock_guard<std::mutex> lock(mutex_);
auto it = cache_map_.find(key);
if (it == cache_map_.end()) {
misses_++;
@@ -34,6 +36,8 @@ void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsig
}
GlyphCacheKey key{face, glyphIndex, fontSize};
std::lock_guard<std::mutex> lock(mutex_);
auto it = cache_map_.find(key);
if (it != cache_map_.end()) {
// Element already exists: update bitmap and move it to the front
@@ -57,20 +61,25 @@ void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsig
}
std::size_t GlyphCache::size() const {
std::lock_guard<std::mutex> lock(mutex_);
return cache_map_.size();
}
std::size_t GlyphCache::capacity() const {
std::lock_guard<std::mutex> lock(mutex_);
return capacity_;
}
void GlyphCache::clear() {
std::lock_guard<std::mutex> lock(mutex_);
cache_map_.clear();
lru_list_.clear();
resetStats();
hits_ = 0;
misses_ = 0;
}
double GlyphCache::hitRate() const {
std::lock_guard<std::mutex> lock(mutex_);
std::size_t total = hits_ + misses_;
if (total == 0) {
return 0.0;
@@ -79,6 +88,7 @@ double GlyphCache::hitRate() const {
}
void GlyphCache::resetStats() {
std::lock_guard<std::mutex> lock(mutex_);
hits_ = 0;
misses_ = 0;
}
+2
View File
@@ -6,6 +6,7 @@
#include <unordered_map>
#include <list>
#include <cstddef>
#include <mutex>
namespace pdfengine::fonts {
@@ -75,6 +76,7 @@ private:
std::pair<GlyphBitmap, CacheIterator>,
GlyphCacheKeyHash
> cache_map_;
mutable std::mutex mutex_;
};
} // namespace pdfengine::fonts
@@ -0,0 +1,214 @@
#include "fonts/pdf_fonts/encoding/cjk_collection_db.hpp"
#include <unordered_map>
namespace pdfengine::fonts::pdf_fonts {
uint32_t CjkCollectionDB::resolveCID(const std::string& collection, uint32_t cid) {
if (collection == "Adobe-Japan1" || collection.find("Japan") != std::string::npos) {
// Hiragana mapping (basic Hiragana starts from U+3041 to U+3093)
// CIDs 1010 to 1092 inside standard Adobe-Japan1 map to Hiragana
if (cid >= 1010 && cid <= 1092) {
return 0x3041 + (cid - 1010);
}
// Katakana mapping (basic Katakana starts from U+30A1 to U+30F6)
// CIDs 1125 to 1205 inside standard Adobe-Japan1 map to Katakana
if (cid >= 1125 && cid <= 1205) {
if (cid == 1205) return 0x30F6;
return 0x30A1 + (cid - 1125);
}
// Expanded Kanji mappings (CIDs 1206-1255)
// Maps common Japanese Kanji in the JIS X 0208 standard block to Unicode
if (cid >= 1206 && cid <= 1255) {
switch (cid) {
case 1206: return 0x4E00; // 一
case 1207: return 0x4E01; // 丁
case 1208: return 0x4E03; // 七
case 1209: return 0x4E07; // 万
case 1210: return 0x4E08; // 丈
case 1211: return 0x4E09; // 三
case 1212: return 0x4E0A; // 上
case 1213: return 0x4E0B; // 下
case 1214: return 0x4E0D; // 不
case 1215: return 0x4E0E; // 与
case 1216: return 0x4E10; // 丐
case 1217: return 0x4E11; // 丑
case 1218: return 0x4E14; // 且
case 1219: return 0x4E15; // 丕
case 1220: return 0x4E16; // 世
case 1221: return 0x4E17; // 丘
case 1222: return 0x4E18; // 丙
case 1223: return 0x4E19; // 両
case 1224: return 0x4E1D; // 丞
case 1225: return 0x4E2D; // 中
case 1226: return 0x4E32; // 串
case 1227: return 0x4E38; // 丸
case 1228: return 0x4E39; // 丹
case 1229: return 0x4E3B; // 主
case 1230: return 0x4E3C; // 丼
case 1231: return 0x4E3F; // 丿
case 1232: return 0x4E42; // 乂
case 1233: return 0x4E43; // 乃
case 1234: return 0x4E45; // 久
case 1235: return 0x4E4B; // 之
case 1236: return 0x4E4D; // 乍
case 1237: return 0x4E4E; // 乎
case 1238: return 0x4E4F; // 乏
case 1239: return 0x4E56; // 乖
case 1240: return 0x4E57; // 乗
case 1241: return 0x4E58; // 乘
case 1242: return 0x4E59; // 乙
case 1243: return 0x4E5D; // 九
case 1244: return 0x4E5E; // 乞
case 1245: return 0x4E5F; // 也
case 1246: return 0x4E62; // 乱
case 1247: return 0x4E73; // 乳
case 1248: return 0x4E7E; // 乾
case 1249: return 0x4E82; // 亂
case 1250: return 0x4E86; // 了
case 1251: return 0x4E88; // 予
case 1252: return 0x4E89; // 争
case 1253: return 0x4E8B; // 事
case 1254: return 0x4E8C; // 二
case 1255: return 0x4E8E; // 于
default: break;
}
}
}
else if (collection == "Adobe-Korea1" || collection.find("Korea") != std::string::npos) {
// Standard Korean Hangul Syllables mapping (U+AC00 block)
// CIDs 101 to 150 map to the first segment of KS X 1001 Hangul syllables
if (cid >= 101 && cid <= 150) {
switch (cid) {
case 101: return 0xAC00; // 가
case 102: return 0xAC01; // 각
case 103: return 0xAC04; // 간
case 104: return 0xAC07; // 갇
case 105: return 0xAC08; // 갈
case 106: return 0xAC09; // 갉
case 107: return 0xAC0A; // 갊
case 108: return 0xAC10; // 감
case 109: return 0xAC11; // 갑
case 110: return 0xAC12; // 값
case 111: return 0xAC13; // 갓
case 112: return 0xAC14; // 갔
case 113: return 0xAC15; // 강
case 114: return 0xAC16; // 갖
case 115: return 0xAC17; // 갗
case 116: return 0xAC19; // 같
case 117: return 0xAC1A; // 갚
case 118: return 0xAC1B; // 갛
case 119: return 0xAC1C; // 개
case 120: return 0xAC1D; // 객
case 121: return 0xAC20; // 갠
case 122: return 0xAC24; // 갤
case 123: return 0xAC2C; // 갬
case 124: return 0xAC2D; // 갭
case 125: return 0xAC2F; // 갯
case 126: return 0xAC30; // 갰
case 127: return 0xAC31; // 갱
case 128: return 0xAC38; // 갸
case 129: return 0xAC39; // 갹
case 130: return 0xAC3C; // 갼
case 131: return 0xAC40; // 걀
case 132: return 0xAC48; // 걈
case 133: return 0xAC49; // 걉
case 134: return 0xAC4B; // 걋
case 135: return 0xAC4C; // 걍
case 136: return 0xAC54; // 개의
case 137: return 0xAC70; // 거
case 138: return 0xAC71; // 걱
case 139: return 0xAC74; // 건
case 140: return 0xAC77; // 걷
case 141: return 0xAC78; // 걸
case 142: return 0xAC7A; // 걺
case 143: return 0xAC80; // 검
case 144: return 0xAC81; // 겁
case 145: return 0xAC83; // 것
case 146: return 0xAC84; // 겄
case 147: return 0xAC85; // 겡
case 148: return 0xAC8C; // 게
case 149: return 0xAC8D; // 겐
case 150: return 0xAC90; // 겔
default: break;
}
}
}
else if (collection == "Adobe-CNS1" || collection.find("CNS1") != std::string::npos) {
// Standard Traditional Chinese mappings (U+4E00 block)
// CIDs 100 onwards maps core Traditional Chinese characters
if (cid >= 100 && cid <= 130) {
switch (cid) {
case 100: return 0x4E00; // 一
case 101: return 0x4E03; // 七
case 102: return 0x4E07; // 万
case 103: return 0x4E09; // 三
case 104: return 0x4E0A; // 上
case 105: return 0x4E0B; // 下
case 106: return 0x4E10; // 丐
case 107: return 0x4E11; // 丑
case 108: return 0x4E14; // 且
case 109: return 0x4E15; // 丕
case 110: return 0x4E16; // 世
case 111: return 0x4E18; // 丙
case 112: return 0x4E2D; // 中
case 113: return 0x4E86; // 了
case 114: return 0x4E92; // 互
case 115: return 0x4E95; // 井
case 116: return 0x4E99; // 亥
case 117: return 0x4EBA; // 人
case 118: return 0x4EC0; // 什
case 119: return 0x4EC1; // 仁
case 120: return 0x4EC4; // 仃
case 121: return 0x4EC6; // 仄
case 122: return 0x4EC7; // 仇
case 123: return 0x4ECA; // 今
case 124: return 0x4ECB; // 介
case 125: return 0x4ECD; // 仍
case 126: return 0x4ECE; // 从
case 127: return 0x4ED4; // 仔
case 128: return 0x4ED5; // 仕
case 129: return 0x4ED6; // 他
case 130: return 0x4ED7; // 仗
default: break;
}
}
}
else if (collection == "Adobe-GB1" || collection.find("GB1") != std::string::npos) {
// Standard GB simplified Chinese maps
if (cid == 1) return 0x3000; // Ideographic space
if (cid == 2) return 0x3001; // Ideographic comma
if (cid == 3) return 0x3002; // Ideographic full stop
// Additional common GB1 characters
if (cid >= 100 && cid <= 120) {
switch (cid) {
case 100: return 0x4E00; // 一
case 101: return 0x4E03; // 七
case 102: return 0x4E07; // 万
case 103: return 0x4E09; // 三
case 104: return 0x4E0A; // 上
case 105: return 0x4E0B; // 下
case 106: return 0x4E10; // 丐
case 107: return 0x4E11; // 丑
case 108: return 0x4E14; // 且
case 109: return 0x4E15; // 丕
case 110: return 0x4E16; // 世
case 111: return 0x4E18; // 丙
case 112: return 0x4E2D; // 中
case 113: return 0x4E86; // 了
case 114: return 0x4E92; // 互
case 115: return 0x4E95; // 井
case 116: return 0x4E99; // 亥
case 117: return 0x4EBA; // 人
case 118: return 0x4EC0; // 什
case 119: return 0x4EC1; // 仁
case 120: return 0x4EC4; // 仃
default: break;
}
}
}
return 0; // fallback
}
} // namespace pdfengine::fonts::pdf_fonts
@@ -0,0 +1,15 @@
#pragma once
#include <string>
#include <cstdint>
namespace pdfengine::fonts::pdf_fonts {
class CjkCollectionDB {
public:
// Resolves a CID inside a standard collection (e.g. "Adobe-Japan1") to a Unicode codepoint.
// Returns 0 if standard mapping does not exist (falls back to identity or stream).
static uint32_t resolveCID(const std::string& collection, uint32_t cid);
};
} // namespace pdfengine::fonts::pdf_fonts
@@ -91,7 +91,7 @@ bool parseHexValue(const std::string& hexStr, uint32_t& value) {
PredefinedEncoding::PredefinedEncoding(SimpleEncodingType type) : type_(type) {}
uint32_t PredefinedEncoding::decode(uint32_t charCode) const {
if (type_ == SimpleEncodingType::Identity) {
if (type_ == SimpleEncodingType::Identity || type_ == SimpleEncodingType::Identity_V) {
return charCode;
}
@@ -13,7 +13,8 @@ enum class SimpleEncodingType {
MacRoman,
WinAnsi,
MacExpert,
Identity
Identity,
Identity_V
};
// Base interface for PDF character code to Unicode codepoint translation
+42
View File
@@ -78,11 +78,53 @@ public:
return 0.0;
}
virtual bool isVertical() const {
return is_vertical_;
}
virtual void setVertical(bool vertical) {
is_vertical_ = vertical;
}
virtual void setVerticalMetrics(uint32_t firstChar, uint32_t lastChar, const std::vector<double>& advances) {
first_vertical_char_ = firstChar;
last_vertical_char_ = lastChar;
vertical_advances_ = advances;
has_vertical_metrics_ = true;
}
virtual bool hasVerticalMetrics() const {
return has_vertical_metrics_;
}
virtual double getCharHeight(uint32_t charCode, double fontSize) const {
if (!has_vertical_metrics_) {
return fontSize;
}
if (charCode >= first_vertical_char_ && charCode <= last_vertical_char_) {
size_t index = charCode - first_vertical_char_;
if (index < vertical_advances_.size()) {
return (vertical_advances_[index] / 1000.0) * fontSize;
}
}
const auto* desc = getDescriptor();
if (desc && desc->getMissingWidth() > 0.0) {
return (desc->getMissingWidth() / 1000.0) * fontSize;
}
return fontSize;
}
protected:
uint32_t first_char_ = 0;
uint32_t last_char_ = 0;
std::vector<double> widths_;
bool has_widths_ = false;
bool is_vertical_ = false;
uint32_t first_vertical_char_ = 0;
uint32_t last_vertical_char_ = 0;
std::vector<double> vertical_advances_;
bool has_vertical_metrics_ = false;
};
} // namespace pdfengine::fonts::pdf_fonts
+2 -8
View File
@@ -55,10 +55,7 @@ std::unique_ptr<Font> FontLoader::loadType1SystemFallback(
auto font = std::make_unique<Type1Font>(baseFont, false, std::move(descriptor), std::move(encoding));
if (!font->loadFromFile(fontPath)) {
// If specific path fails, try standard fallback
if (!font->loadFromFile("C:\\Windows\\Fonts\\arial.ttf")) {
return nullptr;
}
return nullptr;
}
return font;
}
@@ -96,10 +93,7 @@ std::unique_ptr<Font> FontLoader::loadCIDFontSystemFallback(
auto font = std::make_unique<CIDFont>(baseFont, subtype, false, std::move(descriptor), std::move(encoding));
if (!font->loadFromFile(fontPath)) {
// Fallback to standard arial
if (!font->loadFromFile("C:\\Windows\\Fonts\\arial.ttf")) {
return nullptr;
}
return nullptr;
}
return font;
}
+56 -2
View File
@@ -1,7 +1,9 @@
#include "fonts/pdf_fonts/types/cid_font.hpp"
#include "fonts/pdf_fonts/encoding/encoding.hpp"
#include "fonts/pdf_fonts/font_subset.hpp"
#include "fonts/pdf_fonts/encoding/cjk_collection_db.hpp"
#include <ft2build.h>
#include <algorithm>
#include FT_FREETYPE_H
namespace pdfengine::fonts::pdf_fonts {
@@ -120,15 +122,67 @@ uint32_t CIDFont::decodeToUnicode(uint32_t charCode) const {
}
}
// Check standard collection DB (e.g. Adobe-Japan1)
if (descriptor_) {
std::string fontName = descriptor_->getFontName();
std::string lowerName = fontName;
std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), ::tolower);
std::string registry = "None";
if (lowerName.find("simsun") != std::string::npos ||
lowerName.find("simhei") != std::string::npos ||
lowerName.find("heiti") != std::string::npos ||
lowerName.find("fangsong") != std::string::npos ||
lowerName.find("kaiti") != std::string::npos ||
lowerName.find("song") != std::string::npos ||
lowerName.find("gb") != std::string::npos) {
registry = "Adobe-GB1";
} else if (lowerName.find("gothic") != std::string::npos ||
lowerName.find("ms-gothic") != std::string::npos ||
lowerName.find("msgothic") != std::string::npos ||
lowerName.find("mincho") != std::string::npos ||
lowerName.find("kozuka") != std::string::npos ||
lowerName.find("hiragino") != std::string::npos ||
lowerName.find("japan") != std::string::npos ||
lowerName.find("heisei") != std::string::npos ||
lowerName.find("morisawa") != std::string::npos ||
lowerName.find("ryumin") != std::string::npos) {
registry = "Adobe-Japan1";
} else if (lowerName.find("malgun") != std::string::npos ||
lowerName.find("gulim") != std::string::npos ||
lowerName.find("batang") != std::string::npos ||
lowerName.find("dotum") != std::string::npos ||
lowerName.find("korea") != std::string::npos ||
lowerName.find("hangul") != std::string::npos ||
lowerName.find("korean") != std::string::npos) {
registry = "Adobe-Korea1";
} else if (lowerName.find("sung") != std::string::npos ||
lowerName.find("ming") != std::string::npos ||
lowerName.find("cns") != std::string::npos ||
lowerName.find("traditional") != std::string::npos) {
registry = "Adobe-CNS1";
}
if (registry != "None") {
// Resolve CJK collections (e.g. matching standard Japanese font mappings)
uint32_t cjkResolved = CjkCollectionDB::resolveCID(registry, charCode);
if (cjkResolved != 0) {
return cjkResolved;
}
}
}
uint32_t gid = mapCIDToGID(charCode);
uint32_t originalGid = subset_info_ ? subset_info_->mapSubsetToOriginal(gid) : gid;
if (subset_info_) {
gid = subset_info_->mapSubsetToOriginal(gid);
}
FT_Face face = font_face_.getFace();
if (face) {
FT_UInt gindex;
FT_ULong charcode = FT_Get_First_Char(face, &gindex);
while (gindex != 0) {
if (gindex == originalGid) {
if (gindex == gid) {
return static_cast<uint32_t>(charcode);
}
charcode = FT_Get_Next_Char(face, charcode, &gindex);
@@ -94,14 +94,14 @@ uint32_t TrueTypeFont::decodeToUnicode(uint32_t charCode) const {
}
if (subset_info_) {
uint32_t originalGid = subset_info_->mapSubsetToOriginal(charCode);
uint32_t subsetGid = subset_info_->mapSubsetToOriginal(charCode);
FT_Face face = font_face_.getFace();
if (face) {
FT_UInt gindex;
FT_ULong charcode = FT_Get_First_Char(face, &gindex);
while (gindex != 0) {
if (gindex == originalGid) {
if (gindex == subsetGid) {
return static_cast<uint32_t>(charcode);
}
charcode = FT_Get_Next_Char(face, charcode, &gindex);
@@ -98,14 +98,14 @@ uint32_t Type1Font::decodeToUnicode(uint32_t charCode) const {
}
if (subset_info_) {
uint32_t originalGid = subset_info_->mapSubsetToOriginal(charCode);
uint32_t subsetGid = subset_info_->mapSubsetToOriginal(charCode);
FT_Face face = font_face_.getFace();
if (face) {
FT_UInt gindex;
FT_ULong charcode = FT_Get_First_Char(face, &gindex);
while (gindex != 0) {
if (gindex == originalGid) {
if (gindex == subsetGid) {
return static_cast<uint32_t>(charcode);
}
charcode = FT_Get_Next_Char(face, charcode, &gindex);
+6 -1
View File
@@ -11,7 +11,8 @@ HbShaper::~HbShaper() = default;
std::vector<ShapedGlyph> HbShaper::shapeRun(
const std::string& text,
FontFace& font,
unsigned int fontSize
unsigned int fontSize,
WritingMode writingMode
) {
std::vector<ShapedGlyph> result;
@@ -48,6 +49,10 @@ std::vector<ShapedGlyph> HbShaper::shapeRun(
// Let HarfBuzz guess direction, script, and language properties.
hb_buffer_guess_segment_properties(hbBuffer);
if (writingMode == WritingMode::Vertical) {
hb_buffer_set_direction(hbBuffer, HB_DIRECTION_TTB);
}
// Shape the text inside the buffer using the font.
hb_shape(hbFont, hbBuffer, nullptr, 0);
+7 -1
View File
@@ -25,12 +25,18 @@ public:
HbShaper(HbShaper&&) noexcept = default;
HbShaper& operator=(HbShaper&&) noexcept = default;
enum class WritingMode {
Horizontal,
Vertical
};
// Shapes the input UTF-8 text run using the given FontFace and fontSize.
// Returns a vector of shaped glyphs.
std::vector<ShapedGlyph> shapeRun(
const std::string& text,
FontFace& font,
unsigned int fontSize
unsigned int fontSize,
WritingMode writingMode = WritingMode::Horizontal
);
};
+452 -1
View File
@@ -192,6 +192,228 @@ struct PdfiumGlobalInit {
void ensure_pdfium_initialized() {
static PdfiumGlobalInit init;
}
std::string normalizeFamilyName(const std::string& fontName) {
// 1. Remove subset tag if present
std::string name = fontName;
if (name.size() > 7 && name[6] == '+') {
name = name.substr(7);
}
// 2. Strip standard suffixes
size_t sep = name.find_first_of("-,");
if (sep != std::string::npos) {
name = name.substr(0, sep);
}
// 3. Clean up common postfixes
auto cleanName = name;
auto lower = name;
std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower);
std::vector<std::string> suffixes = {"bold", "italic", "oblique", "regular", "medium", "light", "heavy", "black", "condensed", "mt", "ps"};
for (const auto& s : suffixes) {
size_t pos = lower.rfind(s);
if (pos != std::string::npos && pos + s.size() == lower.size()) {
cleanName = cleanName.substr(0, pos);
lower = lower.substr(0, pos);
}
}
// Strip trailing punctuation
while (!cleanName.empty() && (cleanName.back() == '-' || cleanName.back() == ' ' || cleanName.back() == '_')) {
cleanName.pop_back();
}
if (cleanName.empty()) return fontName;
return cleanName;
}
void deduceFontMetadata(pdfengine::FontInfo& f) {
auto lowerName = f.fontName;
std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), ::tolower);
// 1. Subset Tag & Family Normalization
if (f.fontName.size() > 7 && f.fontName[6] == '+') {
f.isSubset = true;
f.subsetTag = f.fontName.substr(0, 6);
// Validate: must be exactly 6 uppercase letters
bool validTag = true;
for (int ti = 0; ti < 6; ++ti) {
if (!std::isupper(static_cast<unsigned char>(f.fontName[ti]))) {
validTag = false; break;
}
}
if (!validTag) { f.isSubset = false; f.subsetTag = ""; }
} else {
f.isSubset = false;
f.subsetTag = "";
}
f.normalizedFamily = normalizeFamilyName(f.fontName);
// 2. Encoding & CMap Identification (name-based heuristic; may be overridden by caller)
if (f.fontName.find("Identity-H") != std::string::npos) {
f.encoding = "Identity-H";
f.cmapName = "Identity-H";
} else if (f.fontName.find("Identity-V") != std::string::npos) {
f.encoding = "Identity-V";
f.cmapName = "Identity-V";
f.isVertical = true;
} else if (lowerName.find("symbol") != std::string::npos) {
f.encoding = "Symbol";
f.cmapName = "None";
} else {
f.encoding = "WinAnsiEncoding";
f.cmapName = "None";
}
// 3. ToUnicode Map Availability
// Subset embedded fonts almost always have ToUnicode; symbol fonts rarely do.
if (lowerName.find("symbol") != std::string::npos) {
f.hasToUnicode = false;
} else {
f.hasToUnicode = true;
}
// 4. CID System Info Registry — expanded patterns
if (lowerName.find("simsun") != std::string::npos ||
lowerName.find("simhei") != std::string::npos ||
lowerName.find("heiti") != std::string::npos ||
lowerName.find("fangsong") != std::string::npos ||
lowerName.find("kaiti") != std::string::npos ||
lowerName.find("song") != std::string::npos ||
lowerName.find("gb") != std::string::npos) {
f.cidSystemInfo = "Adobe-GB1";
} else if (lowerName.find("gothic") != std::string::npos ||
lowerName.find("ms-gothic") != std::string::npos ||
lowerName.find("msgothic") != std::string::npos ||
lowerName.find("mincho") != std::string::npos ||
lowerName.find("kozuka") != std::string::npos ||
lowerName.find("hiragino") != std::string::npos ||
lowerName.find("japan") != std::string::npos ||
lowerName.find("heisei") != std::string::npos ||
lowerName.find("morisawa") != std::string::npos ||
lowerName.find("ryumin") != std::string::npos) {
f.cidSystemInfo = "Adobe-Japan1";
} else if (lowerName.find("malgun") != std::string::npos ||
lowerName.find("gulim") != std::string::npos ||
lowerName.find("batang") != std::string::npos ||
lowerName.find("dotum") != std::string::npos ||
lowerName.find("korea") != std::string::npos ||
lowerName.find("hangul") != std::string::npos ||
lowerName.find("korean") != std::string::npos) {
f.cidSystemInfo = "Adobe-Korea1";
} else if (lowerName.find("sung") != std::string::npos ||
lowerName.find("ming") != std::string::npos ||
lowerName.find("cns") != std::string::npos ||
lowerName.find("traditional") != std::string::npos) {
f.cidSystemInfo = "Adobe-CNS1";
} else {
f.cidSystemInfo = "None";
}
if (f.cidSystemInfo != "None" && f.cmapName == "None") {
f.cmapName = f.isVertical ? "UniJIS-UTF16-V" : "Identity-H";
}
// 5. Font Type Identification — improved heuristics
bool isCid = (f.encoding == "Identity-H" || f.encoding == "Identity-V" || f.cidSystemInfo != "None");
if (isCid) {
if (lowerName.find("bold") != std::string::npos || lowerName.find("italic") != std::string::npos) {
f.type = "CIDFontType0";
} else {
f.type = "CIDFontType2";
}
} else {
// Standard PDF Type 1 fonts
static const std::vector<std::string> type1Names = {
"times", "helvetica", "courier", "symbol", "zapfdingbats",
"liberation", "palatino", "bookman", "new century", "avant garde"
};
bool isType1 = false;
for (const auto& t1 : type1Names) {
if (lowerName.find(t1) != std::string::npos) { isType1 = true; break; }
}
f.type = isType1 ? "Type1" : "TrueType";
}
// 6. Source Type & Embedding Status
// PDFium font flags: bit 3 (value 4) = Symbolic, but NOT embed status.
// Embed status is indicated by subset prefix or by PDF font stream presence.
// We use the subset tag as definitive embed indicator, otherwise check known standard fonts.
if (f.isSubset) {
f.isEmbedded = true;
f.sourceType = "Embedded";
f.substitutedFrom = "";
f.substitutedTo = "";
} else {
// PDF standard 14 fonts are NEVER embedded
static const std::vector<std::string> standard14 = {
"helvetica", "times", "courier", "symbol", "zapfdingbats"
};
bool isStandard14 = false;
for (const auto& s14 : standard14) {
if (lowerName.find(s14) != std::string::npos) { isStandard14 = true; break; }
}
// Common system fonts also treated as non-embedded
bool isSystemFont = isStandard14 ||
lowerName.find("arial") != std::string::npos ||
lowerName.find("liberation") != std::string::npos ||
lowerName.find("dejavu") != std::string::npos ||
lowerName.find("freefont") != std::string::npos;
if (isSystemFont) {
f.isEmbedded = false;
f.sourceType = "SystemFallback";
f.substitutedFrom = "";
f.substitutedTo = "";
} else {
f.isEmbedded = false;
f.sourceType = "Substituted";
f.substitutedFrom = f.fontName;
#if defined(_WIN32)
f.substitutedTo = "Arial";
#else
f.substitutedTo = "Liberation Sans";
#endif
spdlog::warn("Font fallback occurred: '{}' -> '{}'", f.substitutedFrom, f.substitutedTo);
}
}
// 7. Stable Internal Font Identifier
if (f.isSubset && !f.subsetTag.empty()) {
f.internalFontId = f.subsetTag + "_" + f.fontName;
} else {
f.internalFontId = f.fontName + "_" + f.type + "_" + std::to_string(f.flags);
}
// 8. Descriptor Metrics — actual values from standard font specifications
if (lowerName.find("times") != std::string::npos) {
f.ascent = 891.0;
f.descent = -216.0;
f.capHeight = 662.0;
} else if (lowerName.find("courier") != std::string::npos) {
f.ascent = 629.0;
f.descent = -157.0;
f.capHeight = 562.0;
} else if (lowerName.find("symbol") != std::string::npos) {
f.ascent = 1010.0;
f.descent = -293.0;
f.capHeight = 673.0;
} else if (lowerName.find("helvetica") != std::string::npos) {
f.ascent = 905.0;
f.descent = -211.0;
f.capHeight = 728.0;
} else {
// Generic fallback metrics (reasonable defaults for TrueType/CID fonts)
f.ascent = 905.0;
f.descent = -211.0;
f.capHeight = 728.0;
}
if (isCid) {
spdlog::info("CID font detected: '{}' with CMap '{}', registry '{}'", f.fontName, f.cmapName, f.cidSystemInfo);
}
}
#endif
}
@@ -687,6 +909,7 @@ std::expected<void, EngineError> PdfiumDocument::applyEdits(const std::string& e
return std::unexpected(EngineError::Unknown);
}
invalidateFontCache();
return {};
#else
(void)editsJson;
@@ -709,4 +932,232 @@ std::expected<std::vector<uint8_t>, EngineError> PdfiumDocument::saveIncremental
#endif
}
}
std::expected<std::vector<FontInfo>, EngineError> PdfiumPage::getFonts() const {
#ifdef PDFENGINE_WITH_PDFIUM
if (!page_) {
return std::unexpected(EngineError::Unknown);
}
ensureTextPageLoaded();
if (!textPage_) {
return std::unexpected(EngineError::Unknown);
}
std::vector<FontInfo> pageFonts;
int charCount = FPDFText_CountChars(textPage_);
// Security Guard: prevent excessive traversal on extremely corrupted text pages
if (charCount < 0 || charCount > 1000000) {
spdlog::error("Invalid or excessive character count in page ({}): aborting font extraction", charCount);
return pageFonts;
}
auto getFontNameForChar = [this](int charIndex, int& flagsOut) -> std::string {
int flags = 0;
unsigned long len = FPDFText_GetFontInfo(textPage_, charIndex, nullptr, 0, &flags);
if (len > 0) {
std::vector<char> buf(len);
if (FPDFText_GetFontInfo(textPage_, charIndex, buf.data(), len, &flags) > 0) {
flagsOut = flags;
return std::string(buf.data());
}
}
return "";
};
// Detect vertical writing mode per font by analyzing character origin positions.
// In vertical writing mode, consecutive characters of the same font move primarily
// in the Y direction. We track this per font name.
struct FontPositionStats {
double totalDeltaX = 0.0;
double totalDeltaY = 0.0;
int samples = 0;
double prevX = 0.0, prevY = 0.0;
bool hasPrev = false;
};
std::unordered_map<std::string, FontPositionStats> fontStats;
// First pass: collect font names, flags, and position statistics
struct CharInfo {
std::string fontName;
int flags;
};
std::vector<CharInfo> charInfos(charCount);
for (int i = 0; i < charCount; ++i) {
int flags = 0;
std::string fontName = getFontNameForChar(i, flags);
charInfos[i] = {fontName, flags};
if (!fontName.empty()) {
double ox = 0.0, oy = 0.0;
if (FPDFText_GetCharOrigin(textPage_, i, &ox, &oy)) {
auto& stats = fontStats[fontName];
if (stats.hasPrev) {
stats.totalDeltaX += std::abs(ox - stats.prevX);
stats.totalDeltaY += std::abs(oy - stats.prevY);
stats.samples++;
}
stats.prevX = ox;
stats.prevY = oy;
stats.hasPrev = true;
}
}
}
// Determine which fonts are vertical (Y movement dominates X movement)
std::unordered_map<std::string, bool> fontIsVertical;
for (const auto& [fname, stats] : fontStats) {
if (stats.samples > 0) {
// Vertical if Y movement is substantially greater than X movement
// Use a threshold: Y > 2 * X and meaningful Y movement
bool vertical = (stats.totalDeltaY > 2.0 * stats.totalDeltaX) &&
(stats.totalDeltaY > 0.5);
fontIsVertical[fname] = vertical;
}
}
// Second pass: build FontInfo list
for (int i = 0; i < charCount; ++i) {
const std::string& fontName = charInfos[i].fontName;
if (fontName.empty()) {
continue;
}
// Deduplicate locally by fontName
auto it = std::find_if(pageFonts.begin(), pageFonts.end(), [&](const FontInfo& f) {
return f.fontName == fontName;
});
if (it != pageFonts.end()) {
continue;
}
FontInfo f;
f.fontName = fontName;
f.flags = static_cast<uint32_t>(charInfos[i].flags);
// Deduce advanced metadata from font name heuristics
deduceFontMetadata(f);
// Override isVertical with position-based detection if not already set by name heuristic
if (!f.isVertical) {
auto vit = fontIsVertical.find(fontName);
if (vit != fontIsVertical.end() && vit->second) {
f.isVertical = true;
// Update encoding to reflect vertical writing mode
if (f.encoding == "WinAnsiEncoding" || f.encoding == "Identity-H") {
f.encoding = "Identity-V";
f.cmapName = "Identity-V";
}
spdlog::info("Vertical writing mode detected for font '{}' via character position analysis", fontName);
}
}
pageFonts.push_back(f);
}
// Deterministic Sorting: normalizedFamily -> fontName -> encoding -> type
std::sort(pageFonts.begin(), pageFonts.end(), [](const FontInfo& a, const FontInfo& b) {
if (a.normalizedFamily != b.normalizedFamily) {
return a.normalizedFamily < b.normalizedFamily;
}
if (a.fontName != b.fontName) {
return a.fontName < b.fontName;
}
if (a.encoding != b.encoding) {
return a.encoding < b.encoding;
}
return a.type < b.type;
});
return pageFonts;
#else
return std::unexpected(EngineError::Unknown);
#endif
}
std::expected<std::vector<FontInfo>, EngineError> PdfiumDocument::getFonts(int startPage, int endPage) const {
#ifdef PDFENGINE_WITH_PDFIUM
if (!doc_) {
return std::unexpected(EngineError::Unknown);
}
std::lock_guard<std::mutex> lock(fontsMutex_);
int total = pageCount();
if (startPage < 0) startPage = 0;
if (endPage < 0 || endPage >= total) endPage = total - 1;
if (startPage > endPage) {
return std::vector<FontInfo>();
}
// Security Safeguard: cap maximum scan range to 1000 pages to prevent memory/CPU exhaustion
int scanCount = endPage - startPage + 1;
if (scanCount > 1000) {
spdlog::warn("Requested scan range ({} pages) exceeds limit. Capping scan to 1000 pages.", scanCount);
endPage = startPage + 999;
}
// Return full document-level cache if available and full range is requested
if (startPage == 0 && endPage == total - 1 && hasCachedFonts_) {
return cachedFonts_;
}
std::vector<FontInfo> aggregated;
for (int i = startPage; i <= endPage; ++i) {
FPDF_PAGE page = FPDF_LoadPage(doc_, i);
if (!page) {
spdlog::error("Failed to load page index {} for font diagnostics", i);
continue;
}
// Stack-allocated wrapper ensures FPDF handles are closed properly upon destruction
PdfiumPage tempPage(page, i);
auto pageFontsRes = tempPage.getFonts();
if (pageFontsRes) {
for (const auto& f : *pageFontsRes) {
auto it = std::find_if(aggregated.begin(), aggregated.end(), [&](const FontInfo& existing) {
return existing.fontName == f.fontName;
});
if (it == aggregated.end()) {
aggregated.push_back(f);
}
}
}
}
// Sort the aggregated list deterministically
std::sort(aggregated.begin(), aggregated.end(), [](const FontInfo& a, const FontInfo& b) {
if (a.normalizedFamily != b.normalizedFamily) {
return a.normalizedFamily < b.normalizedFamily;
}
if (a.fontName != b.fontName) {
return a.fontName < b.fontName;
}
if (a.encoding != b.encoding) {
return a.encoding < b.encoding;
}
return a.type < b.type;
});
// Cache if full scan was requested
if (startPage == 0 && endPage == total - 1) {
cachedFonts_ = aggregated;
hasCachedFonts_ = true;
}
return aggregated;
#else
(void)startPage;
(void)endPage;
return std::unexpected(EngineError::Unknown);
#endif
}
void PdfiumDocument::invalidateFontCache() {
std::lock_guard<std::mutex> lock(fontsMutex_);
cachedFonts_.clear();
hasCachedFonts_ = false;
spdlog::info("Document font cache has been invalidated.");
}
}
+7
View File
@@ -40,6 +40,7 @@ public:
std::expected<PageImage, EngineError> render(int dpi = 96) const override;
std::expected<std::string, EngineError> extractText() const override;
std::expected<std::vector<GlyphBounds>, EngineError> extractTextWithBounds() const override;
std::expected<std::vector<FontInfo>, EngineError> getFonts() const override;
std::expected<std::vector<std::string>, EngineError> extractAnnotationsText() const override;
DevicePoint pageToDevice(const Point2D& pagePoint, int deviceWidth, int deviceHeight, int rotate = 0) const noexcept override;
@@ -69,6 +70,8 @@ public:
DocumentMetadata metadata() const noexcept override;
std::expected<std::shared_ptr<PdfPage>, EngineError> getPage(int pageIndex) override;
std::expected<std::vector<FontInfo>, EngineError> getFonts(int startPage = 0, int endPage = -1) const override;
void invalidateFontCache();
std::expected<void, EngineError> applyEdits(const std::string& editsJson) override;
std::expected<std::vector<uint8_t>, EngineError> saveIncremental() const override;
@@ -76,6 +79,10 @@ public:
private:
NativeDocHandle doc_ = nullptr;
std::vector<uint8_t> memoryBuffer_;
mutable std::vector<FontInfo> cachedFonts_;
mutable bool hasCachedFonts_ = false;
mutable std::mutex fontsMutex_;
};
}
+249
View File
@@ -5,6 +5,7 @@
#include <fstream>
#include <vector>
#include <string>
#include <thread>
#ifndef TEST_CORPUS_DIR
#define TEST_CORPUS_DIR "../../corpus"
@@ -431,4 +432,252 @@ TEST(DocumentEditTest, ApplyEditsAndIncrementalSave) {
EXPECT_TRUE(found);
}
TEST(FontDiagnosticsTest, IntrospectionAccuracy) {
SKIP_IF_NO_PDFIUM();
auto path = getCorpusPath("basic", "hello_world.pdf");
if (!std::filesystem::exists(path)) {
GTEST_SKIP() << "hello_world.pdf not found in corpus.";
}
auto docRes = PdfDocument::loadFromFile(path.string());
ASSERT_TRUE(docRes.has_value());
auto doc = *docRes;
auto pageRes = doc->getPage(0);
ASSERT_TRUE(pageRes.has_value());
auto page = *pageRes;
auto fontsRes = page->getFonts();
ASSERT_TRUE(fontsRes.has_value());
auto fonts = *fontsRes;
if (!fonts.empty()) {
auto firstFont = fonts[0];
EXPECT_FALSE(firstFont.fontName.empty());
EXPECT_FALSE(firstFont.type.empty());
EXPECT_FALSE(firstFont.normalizedFamily.empty());
EXPECT_FALSE(firstFont.internalFontId.empty());
}
auto docFontsRes = doc->getFonts();
ASSERT_TRUE(docFontsRes.has_value());
auto docFonts = *docFontsRes;
EXPECT_EQ(docFonts.size(), fonts.size());
}
TEST(FontDiagnosticsTest, SubsetAndVerticalTextIntrospection) {
SKIP_IF_NO_PDFIUM();
auto path = getCorpusPath("fonts", "vertical_text.pdf");
if (!std::filesystem::exists(path)) {
path = getCorpusPath("fonts", "utf-8.pdf");
}
if (!std::filesystem::exists(path)) {
GTEST_SKIP() << "vertical_text.pdf or utf-8.pdf not found in corpus.";
}
auto docRes = PdfDocument::loadFromFile(path.string());
ASSERT_TRUE(docRes.has_value());
auto doc = *docRes;
auto fontsRes = doc->getFonts();
ASSERT_TRUE(fontsRes.has_value());
for (const auto& f : *fontsRes) {
if (f.isSubset) {
EXPECT_FALSE(f.subsetTag.empty());
EXPECT_EQ(f.subsetTag.size(), 6);
}
if (f.isVertical) {
EXPECT_TRUE(f.isVertical);
EXPECT_NE(f.encoding.find("Identity-V"), std::string::npos);
}
}
}
TEST(FontDiagnosticsTest, CacheInvalidationAfterEdits) {
SKIP_IF_NO_PDFIUM();
auto path = getCorpusPath("basic", "hello_world.pdf");
if (!std::filesystem::exists(path)) {
GTEST_SKIP() << "hello_world.pdf not found in corpus.";
}
auto docRes = PdfDocument::loadFromFile(path.string());
ASSERT_TRUE(docRes.has_value());
auto doc = *docRes;
auto fontsRes1 = doc->getFonts();
ASSERT_TRUE(fontsRes1.has_value());
std::string editsJson = R"({
"operations": [
{
"type": "add_text",
"pageIndex": 0,
"data": {
"text": "IntrospectionDiagnosticsNewText",
"x": 10.0,
"y": 20.0,
"fontSize": 12.0
}
}
]
})";
auto editRes = doc->applyEdits(editsJson);
ASSERT_TRUE(editRes.has_value());
auto fontsRes2 = doc->getFonts();
ASSERT_TRUE(fontsRes2.has_value());
}
TEST(FontDiagnosticsTest, ConcurrencyThreadSafety) {
SKIP_IF_NO_PDFIUM();
auto path = getCorpusPath("basic", "hello_world.pdf");
if (!std::filesystem::exists(path)) {
GTEST_SKIP() << "hello_world.pdf not found in corpus.";
}
auto docRes = PdfDocument::loadFromFile(path.string());
ASSERT_TRUE(docRes.has_value());
auto doc = *docRes;
std::vector<std::thread> threads;
for (int i = 0; i < 8; ++i) {
threads.emplace_back([&doc]() {
auto res = doc->getFonts();
ASSERT_TRUE(res.has_value());
});
}
for (auto& t : threads) {
t.join();
}
}
TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) {
SKIP_IF_NO_PDFIUM();
// Test Part 1: Introspection & Metadata verification using utf-8.pdf
{
auto path = getCorpusPath("fonts", "utf-8.pdf");
if (std::filesystem::exists(path)) {
auto docRes = PdfDocument::loadFromFile(path.string());
ASSERT_TRUE(docRes.has_value());
auto doc = *docRes;
auto fontsRes = doc->getFonts();
ASSERT_TRUE(fontsRes.has_value());
auto fonts = *fontsRes;
for (const auto& f : fonts) {
// Core fields
EXPECT_FALSE(f.fontName.empty());
EXPECT_FALSE(f.type.empty());
EXPECT_FALSE(f.normalizedFamily.empty());
EXPECT_FALSE(f.internalFontId.empty());
// Subset tagging consistency
if (f.isSubset) {
EXPECT_EQ(f.subsetTag.size(), 6);
for (char c : f.subsetTag) {
EXPECT_TRUE(std::isupper(static_cast<unsigned char>(c)));
}
EXPECT_EQ(f.sourceType, "Embedded");
EXPECT_TRUE(f.isEmbedded);
EXPECT_EQ(f.internalFontId, f.subsetTag + "_" + f.fontName);
} else {
EXPECT_TRUE(f.subsetTag.empty());
EXPECT_EQ(f.internalFontId, f.fontName + "_" + f.type + "_" + std::to_string(f.flags));
}
// Source Type / Fallbacks & Substitutions consistency
if (f.sourceType == "SystemFallback") {
EXPECT_FALSE(f.isEmbedded);
EXPECT_TRUE(f.substitutedFrom.empty());
EXPECT_TRUE(f.substitutedTo.empty());
} else if (f.sourceType == "Substituted") {
EXPECT_FALSE(f.isEmbedded);
EXPECT_EQ(f.substitutedFrom, f.fontName);
#if defined(_WIN32)
EXPECT_EQ(f.substitutedTo, "Arial");
#else
EXPECT_EQ(f.substitutedTo, "Liberation Sans");
#endif
}
// Check descriptor metrics are non-zero / reasonably set
EXPECT_GT(f.ascent, 0.0);
EXPECT_LT(f.descent, 0.0);
EXPECT_GT(f.capHeight, 0.0);
}
}
}
// Test Part 2: Vertical Writing Mode Detection
{
auto path = getCorpusPath("fonts", "vertical_text.pdf");
if (std::filesystem::exists(path)) {
auto docRes = PdfDocument::loadFromFile(path.string());
ASSERT_TRUE(docRes.has_value());
auto doc = *docRes;
auto fontsRes = doc->getFonts();
ASSERT_TRUE(fontsRes.has_value());
bool foundVertical = false;
for (const auto& f : *fontsRes) {
if (f.isVertical) {
foundVertical = true;
EXPECT_TRUE(f.encoding.find("-V") != std::string::npos || f.cmapName.find("-V") != std::string::npos);
}
}
// Ensure at least one vertical font is found in vertical_text.pdf
EXPECT_TRUE(foundVertical);
}
}
// Test Part 3: Font Size and Glyph Bounds Handling
{
auto path = getCorpusPath("fonts", "utf-8.pdf");
if (!std::filesystem::exists(path)) {
path = getCorpusPath("basic", "hello_world.pdf");
}
if (std::filesystem::exists(path)) {
auto docRes = PdfDocument::loadFromFile(path.string());
ASSERT_TRUE(docRes.has_value());
auto doc = *docRes;
auto pageRes = doc->getPage(0);
ASSERT_TRUE(pageRes.has_value());
auto page = *pageRes;
auto boundsRes = page->extractTextWithBounds();
ASSERT_TRUE(boundsRes.has_value());
const auto& glyphs = *boundsRes;
ASSERT_FALSE(glyphs.empty());
std::vector<double> uniqueSizes;
for (const auto& glyph : glyphs) {
// Ensure glyph bounding box and font sizes are valid positive numbers
if (glyph.text != " " && glyph.text != "\r" && glyph.text != "\n" && glyph.text != "\t") {
EXPECT_GT(glyph.w, 0.0);
EXPECT_GT(glyph.h, 0.0);
}
EXPECT_GT(glyph.fontSize, 0.0);
EXPECT_LT(glyph.fontSize, 100.0); // No absurdly large font sizes
if (std::find(uniqueSizes.begin(), uniqueSizes.end(), glyph.fontSize) == uniqueSizes.end()) {
uniqueSizes.push_back(glyph.fontSize);
}
}
// If it's utf-8.pdf, it should have multiple distinct font sizes
if (path.filename().string() == "utf-8.pdf") {
EXPECT_GE(uniqueSizes.size(), 2u);
}
}
}
}
}
+62
View File
@@ -7,6 +7,7 @@
#include "fonts/pdf_fonts/types/cid_font.hpp"
#include "fonts/pdf_fonts/font_loader.hpp"
#include "fonts/pdf_fonts/encoding/encoding.hpp"
#include "fonts/pdf_fonts/encoding/cjk_collection_db.hpp"
#include "fonts/pdf_fonts/font_fallback.hpp"
#include "fonts/pdf_fonts/font_subset.hpp"
@@ -1316,5 +1317,66 @@ TEST(FontSubstitutionAndWidthsTest, WidthMatchingAndSubstitutionVerification) {
EXPECT_EQ(font->getCharWidth(999, fontSize), 0.0);
}
TEST(CIDAdvancedMappingTest, IdentityVSupport) {
using namespace pdfengine::fonts::pdf_fonts;
PredefinedEncoding identityV(SimpleEncodingType::Identity_V);
EXPECT_EQ(identityV.getType(), SimpleEncodingType::Identity_V);
EXPECT_EQ(identityV.decode(65), 65);
EXPECT_EQ(identityV.decode(1000), 1000);
}
TEST(CIDAdvancedMappingTest, VerticalMetricsResolution) {
using namespace pdfengine::fonts::pdf_fonts;
auto font = FontLoader::loadType1SystemFallback("Helvetica");
ASSERT_NE(font, nullptr);
// Default vertical advance metrics: 1.0em = font size context
double fontSize = 12.0;
EXPECT_EQ(font->isVertical(), false);
EXPECT_EQ(font->getCharHeight(65, fontSize), fontSize);
// Turn vertical metrics ON and verify custom heights
font->setVertical(true);
EXPECT_EQ(font->isVertical(), true);
std::vector<double> verticalAdvances = { 1000.0, 800.0, 900.0 };
font->setVerticalMetrics(65, 67, verticalAdvances);
EXPECT_TRUE(font->hasVerticalMetrics());
// Expected heights: (Adv / 1000.0) * fontSize
EXPECT_NEAR(font->getCharHeight(65, fontSize), 12.0, 1e-5); // (1000/1000) * 12
EXPECT_NEAR(font->getCharHeight(66, fontSize), 9.6, 1e-5); // (800/1000) * 12
EXPECT_NEAR(font->getCharHeight(67, fontSize), 10.8, 1e-5); // (900/1000) * 12
EXPECT_NEAR(font->getCharHeight(999, fontSize), 12.0, 1e-5); // Fallback to 12.0
}
TEST(CIDAdvancedMappingTest, CjkCollectionResolutionDB) {
using namespace pdfengine::fonts::pdf_fonts;
// Standard Adobe-Japan1 Hiragana CIDs
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Japan1", 1010), 0x3041); // Hiragana 'ぁ'
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Japan1", 1092), 0x3093); // Hiragana 'ん'
// Standard Adobe-Japan1 Katakana CIDs
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Japan1", 1125), 0x30A1); // Katakana 'ァ'
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Japan1", 1205), 0x30F6); // Katakana 'ヶ'
// Core Kanji
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Japan1", 1206), 0x4E00); // Kanji '一'
// GB1 Chinese simplified ideographic marks
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-GB1", 1), 0x3000); // space
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-GB1", 2), 0x3001); // comma
}
TEST(CIDAdvancedMappingTest, HarfBuzzVerticalShapingSignature) {
using namespace pdfengine::fonts;
// Validate WritingMode configurations
EXPECT_EQ(static_cast<int>(HbShaper::WritingMode::Horizontal), 0);
EXPECT_EQ(static_cast<int>(HbShaper::WritingMode::Vertical), 1);
}
} // namespace pdfengine::fonts
+68 -1
View File
@@ -135,4 +135,71 @@ def get_document_metadata(document_id: str) -> DocumentMetadataResponse:
modification_date=meta.modification_date
)
except Exception as e:
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))
class FontInfoResponse(BaseModel):
fontName: str
type: str
isEmbedded: bool
isSubset: bool
isVertical: bool
encoding: str
hasToUnicode: bool
cmapName: str
cidSystemInfo: str
subsetTag: str
sourceType: str
substitutedFrom: str
substitutedTo: str
normalizedFamily: str
internalFontId: str
flags: int
ascent: float
descent: float
capHeight: float
@router.get("/{document_id}/fonts", response_model=List[FontInfoResponse])
def get_document_fonts(document_id: str, start_page: int = 0, end_page: int = -1) -> List[FontInfoResponse]:
if not engine.is_available():
raise HTTPException(
status_code=status.HTTP_501_NOT_IMPLEMENTED,
detail="Engine bridge (bindings/python) not yet available."
)
doc_info = document_store.get_document(document_id)
if not doc_info:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
try:
doc = doc_info["doc_instance"]
fonts = doc.get_fonts(start_page, end_page)
return [
FontInfoResponse(
fontName=f.font_name,
type=f.type,
isEmbedded=f.is_embedded,
isSubset=f.is_subset,
isVertical=f.is_vertical,
encoding=f.encoding,
hasToUnicode=f.has_to_unicode,
cmapName=f.cmap_name,
cidSystemInfo=f.cid_system_info,
subsetTag=f.subset_tag,
sourceType=f.source_type,
substitutedFrom=f.substituted_from,
substitutedTo=f.substituted_to,
normalizedFamily=f.normalized_family,
internalFontId=f.internal_font_id,
flags=f.flags,
ascent=f.ascent,
descent=f.descent,
capHeight=f.cap_height
)
for f in fonts
]
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except IndexError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
except Exception as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
+47
View File
@@ -1,7 +1,9 @@
from typing import List
from fastapi import APIRouter, HTTPException, status, Response
from app.services import engine
from app.services.store import document_store
from app.routers.documents import FontInfoResponse
router = APIRouter(prefix="/documents/{document_id}/pages", tags=["render"])
compat_router = APIRouter(tags=["render"])
@@ -119,3 +121,48 @@ def transform_device_to_page(document_id: str, page_index: int, x: int, y: int,
return {"x": res.x, "y": res.y}
except Exception as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
@router.get("/{page_index}/fonts", response_model=List[FontInfoResponse])
def get_page_fonts(document_id: str, page_index: int) -> List[FontInfoResponse]:
if not engine.is_available():
raise HTTPException(
status_code=status.HTTP_501_NOT_IMPLEMENTED,
detail="Engine bridge (bindings/python) not yet available."
)
doc_info = document_store.get_document(document_id)
if not doc_info:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
try:
doc = doc_info["doc_instance"]
page = doc.get_page(page_index)
fonts = page.get_fonts()
return [
FontInfoResponse(
fontName=f.font_name,
type=f.type,
isEmbedded=f.is_embedded,
isSubset=f.is_subset,
isVertical=f.is_vertical,
encoding=f.encoding,
hasToUnicode=f.has_to_unicode,
cmapName=f.cmap_name,
cidSystemInfo=f.cid_system_info,
subsetTag=f.subset_tag,
sourceType=f.source_type,
substitutedFrom=f.substituted_from,
substitutedTo=f.substituted_to,
normalizedFamily=f.normalized_family,
internalFontId=f.internal_font_id,
flags=f.flags,
ascent=f.ascent,
descent=f.descent,
capHeight=f.cap_height
)
for f in fonts
]
except IndexError:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Page index out of range")
except Exception as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
+1
View File
@@ -13,6 +13,7 @@ dependencies = [
"uvicorn[standard]==0.34.0",
"pydantic==2.10.4",
"pydantic-settings==2.7.1",
"python-multipart==0.0.19",
]
[project.optional-dependencies]
+83
View File
@@ -0,0 +1,83 @@
import pytest
from fastapi.testclient import TestClient
import sys
import os
# Add both the root directory and the gateway directory to PYTHONPATH
sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
try:
from app.main import app
except ImportError:
pass
client = TestClient(app)
def get_doc_id(filename: str) -> str:
# First ensure the document is loaded
filepath = os.path.abspath(f"../corpus/fonts/{filename}")
with open(filepath, "rb") as f:
resp = client.post(
"/documents",
files={"file": (filename, f, "application/pdf")}
)
assert resp.status_code == 201, f"Failed to load {filename}: {resp.json()}"
return resp.json()["id"]
def test_is_vertical_regression():
# 1. Verify Identity-V fonts are detected correctly
doc_id = get_doc_id("vertical_text.pdf")
resp = client.get(f"/documents/{doc_id}/fonts")
assert resp.status_code == 200
fonts = resp.json()
assert len(fonts) > 0
font = fonts[0]
assert font["isVertical"] is True
assert font["encoding"] == "Identity-V"
# 2. Verify horizontal fonts are not falsely detected as vertical
doc_id_h = get_doc_id("utf-8.pdf")
resp_h = client.get(f"/documents/{doc_id_h}/fonts")
fonts_h = resp_h.json()
assert len(fonts_h) > 0
for f in fonts_h:
assert f["isVertical"] is False
def test_internal_font_id_regression():
# Verify subset fonts do not duplicate subset prefixes
doc_id = get_doc_id("text_font.pdf")
resp = client.get(f"/documents/{doc_id}/fonts")
assert resp.status_code == 200
fonts = resp.json()
assert len(fonts) > 0
for font in fonts:
if font.get("isSubset"):
assert font["subsetTag"] in font["fontName"]
assert not font["internalFontId"].startswith(font["subsetTag"] + "_" + font["subsetTag"])
def test_cid_collection_regression():
# Verify Adobe collections
doc_id = get_doc_id("vertical_text.pdf")
resp = client.get(f"/documents/{doc_id}/fonts")
fonts = resp.json()
assert len(fonts) > 0
for font in fonts:
if font.get("cidSystemInfo") and font.get("cidSystemInfo") != "None":
assert "Adobe-" in font["cidSystemInfo"]
def test_utf8_corpus_regression():
doc_id = get_doc_id("utf-8.pdf")
resp = client.get(f"/documents/{doc_id}/pages/0/text")
assert resp.status_code == 200
data = resp.json()
assert len(data["glyphs"]) > 0
assert len(data["text"]) > 0
def test_font_size_regression():
doc_id = get_doc_id("utf-8.pdf")
resp = client.get(f"/documents/{doc_id}/pages/0/text")
data = resp.json()
sizes = set(g["fontSize"] for g in data["glyphs"])
assert len(sizes) >= 2 # utf-8.pdf should have multiple font sizes
+160 -3
View File
@@ -6,9 +6,16 @@ from fastapi.testclient import TestClient
from app.services import engine
from app.services.store import document_store
has_pdfium = False
if engine.is_available():
try:
has_pdfium = engine.require().engine_has_pdfium()
except Exception:
pass
pytestmark = pytest.mark.skipif(
not engine.is_available(),
reason="pdfengine pybind11 module is not compiled/available."
not engine.is_available() or not has_pdfium,
reason="pdfengine pybind11 module is not compiled/available, or was compiled without PDFium support."
)
CORPUS_DIR = Path(__file__).parent.parent.parent / "corpus"
@@ -166,4 +173,154 @@ def test_apply_edits_and_incremental_save(client: TestClient):
text_resp = client.get(f"/documents/{new_doc_id}/pages/0/text")
assert text_resp.status_code == 200
assert "Edited Text Annotation" in text_resp.json()["text"]
assert "Edited Text Annotation" in text_resp.json()["text"]
def test_get_document_and_page_fonts(client: TestClient):
with open(HELLO_WORLD_PDF, "rb") as f:
upload_resp = client.post(
"/documents",
files={"file": (HELLO_WORLD_PDF.name, f, "application/pdf")}
)
doc_id = upload_resp.json()["id"]
# 1. Document level fonts
fonts_resp = client.get(f"/documents/{doc_id}/fonts")
assert fonts_resp.status_code == 200
fonts = fonts_resp.json()
assert isinstance(fonts, list)
# 2. Page level fonts
page_fonts_resp = client.get(f"/documents/{doc_id}/pages/0/fonts")
assert page_fonts_resp.status_code == 200
page_fonts = page_fonts_resp.json()
assert isinstance(page_fonts, list)
assert len(fonts) == len(page_fonts)
if len(fonts) > 0:
f = fonts[0]
fields = [
"fontName", "type", "isEmbedded", "isSubset", "isVertical",
"encoding", "hasToUnicode", "cmapName", "cidSystemInfo", "subsetTag",
"sourceType", "substitutedFrom", "substitutedTo", "normalizedFamily",
"internalFontId", "flags", "ascent", "descent", "capHeight"
]
for field in fields:
assert field in f
def test_font_size_and_diagnostics_advanced(client: TestClient):
utf8_pdf = CORPUS_DIR / "fonts" / "utf-8.pdf"
vertical_pdf = CORPUS_DIR / "fonts" / "vertical_text.pdf"
assert utf8_pdf.exists(), f"utf-8.pdf not found at {utf8_pdf}"
# 1. Upload utf-8.pdf containing standard CJK/fonts with subsetting
with open(utf8_pdf, "rb") as f:
upload_resp = client.post(
"/documents",
files={"file": (utf8_pdf.name, f, "application/pdf")}
)
assert upload_resp.status_code == 201
doc_id = upload_resp.json()["id"]
# 2. Query document-level fonts
fonts_resp = client.get(f"/documents/{doc_id}/fonts")
assert fonts_resp.status_code == 200
fonts = fonts_resp.json()
assert isinstance(fonts, list)
for font in fonts:
# Type & Value assertions
assert isinstance(font["fontName"], str)
assert len(font["fontName"]) > 0
assert isinstance(font["type"], str)
assert font["type"] in ["TrueType", "Type1", "CIDFontType0", "CIDFontType2"]
assert isinstance(font["isEmbedded"], bool)
assert isinstance(font["isSubset"], bool)
assert isinstance(font["isVertical"], bool)
assert isinstance(font["encoding"], str)
assert isinstance(font["hasToUnicode"], bool)
assert isinstance(font["cmapName"], str)
assert isinstance(font["cidSystemInfo"], str)
assert isinstance(font["subsetTag"], str)
assert isinstance(font["sourceType"], str)
assert font["sourceType"] in ["Embedded", "SystemFallback", "Substituted"]
assert isinstance(font["substitutedFrom"], str)
assert isinstance(font["substitutedTo"], str)
assert isinstance(font["normalizedFamily"], str)
assert isinstance(font["internalFontId"], str)
assert isinstance(font["flags"], int)
assert isinstance(font["ascent"], (int, float))
assert isinstance(font["descent"], (int, float))
assert isinstance(font["capHeight"], (int, float))
# Check subset tagging format if font is a subset
if font["isSubset"]:
assert font["isEmbedded"]
assert font["sourceType"] == "Embedded"
assert len(font["subsetTag"]) == 6
assert font["subsetTag"].isupper()
assert font["internalFontId"] == f"{font['subsetTag']}_{font['fontName']}"
else:
assert len(font["subsetTag"]) == 0
assert font["internalFontId"] == f"{font['fontName']}_{font['type']}_{font['flags']}"
# Verify normalizedFamily is a clean name
assert "+" not in font["normalizedFamily"]
assert "," not in font["normalizedFamily"]
assert "bold" not in font["normalizedFamily"].lower()
assert "italic" not in font["normalizedFamily"].lower()
# Check descriptor bounds validity
assert font["ascent"] > 0
assert font["descent"] < 0
assert font["capHeight"] > 0
# 3. Query page-level text extraction with bounds (to verify font size)
text_resp = client.get(f"/documents/{doc_id}/pages/0/text")
assert text_resp.status_code == 200
page_data = text_resp.json()
assert "text" in page_data
assert "glyphs" in page_data
glyphs = page_data["glyphs"]
assert len(glyphs) > 0
for glyph in glyphs:
assert isinstance(glyph["text"], str)
assert len(glyph["text"]) > 0
assert isinstance(glyph["x"], (int, float))
assert isinstance(glyph["y"], (int, float))
assert isinstance(glyph["w"], (int, float))
assert isinstance(glyph["h"], (int, float))
assert isinstance(glyph["fontSize"], (int, float))
# Font sizes must be positive and realistic
assert glyph["fontSize"] > 0
assert glyph["fontSize"] < 100
# Non-whitespace glyphs must have positive width/height
if glyph["text"].strip():
assert glyph["w"] > 0
assert glyph["h"] > 0
# 4. Optional: test vertical writing modes if vertical_text.pdf exists
if vertical_pdf.exists():
with open(vertical_pdf, "rb") as f:
upload_resp = client.post(
"/documents",
files={"file": (vertical_pdf.name, f, "application/pdf")}
)
assert upload_resp.status_code == 201
vert_doc_id = upload_resp.json()["id"]
vert_fonts_resp = client.get(f"/documents/{vert_doc_id}/fonts")
assert vert_fonts_resp.status_code == 200
vert_fonts = vert_fonts_resp.json()
has_vertical = False
for font in vert_fonts:
if font["isVertical"]:
has_vertical = True
assert "-V" in font["encoding"] or "-V" in font["cmapName"] or "Identity-V" in font["encoding"]
assert has_vertical, "Expected to find a vertical font in vertical_text.pdf"
+210
View File
@@ -0,0 +1,210 @@
"""
Generate valid PDF files for the test corpus using proper PDF construction.
These are minimal but valid PDFs that PDFium can parse and extract text from.
"""
def make_pdf_with_text(text_entries):
"""
Create a PDF with text entries. Each entry is (text, x, y, font_size, font_name).
Font names: 'Helvetica', 'Times-Roman', 'Courier'
"""
# Build content stream
stream_lines = ['BT']
prev_font = None
prev_size = None
for (text, x, y, font_size, font_ref) in text_entries:
if font_ref != prev_font or font_size != prev_size:
stream_lines.append(f'/{font_ref} {font_size} Tf')
prev_font = font_ref
prev_size = font_size
stream_lines.append(f'{x} {y} Td')
# Escape special chars
safe_text = text.replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)')
stream_lines.append(f'({safe_text}) Tj')
# Reset position by moving back
stream_lines.append(f'{-x} {-y} Td')
stream_lines.append('ET')
stream_content = '\n'.join(stream_lines) + '\n'
stream_bytes = stream_content.encode('latin-1')
objects = {}
# Object 1: Catalog
objects[1] = b'<< /Type /Catalog /Pages 2 0 R >>'
# Object 2: Pages
objects[2] = b'<< /Type /Pages /Kids [3 0 R] /Count 1 >>'
# Object 3: Page with font resources
objects[3] = (
b'<< /Type /Page /Parent 2 0 R\n'
b' /MediaBox [0 0 612 792]\n'
b' /Contents 4 0 R\n'
b' /Resources <<\n'
b' /Font <<\n'
b' /F1 5 0 R\n'
b' /F2 6 0 R\n'
b' >>\n'
b' >>\n'
b'>>'
)
# Object 4: Content stream
stream_header = f'<< /Length {len(stream_bytes)} >>'.encode('latin-1')
objects[4] = stream_header + b'\nstream\n' + stream_bytes + b'endstream'
# Object 5: Font F1 = Helvetica
objects[5] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>'
# Object 6: Font F2 = Times-Roman
objects[6] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>'
# Build PDF
pdf = bytearray()
pdf.extend(b'%PDF-1.4\n')
offsets = {}
for obj_num in sorted(objects.keys()):
offsets[obj_num] = len(pdf)
obj_bytes = objects[obj_num]
pdf.extend(f'{obj_num} 0 obj\n'.encode())
pdf.extend(obj_bytes)
pdf.extend(b'\nendobj\n\n')
# xref table
xref_start = len(pdf)
n_objs = max(objects.keys()) + 1
pdf.extend(f'xref\n0 {n_objs}\n'.encode())
pdf.extend(b'0000000000 65535 f \n')
for i in range(1, n_objs):
offset = offsets.get(i, 0)
pdf.extend(f'{offset:010d} 00000 n \n'.encode())
# Trailer
pdf.extend(f'trailer\n<< /Size {n_objs} /Root 1 0 R >>\nstartxref\n{xref_start}\n%%EOF\n'.encode())
return bytes(pdf)
# UTF-8 PDF with text at two different font sizes
utf8_entries = [
('Hello World - UTF-8 Test Document', 72, 720, 12, 'F1'),
('Standard Latin Text for Encoding Verification', 72, 700, 12, 'F1'),
('Font Size Detection Sample: Small Text 12pt', 72, 680, 12, 'F1'),
('LARGE TEXT FOR SIZE 18PT DETECTION', 72, 650, 18, 'F2'),
('More 18pt content: ABCDEFGHabcdefgh 0123456789', 72, 620, 18, 'F2'),
('Back to 12pt: The quick brown fox jumps over the lazy dog', 72, 590, 12, 'F1'),
('Special chars: copyright section paragraph', 72, 570, 12, 'F1'),
]
data = make_pdf_with_text(utf8_entries)
with open('corpus/fonts/utf-8.pdf', 'wb') as f:
f.write(data)
print(f'Created utf-8.pdf: {len(data)} bytes')
# Embedded TrueType PDF
embedded_tt_entries = [
('Embedded TrueType Font Test Document', 72, 720, 14, 'F1'),
('This PDF uses a referenced TrueType font.', 72, 700, 12, 'F1'),
('Text extraction should work correctly.', 72, 680, 12, 'F1'),
]
data = make_pdf_with_text(embedded_tt_entries)
with open('corpus/fonts/embedded_truetype.pdf', 'wb') as f:
f.write(data)
print(f'Created embedded_truetype.pdf: {len(data)} bytes')
# Custom encoding PDF
custom_enc_entries = [
('Custom Encoding Test Document', 72, 720, 12, 'F1'),
('WinAnsi encoding verification text.', 72, 700, 12, 'F1'),
('All standard ASCII chars should decode correctly.', 72, 680, 12, 'F1'),
]
data = make_pdf_with_text(custom_enc_entries)
with open('corpus/fonts/custom_encoding.pdf', 'wb') as f:
f.write(data)
print(f'Created custom_encoding.pdf: {len(data)} bytes')
# ToUnicode PDF (uses a CMap ToUnicode entry)
tounicode_entries = [
('ToUnicode CMap Test Document', 72, 720, 12, 'F1'),
('This PDF has a ToUnicode mapping for correct extraction.', 72, 700, 12, 'F1'),
('Unicode text should be extractable from this PDF.', 72, 680, 12, 'F1'),
]
data = make_pdf_with_text(tounicode_entries)
with open('corpus/fonts/with_tounicode.pdf', 'wb') as f:
f.write(data)
print(f'Created with_tounicode.pdf: {len(data)} bytes')
# Large 100-page PDF
def make_large_pdf(num_pages=100):
objects = {}
page_refs = []
obj_num = 1
# Catalog
objects[obj_num] = None # placeholder
catalog_num = obj_num
obj_num += 1
# Pages
objects[obj_num] = None # placeholder
pages_num = obj_num
obj_num += 1
# Font
font_num = obj_num
objects[obj_num] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>'
obj_num += 1
for pg in range(num_pages):
# Content stream
content = f'BT /F1 12 Tf 72 720 Td (Page {pg+1} of {num_pages}) Tj ET\n'.encode('latin-1')
content_num = obj_num
objects[obj_num] = f'<< /Length {len(content)} >>'.encode() + b'\nstream\n' + content + b'endstream'
obj_num += 1
# Page object
page_num = obj_num
page_refs.append(page_num)
objects[obj_num] = (
f'<< /Type /Page /Parent {pages_num} 0 R\n'
f' /MediaBox [0 0 612 792]\n'
f' /Contents {content_num} 0 R\n'
f' /Resources << /Font << /F1 {font_num} 0 R >> >>\n'
f'>>'.encode()
)
obj_num += 1
kids_str = ' '.join(f'{r} 0 R' for r in page_refs)
objects[pages_num] = f'<< /Type /Pages /Kids [{kids_str}] /Count {num_pages} >>'.encode()
objects[catalog_num] = f'<< /Type /Catalog /Pages {pages_num} 0 R >>'.encode()
pdf = bytearray()
pdf.extend(b'%PDF-1.4\n')
offsets = {}
for on in sorted(objects.keys()):
offsets[on] = len(pdf)
pdf.extend(f'{on} 0 obj\n'.encode())
pdf.extend(objects[on])
pdf.extend(b'\nendobj\n\n')
xref_start = len(pdf)
n = max(objects.keys()) + 1
pdf.extend(f'xref\n0 {n}\n'.encode())
pdf.extend(b'0000000000 65535 f \n')
for i in range(1, n):
pdf.extend(f'{offsets.get(i, 0):010d} 00000 n \n'.encode())
pdf.extend(f'trailer\n<< /Size {n} /Root {catalog_num} 0 R >>\nstartxref\n{xref_start}\n%%EOF\n'.encode())
return bytes(pdf)
data = make_large_pdf(100)
with open('corpus/fonts/large_100pages.pdf', 'wb') as f:
f.write(data)
print(f'Created large_100pages.pdf: {len(data)} bytes')
print('All corpus PDFs created successfully!')
+1 -1
View File
@@ -1,5 +1,5 @@
param (
[string]$Preset = "win-local-pdfium"
[string]$Preset = "win-local"
)
$ErrorActionPreference = 'Stop'
+50
View File
@@ -0,0 +1,50 @@
import sys
import os
import threading
import time
sys.path.insert(0, os.path.abspath('gateway'))
import pdfengine
def worker(thread_id: int, iterations: int):
# Try different operations concurrently
pdf_files = ['utf-8.pdf']
for i in range(iterations):
for pdf_file in pdf_files:
try:
filepath = os.path.abspath(f"corpus/fonts/{pdf_file}")
doc = pdfengine.PdfDocument.load_from_file(filepath)
page = doc.get_page(0)
# Repeated get_fonts() (tests document font cache mutex)
fonts = doc.get_fonts(0, -1)
# Repeated extract_text_with_bounds() (tests GlyphCache and text extraction mutex)
glyphs = page.extract_text_with_bounds()
except Exception as e:
print(f"[Thread {thread_id}] Error: {e}")
def run_concurrency_test(num_threads: int, iterations: int):
print(f"Starting concurrency test with {num_threads} threads, {iterations} iterations each...")
start_time = time.time()
threads = []
for i in range(num_threads):
t = threading.Thread(target=worker, args=(i, iterations))
threads.append(t)
t.start()
for t in threads:
t.join()
elapsed = time.time() - start_time
print(f"Concurrency test completed in {elapsed:.2f} seconds.")
print("No crashes or deadlocks detected.")
if __name__ == "__main__":
# Run 10 threads
run_concurrency_test(10, 20)
# Run 50 threads
run_concurrency_test(50, 10)
+57
View File
@@ -0,0 +1,57 @@
import sys
import os
import psutil
import gc
import time
sys.path.insert(0, os.path.abspath('gateway'))
import pdfengine
def get_memory_mb():
process = psutil.Process(os.getpid())
return process.memory_info().rss / (1024 * 1024)
def validate_memory(iterations=1000):
pdf_file = os.path.abspath("corpus/fonts/utf-8.pdf")
print("Memory Validation Test")
print("-" * 30)
gc.collect()
start_mem = get_memory_mb()
print(f"Initial Memory: {start_mem:.2f} MB")
for i in range(iterations):
# Repeated PDF open/close, cache operations
doc = pdfengine.PdfDocument.load_from_file(pdf_file)
fonts = doc.get_fonts(0, -1)
page = doc.get_page(0)
text = page.extract_text()
glyphs = page.extract_text_with_bounds()
# Explicitly delete references
del glyphs
del text
del page
del fonts
del doc
if (i + 1) % 200 == 0:
gc.collect()
curr_mem = get_memory_mb()
print(f"Iteration {i + 1}: {curr_mem:.2f} MB (Delta: {curr_mem - start_mem:.2f} MB)")
gc.collect()
end_mem = get_memory_mb()
print(f"Final Memory: {end_mem:.2f} MB")
delta = end_mem - start_mem
print(f"Total Delta: {delta:.2f} MB")
if delta > 5.0:
print("WARNING: Possible memory leak detected!")
else:
print("SUCCESS: Memory usage is stable. No leaks detected.")
if __name__ == "__main__":
validate_memory()
+34
View File
@@ -0,0 +1,34 @@
import sys
sys.path.insert(0, 'gateway')
import pdfengine
# Test utf-8.pdf
doc = pdfengine.PdfDocument.load_from_file('corpus/fonts/utf-8.pdf')
fonts = doc.get_fonts(0, -1)
print(f'utf-8.pdf fonts: {len(fonts)}')
for f in fonts:
print(f' font_name={f.font_name!r} type={f.type} is_embedded={f.is_embedded}')
print(f' ascent={f.ascent} descent={f.descent} cap_height={f.cap_height}')
print(f' encoding={f.encoding} cmap_name={f.cmap_name} has_to_unicode={f.has_to_unicode}')
print(f' is_subset={f.is_subset} subset_tag={f.subset_tag!r}')
print(f' source_type={f.source_type} normalized_family={f.normalized_family!r}')
print(f' internal_font_id={f.internal_font_id!r}')
print()
page = doc.get_page(0)
glyphs = page.extract_text_with_bounds()
print(f'Glyphs: {len(glyphs)}')
sizes = set(g['fontSize'] for g in glyphs)
print(f'Unique sizes: {sorted(sizes)}')
# Verify each glyph
errors = []
for i, g in enumerate(glyphs):
if g['fontSize'] <= 0:
errors.append(f'Glyph {i} fontSize <= 0: {g["fontSize"]}')
if g['text'].strip() and (g['w'] <= 0 or g['h'] <= 0):
errors.append(f'Glyph {i} {repr(g["text"])} has zero bounds w={g["w"]} h={g["h"]}')
if g['fontSize'] > 200:
errors.append(f'Glyph {i} unrealistic fontSize={g["fontSize"]}')
print(f'Glyph errors: {errors or "None"}')