Merge pull request 'saqib' (#26) from saqib into dev
Reviewed-on: https://gitea.maskantech.in/gitea_admin/pdf/pulls/26
This commit is contained in:
+207
@@ -0,0 +1,207 @@
|
||||
"""Live API validation script for the PDF engine gateway."""
|
||||
import sys, json, urllib.request, urllib.parse
|
||||
sys.path.insert(0, 'gateway')
|
||||
|
||||
BASE = 'http://localhost:8000'
|
||||
|
||||
def http_get(url):
|
||||
try:
|
||||
resp = urllib.request.urlopen(url)
|
||||
return json.loads(resp.read()), resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
return json.loads(e.read()), e.code
|
||||
|
||||
def multipart_upload(url, filepath, filename):
|
||||
boundary = 'Xboundary1234X'
|
||||
with open(filepath, 'rb') as f:
|
||||
file_data = f.read()
|
||||
header = (
|
||||
'--' + boundary + '\r\n'
|
||||
'Content-Disposition: form-data; name="file"; filename="' + filename + '"\r\n'
|
||||
'Content-Type: application/pdf\r\n\r\n'
|
||||
).encode('utf-8')
|
||||
footer = ('\r\n--' + boundary + '--\r\n').encode('utf-8')
|
||||
body = header + file_data + footer
|
||||
req = urllib.request.Request(
|
||||
url, data=body,
|
||||
headers={'Content-Type': 'multipart/form-data; boundary=' + boundary},
|
||||
method='POST'
|
||||
)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req)
|
||||
return json.loads(resp.read()), resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
return json.loads(e.read()), e.code
|
||||
|
||||
results = {}
|
||||
|
||||
# 1. Health check
|
||||
health, status = http_get(f'{BASE}/health')
|
||||
print(f"[HEALTH] status={status} response={health}")
|
||||
results['health'] = {'status': status, 'passed': status == 200}
|
||||
|
||||
# 2. Upload hello_world.pdf
|
||||
print("\n=== UPLOADING hello_world.pdf ===")
|
||||
doc, status = multipart_upload(f'{BASE}/documents', 'corpus/basic/hello_world.pdf', 'hello_world.pdf')
|
||||
print(f"Upload status={status}")
|
||||
print(f"Response: {doc}")
|
||||
doc_id = doc.get('id')
|
||||
results['upload'] = {'status': status, 'passed': status == 201, 'doc_id': doc_id}
|
||||
|
||||
if not doc_id:
|
||||
print("FATAL: No doc_id, cannot continue")
|
||||
sys.exit(1)
|
||||
|
||||
# 3. GET /documents/{id}/fonts
|
||||
print(f"\n=== GET /documents/{doc_id}/fonts ===")
|
||||
fonts, status = http_get(f'{BASE}/documents/{doc_id}/fonts')
|
||||
print(f"Status={status}, Font count={len(fonts)}")
|
||||
results['doc_fonts'] = {'status': status, 'count': len(fonts), 'passed': status == 200}
|
||||
|
||||
ALL_FONT_FIELDS = [
|
||||
'fontName','type','isEmbedded','isSubset','isVertical',
|
||||
'encoding','hasToUnicode','cmapName','cidSystemInfo','subsetTag',
|
||||
'sourceType','substitutedFrom','substitutedTo','normalizedFamily',
|
||||
'internalFontId','flags','ascent','descent','capHeight'
|
||||
]
|
||||
|
||||
schema_errors = []
|
||||
for f in fonts:
|
||||
missing = [k for k in ALL_FONT_FIELDS if k not in f]
|
||||
if missing:
|
||||
schema_errors.append(f"Missing fields: {missing} in font {f.get('fontName')}")
|
||||
# Type checks
|
||||
for boolField in ['isEmbedded','isSubset','isVertical','hasToUnicode']:
|
||||
if not isinstance(f.get(boolField), bool):
|
||||
schema_errors.append(f"Field {boolField} should be bool, got {type(f.get(boolField))}")
|
||||
for floatField in ['ascent','descent','capHeight']:
|
||||
if not isinstance(f.get(floatField), (int, float)):
|
||||
schema_errors.append(f"Field {floatField} should be float")
|
||||
if not isinstance(f.get('flags'), int):
|
||||
schema_errors.append("Field flags should be int")
|
||||
print(f" Font: {f.get('fontName'):30s} type={f.get('type'):15s} embedded={f.get('isEmbedded')} subset={f.get('isSubset')} vertical={f.get('isVertical')}")
|
||||
print(f" encoding={f.get('encoding')} cmapName={f.get('cmapName')} cidSystemInfo={f.get('cidSystemInfo')}")
|
||||
print(f" subsetTag={f.get('subsetTag')} sourceType={f.get('sourceType')} normalizedFamily={f.get('normalizedFamily')}")
|
||||
print(f" internalFontId={f.get('internalFontId')} flags={f.get('flags')}")
|
||||
print(f" ascent={f.get('ascent')} descent={f.get('descent')} capHeight={f.get('capHeight')}")
|
||||
|
||||
results['doc_fonts']['schema_errors'] = schema_errors
|
||||
print(f"Schema errors: {schema_errors or 'None'}")
|
||||
|
||||
# 4. GET /documents/{id}/pages/0/fonts
|
||||
print(f"\n=== GET /documents/{doc_id}/pages/0/fonts ===")
|
||||
pg_fonts, status = http_get(f'{BASE}/documents/{doc_id}/pages/0/fonts')
|
||||
print(f"Status={status}, Page font count={len(pg_fonts)}")
|
||||
results['page_fonts'] = {'status': status, 'count': len(pg_fonts), 'passed': status == 200}
|
||||
|
||||
# Consistency check: doc-level vs page-level
|
||||
if len(fonts) != len(pg_fonts):
|
||||
print(f"WARNING: doc-level fonts ({len(fonts)}) != page-level fonts ({len(pg_fonts)})")
|
||||
results['page_fonts']['consistency_warning'] = True
|
||||
else:
|
||||
print("Consistency: doc-level and page-level font counts match OK")
|
||||
|
||||
# 5. GET /documents/{id}/pages/0/text
|
||||
print(f"\n=== GET /documents/{doc_id}/pages/0/text ===")
|
||||
text_data, status = http_get(f'{BASE}/documents/{doc_id}/pages/0/text')
|
||||
print(f"Status={status}")
|
||||
results['page_text'] = {'status': status, 'passed': status == 200}
|
||||
|
||||
text = text_data.get('text', '')
|
||||
glyphs = text_data.get('glyphs', [])
|
||||
print(f"Text: {repr(text[:80])}")
|
||||
print(f"Glyph count: {len(glyphs)}")
|
||||
|
||||
glyph_errors = []
|
||||
for i, g in enumerate(glyphs):
|
||||
for field in ['text','x','y','w','h','fontSize']:
|
||||
if field not in g:
|
||||
glyph_errors.append(f"Glyph {i} missing field {field}")
|
||||
if g.get('fontSize', 0) <= 0:
|
||||
glyph_errors.append(f"Glyph {i} '{g.get('text')}' has fontSize <= 0: {g.get('fontSize')}")
|
||||
if g.get('text','').strip():
|
||||
if g.get('w', 0) <= 0 or g.get('h', 0) <= 0:
|
||||
glyph_errors.append(f"Glyph {i} '{g.get('text')}' has zero bounds w={g.get('w')} h={g.get('h')}")
|
||||
if g.get('fontSize', 0) > 200:
|
||||
glyph_errors.append(f"Glyph {i} unrealistic fontSize={g.get('fontSize')}")
|
||||
|
||||
results['page_text']['glyph_count'] = len(glyphs)
|
||||
results['page_text']['glyph_errors'] = glyph_errors
|
||||
print(f"Glyph validation errors: {glyph_errors or 'None'}")
|
||||
if glyphs:
|
||||
print(f"Sample glyphs: {glyphs[:3]}")
|
||||
|
||||
# 6. Test 404 for non-existent document
|
||||
print("\n=== TEST 404 ===")
|
||||
not_found, status = http_get(f'{BASE}/documents/nonexistent-uuid/fonts')
|
||||
print(f"404 test status={status} (expected 404)")
|
||||
results['not_found'] = {'status': status, 'passed': status == 404}
|
||||
|
||||
# 7. Test page out of bounds
|
||||
print("\n=== TEST PAGE OUT OF BOUNDS ===")
|
||||
oob, status = http_get(f'{BASE}/documents/{doc_id}/pages/99/fonts')
|
||||
print(f"OOB test status={status} (expected 4xx)")
|
||||
results['oob'] = {'status': status, 'passed': status in (400, 404)}
|
||||
|
||||
# 8. Upload vertical_text.pdf and check vertical detection
|
||||
print("\n=== UPLOADING vertical_text.pdf ===")
|
||||
vert_doc, status = multipart_upload(f'{BASE}/documents', 'corpus/fonts/vertical_text.pdf', 'vertical_text.pdf')
|
||||
vert_id = vert_doc.get('id')
|
||||
print(f"Upload status={status} doc_id={vert_id}")
|
||||
if vert_id:
|
||||
vert_fonts, status = http_get(f'{BASE}/documents/{vert_id}/fonts')
|
||||
print(f"Vertical PDF fonts ({len(vert_fonts)}):")
|
||||
for vf in vert_fonts:
|
||||
print(f" fontName={vf['fontName']} isVertical={vf['isVertical']} encoding={vf['encoding']}")
|
||||
|
||||
# 9. Upload latin_extended.pdf and validate metrics
|
||||
print("\n=== UPLOADING latin_extended.pdf ===")
|
||||
lat_doc, status = multipart_upload(f'{BASE}/documents', 'corpus/fonts/latin_extended.pdf', 'latin_extended.pdf')
|
||||
lat_id = lat_doc.get('id')
|
||||
if lat_id:
|
||||
lat_fonts, st = http_get(f'{BASE}/documents/{lat_id}/fonts')
|
||||
print(f"Latin extended fonts ({len(lat_fonts)}):")
|
||||
for lf in lat_fonts:
|
||||
ascent_ok = lf['ascent'] > 0
|
||||
descent_ok = lf['descent'] < 0
|
||||
cap_ok = lf['capHeight'] > 0
|
||||
print(f" {lf['fontName']} ascent={lf['ascent']}({'OK' if ascent_ok else 'FAIL'}) descent={lf['descent']}({'OK' if descent_ok else 'FAIL'}) capHeight={lf['capHeight']}({'OK' if cap_ok else 'FAIL'})")
|
||||
|
||||
# 10. Repeated requests (cache consistency)
|
||||
print("\n=== CACHE CONSISTENCY (3 repeated font requests) ===")
|
||||
responses = []
|
||||
for _ in range(3):
|
||||
fonts_rep, st = http_get(f'{BASE}/documents/{doc_id}/fonts')
|
||||
responses.append(len(fonts_rep))
|
||||
print(f"Font counts on repeated requests: {responses}")
|
||||
results['cache_consistency'] = {'counts': responses, 'passed': len(set(responses)) == 1}
|
||||
|
||||
# 11. Invalid PDF upload
|
||||
print("\n=== INVALID PDF UPLOAD ===")
|
||||
boundary = 'Xboundary1234X'
|
||||
garbage = b'NOT A PDF FILE AT ALL 12345'
|
||||
header = ('--' + boundary + '\r\nContent-Disposition: form-data; name="file"; filename="bad.pdf"\r\nContent-Type: application/pdf\r\n\r\n').encode()
|
||||
footer = ('\r\n--' + boundary + '--\r\n').encode()
|
||||
req = urllib.request.Request(
|
||||
f'{BASE}/documents',
|
||||
data=header + garbage + footer,
|
||||
headers={'Content-Type': 'multipart/form-data; boundary=' + boundary},
|
||||
method='POST'
|
||||
)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req)
|
||||
bad_result = json.loads(resp.read()), resp.status
|
||||
print(f"UNEXPECTED SUCCESS: {bad_result}")
|
||||
results['invalid_upload'] = {'passed': False}
|
||||
except urllib.error.HTTPError as e:
|
||||
err_detail = json.loads(e.read())
|
||||
print(f"Invalid PDF correctly rejected: status={e.code} detail={err_detail}")
|
||||
results['invalid_upload'] = {'status': e.code, 'passed': e.code == 400}
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("VALIDATION SUMMARY")
|
||||
print("="*60)
|
||||
for test, res in results.items():
|
||||
passed = res.get('passed', '?')
|
||||
status = res.get('status', '-')
|
||||
print(f" {'PASS' if passed else 'FAIL'}: {test:35s} status={status}")
|
||||
@@ -87,6 +87,30 @@ PYBIND11_MODULE(pdfengine, m) {
|
||||
return py::bytes(reinterpret_cast<const char*>(self.data.data()), self.data.size());
|
||||
});
|
||||
|
||||
py::class_<pdfengine::FontInfo>(m, "FontInfo")
|
||||
.def_readonly("font_name", &pdfengine::FontInfo::fontName)
|
||||
.def_readonly("type", &pdfengine::FontInfo::type)
|
||||
.def_readonly("is_embedded", &pdfengine::FontInfo::isEmbedded)
|
||||
.def_readonly("is_subset", &pdfengine::FontInfo::isSubset)
|
||||
.def_readonly("is_vertical", &pdfengine::FontInfo::isVertical)
|
||||
.def_readonly("encoding", &pdfengine::FontInfo::encoding)
|
||||
.def_readonly("has_to_unicode", &pdfengine::FontInfo::hasToUnicode)
|
||||
.def_readonly("cmap_name", &pdfengine::FontInfo::cmapName)
|
||||
.def_readonly("cid_system_info", &pdfengine::FontInfo::cidSystemInfo)
|
||||
.def_readonly("subset_tag", &pdfengine::FontInfo::subsetTag)
|
||||
.def_readonly("source_type", &pdfengine::FontInfo::sourceType)
|
||||
.def_readonly("substituted_from", &pdfengine::FontInfo::substitutedFrom)
|
||||
.def_readonly("substituted_to", &pdfengine::FontInfo::substitutedTo)
|
||||
.def_readonly("normalized_family", &pdfengine::FontInfo::normalizedFamily)
|
||||
.def_readonly("internal_font_id", &pdfengine::FontInfo::internalFontId)
|
||||
.def_readonly("flags", &pdfengine::FontInfo::flags)
|
||||
.def_readonly("ascent", &pdfengine::FontInfo::ascent)
|
||||
.def_readonly("descent", &pdfengine::FontInfo::descent)
|
||||
.def_readonly("cap_height", &pdfengine::FontInfo::capHeight)
|
||||
.def("__repr__", [](const pdfengine::FontInfo& self) {
|
||||
return "FontInfo(font_name='" + self.fontName + "', type='" + self.type + "', is_embedded=" + (self.isEmbedded ? "True" : "False") + ")";
|
||||
});
|
||||
|
||||
py::class_<pdfengine::PdfPage, std::shared_ptr<pdfengine::PdfPage>>(m, "PdfPage")
|
||||
.def_property_readonly("width", &pdfengine::PdfPage::width)
|
||||
.def_property_readonly("height", &pdfengine::PdfPage::height)
|
||||
@@ -114,6 +138,9 @@ PYBIND11_MODULE(pdfengine, m) {
|
||||
}
|
||||
return py_list;
|
||||
})
|
||||
.def("get_fonts", [](const pdfengine::PdfPage& self) {
|
||||
return get_or_throw(self.getFonts());
|
||||
})
|
||||
.def("page_to_device", &pdfengine::PdfPage::pageToDevice,
|
||||
py::arg("page_point"), py::arg("device_width"), py::arg("device_height"), py::arg("rotate") = 0)
|
||||
.def("device_to_page", &pdfengine::PdfPage::deviceToPage,
|
||||
@@ -133,6 +160,9 @@ PYBIND11_MODULE(pdfengine, m) {
|
||||
.def("get_page", [](pdfengine::PdfDocument& self, int pageIndex) {
|
||||
return get_or_throw(self.getPage(pageIndex));
|
||||
}, py::arg("page_index"))
|
||||
.def("get_fonts", [](const pdfengine::PdfDocument& self, int start_page, int end_page) {
|
||||
return get_or_throw(self.getFonts(start_page, end_page));
|
||||
}, py::arg("start_page") = 0, py::arg("end_page") = -1)
|
||||
.def("apply_edits", [](pdfengine::PdfDocument& self, const std::string& editsJson) {
|
||||
get_or_throw(self.applyEdits(editsJson));
|
||||
}, py::arg("edits_json"))
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
%PDF-1.4
|
||||
1 0 obj
|
||||
<< /Type /Catalog /Pages 2 0 R >>
|
||||
endobj
|
||||
|
||||
2 0 obj
|
||||
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
|
||||
endobj
|
||||
|
||||
3 0 obj
|
||||
<< /Type /Page /Parent 2 0 R
|
||||
/MediaBox [0 0 612 792]
|
||||
/Contents 4 0 R
|
||||
/Resources <<
|
||||
/Font <<
|
||||
/F1 5 0 R
|
||||
/F2 6 0 R
|
||||
>>
|
||||
>>
|
||||
>>
|
||||
endobj
|
||||
|
||||
4 0 obj
|
||||
<< /Length 213 >>
|
||||
stream
|
||||
BT
|
||||
/F1 12 Tf
|
||||
72 720 Td
|
||||
(Custom Encoding Test Document) Tj
|
||||
-72 -720 Td
|
||||
72 700 Td
|
||||
(WinAnsi encoding verification text.) Tj
|
||||
-72 -700 Td
|
||||
72 680 Td
|
||||
(All standard ASCII chars should decode correctly.) Tj
|
||||
-72 -680 Td
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
|
||||
5 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
|
||||
6 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
|
||||
xref
|
||||
0 7
|
||||
0000000000 65535 f
|
||||
0000000009 00000 n
|
||||
0000000059 00000 n
|
||||
0000000117 00000 n
|
||||
0000000290 00000 n
|
||||
0000000554 00000 n
|
||||
0000000652 00000 n
|
||||
trailer
|
||||
<< /Size 7 /Root 1 0 R >>
|
||||
startxref
|
||||
752
|
||||
%%EOF
|
||||
@@ -0,0 +1 @@
|
||||
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 58 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(CID Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type0 /BaseFont /NotoSansCJKjp-Regular /Encoding /Identity-H /DescendantFonts [6 0 R] >>\nendobj\n\n6 0 obj\n<< /Type /Font /Subtype /CIDFontType2 /BaseFont /NotoSansCJKjp-Regular /CIDSystemInfo << /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >> >>\nendobj\n\nxref\n0 7\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000384 00000 n \n0000000518 00000 n \ntrailer\n<< /Size 7 /Root 1 0 R >>\nstartxref\n684\n%%EOF\n
|
||||
@@ -0,0 +1,63 @@
|
||||
%PDF-1.4
|
||||
1 0 obj
|
||||
<< /Type /Catalog /Pages 2 0 R >>
|
||||
endobj
|
||||
|
||||
2 0 obj
|
||||
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
|
||||
endobj
|
||||
|
||||
3 0 obj
|
||||
<< /Type /Page /Parent 2 0 R
|
||||
/MediaBox [0 0 612 792]
|
||||
/Contents 4 0 R
|
||||
/Resources <<
|
||||
/Font <<
|
||||
/F1 5 0 R
|
||||
/F2 6 0 R
|
||||
>>
|
||||
>>
|
||||
>>
|
||||
endobj
|
||||
|
||||
4 0 obj
|
||||
<< /Length 225 >>
|
||||
stream
|
||||
BT
|
||||
/F1 14 Tf
|
||||
72 720 Td
|
||||
(Embedded TrueType Font Test Document) Tj
|
||||
-72 -720 Td
|
||||
/F1 12 Tf
|
||||
72 700 Td
|
||||
(This PDF uses a referenced TrueType font.) Tj
|
||||
-72 -700 Td
|
||||
72 680 Td
|
||||
(Text extraction should work correctly.) Tj
|
||||
-72 -680 Td
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
|
||||
5 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
|
||||
6 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
|
||||
xref
|
||||
0 7
|
||||
0000000000 65535 f
|
||||
0000000009 00000 n
|
||||
0000000059 00000 n
|
||||
0000000117 00000 n
|
||||
0000000290 00000 n
|
||||
0000000566 00000 n
|
||||
0000000664 00000 n
|
||||
trailer
|
||||
<< /Size 7 /Root 1 0 R >>
|
||||
startxref
|
||||
764
|
||||
%%EOF
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 64 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Malformed Font) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /UnknownType /BaseFont /BrokenFont >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000390 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n472\n%%EOF\n
|
||||
@@ -0,0 +1 @@
|
||||
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 60 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Page 1 Mix) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000386 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n488\n%%EOF\n
|
||||
@@ -0,0 +1 @@
|
||||
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 66 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(No ToUnicode Map) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Symbol >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000392 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n464\n%%EOF\n
|
||||
@@ -0,0 +1 @@
|
||||
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 61 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Subset Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /TrueType /BaseFont /ABCDEF+Arial /FirstChar 32 /LastChar 126 >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000387 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n496\n%%EOF\n
|
||||
+61
-54
@@ -1,69 +1,76 @@
|
||||
%PDF-1.7
|
||||
% ò¤ô
|
||||
1 0 obj <<
|
||||
/Type /Catalog
|
||||
/Pages 2 0 R
|
||||
/Outlines 6 0 R
|
||||
%PDF-1.4
|
||||
1 0 obj
|
||||
<< /Type /Catalog /Pages 2 0 R >>
|
||||
endobj
|
||||
|
||||
2 0 obj
|
||||
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
|
||||
endobj
|
||||
|
||||
3 0 obj
|
||||
<< /Type /Page /Parent 2 0 R
|
||||
/MediaBox [0 0 612 792]
|
||||
/Contents 4 0 R
|
||||
/Resources <<
|
||||
/Font <<
|
||||
/F1 5 0 R
|
||||
/F2 6 0 R
|
||||
>>
|
||||
>>
|
||||
>>
|
||||
endobj
|
||||
|
||||
2 0 obj <<
|
||||
/Type /Pages
|
||||
/Kids [3 0 R]
|
||||
/Count 1
|
||||
>>
|
||||
endobj
|
||||
|
||||
3 0 obj <<
|
||||
/Type /Page
|
||||
/Parent 2 0 R
|
||||
/MediaBox [0 0 525 250]
|
||||
/Contents 4 0 R
|
||||
>>
|
||||
endobj
|
||||
|
||||
4 0 obj <<
|
||||
/Length 0
|
||||
>>
|
||||
4 0 obj
|
||||
<< /Length 532 >>
|
||||
stream
|
||||
BT
|
||||
/F1 12 Tf
|
||||
72 720 Td
|
||||
(Hello World - UTF-8 Test Document) Tj
|
||||
-72 -720 Td
|
||||
72 700 Td
|
||||
(Standard Latin Text for Encoding Verification) Tj
|
||||
-72 -700 Td
|
||||
72 680 Td
|
||||
(Font Size Detection Sample: Small Text 12pt) Tj
|
||||
-72 -680 Td
|
||||
/F2 18 Tf
|
||||
72 650 Td
|
||||
(LARGE TEXT FOR SIZE 18PT DETECTION) Tj
|
||||
-72 -650 Td
|
||||
72 620 Td
|
||||
(More 18pt content: ABCDEFGHabcdefgh 0123456789) Tj
|
||||
-72 -620 Td
|
||||
/F1 12 Tf
|
||||
72 590 Td
|
||||
(Back to 12pt: The quick brown fox jumps over the lazy dog) Tj
|
||||
-72 -590 Td
|
||||
72 570 Td
|
||||
(Special chars: copyright section paragraph) Tj
|
||||
-72 -570 Td
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
|
||||
5 0 obj <<
|
||||
/Producer (\357\273\277Man\303\274ally Created)
|
||||
>>
|
||||
5 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
|
||||
6 0 obj <<
|
||||
/Count 1
|
||||
/First 7 0 R
|
||||
/Last 7 0 R
|
||||
>>
|
||||
endobj
|
||||
|
||||
7 0 obj <<
|
||||
/Title <EFBBBF5469746CC3A82031>
|
||||
/Parent 6 0 R
|
||||
>>
|
||||
6 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
|
||||
xref
|
||||
0 8
|
||||
0 7
|
||||
0000000000 65535 f
|
||||
0000000015 00000 n
|
||||
0000000087 00000 n
|
||||
0000000151 00000 n
|
||||
0000000247 00000 n
|
||||
0000000298 00000 n
|
||||
0000000370 00000 n
|
||||
0000000432 00000 n
|
||||
|
||||
trailer <<
|
||||
/Size 8
|
||||
/Info 5 0 R
|
||||
/Root 1 0 R
|
||||
>>
|
||||
|
||||
0000000009 00000 n
|
||||
0000000059 00000 n
|
||||
0000000117 00000 n
|
||||
0000000290 00000 n
|
||||
0000000873 00000 n
|
||||
0000000971 00000 n
|
||||
trailer
|
||||
<< /Size 7 /Root 1 0 R >>
|
||||
startxref
|
||||
504
|
||||
1071
|
||||
%%EOF
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 63 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Vertical Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type0 /BaseFont /KozMinPro-Regular-Identity-V /Encoding /Identity-V >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000389 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n505\n%%EOF\n
|
||||
@@ -0,0 +1,62 @@
|
||||
%PDF-1.4
|
||||
1 0 obj
|
||||
<< /Type /Catalog /Pages 2 0 R >>
|
||||
endobj
|
||||
|
||||
2 0 obj
|
||||
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
|
||||
endobj
|
||||
|
||||
3 0 obj
|
||||
<< /Type /Page /Parent 2 0 R
|
||||
/MediaBox [0 0 612 792]
|
||||
/Contents 4 0 R
|
||||
/Resources <<
|
||||
/Font <<
|
||||
/F1 5 0 R
|
||||
/F2 6 0 R
|
||||
>>
|
||||
>>
|
||||
>>
|
||||
endobj
|
||||
|
||||
4 0 obj
|
||||
<< /Length 233 >>
|
||||
stream
|
||||
BT
|
||||
/F1 12 Tf
|
||||
72 720 Td
|
||||
(ToUnicode CMap Test Document) Tj
|
||||
-72 -720 Td
|
||||
72 700 Td
|
||||
(This PDF has a ToUnicode mapping for correct extraction.) Tj
|
||||
-72 -700 Td
|
||||
72 680 Td
|
||||
(Unicode text should be extractable from this PDF.) Tj
|
||||
-72 -680 Td
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
|
||||
5 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
|
||||
6 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
|
||||
endobj
|
||||
|
||||
xref
|
||||
0 7
|
||||
0000000000 65535 f
|
||||
0000000009 00000 n
|
||||
0000000059 00000 n
|
||||
0000000117 00000 n
|
||||
0000000290 00000 n
|
||||
0000000574 00000 n
|
||||
0000000672 00000 n
|
||||
trailer
|
||||
<< /Size 7 /Root 1 0 R >>
|
||||
startxref
|
||||
772
|
||||
%%EOF
|
||||
@@ -29,6 +29,7 @@ add_library(pdfengine STATIC
|
||||
src/fonts/pdf_fonts/font_subset.cpp
|
||||
src/fonts/pdf_fonts/encoding/encoding.cpp
|
||||
src/fonts/pdf_fonts/encoding/tounicode_parser.cpp
|
||||
src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp
|
||||
)
|
||||
add_library(pdfengine::pdfengine ALIAS pdfengine)
|
||||
|
||||
|
||||
@@ -53,6 +53,30 @@ struct GlyphBounds {
|
||||
double fontSize;
|
||||
};
|
||||
|
||||
struct FontInfo {
|
||||
std::string fontName;
|
||||
std::string type; // "TrueType", "Type1", "CIDFontType0", "CIDFontType2"
|
||||
bool isEmbedded = false;
|
||||
bool isSubset = false;
|
||||
bool isVertical = false;
|
||||
|
||||
// Advanced Introspection & Diagnostics
|
||||
std::string encoding; // "WinAnsiEncoding", "MacRomanEncoding", "Identity-H", "Identity-V", "Symbol", "Custom", "None"
|
||||
bool hasToUnicode = false; // True if font has an active /ToUnicode map
|
||||
std::string cmapName; // e.g. "Identity-H", "Identity-V", "UniJIS-UTF16-H"
|
||||
std::string cidSystemInfo; // e.g. "Adobe-Japan1", "Adobe-GB1", "Adobe-Korea1"
|
||||
std::string subsetTag; // e.g. "ABCDEE" (6-character uppercase tag)
|
||||
std::string sourceType; // "Embedded", "SystemFallback", "Substituted"
|
||||
std::string substitutedFrom; // e.g. "Helvetica" (Original requested font)
|
||||
std::string substitutedTo; // e.g. "Liberation Sans" (Actual fallback font used)
|
||||
std::string normalizedFamily; // e.g. "Arial" (Normalized family grouping name)
|
||||
std::string internalFontId; // Unique stable identifier for internal tracking
|
||||
uint32_t flags = 0; // PDF font descriptor flags
|
||||
double ascent = 0.0; // Font descriptor Ascent metric
|
||||
double descent = 0.0; // Font descriptor Descent metric
|
||||
double capHeight = 0.0; // Font descriptor CapHeight metric
|
||||
};
|
||||
|
||||
class PdfPage {
|
||||
public:
|
||||
virtual ~PdfPage() = default;
|
||||
@@ -66,6 +90,7 @@ public:
|
||||
|
||||
[[nodiscard]] virtual std::expected<std::vector<GlyphBounds>, EngineError> extractTextWithBounds() const = 0;
|
||||
|
||||
[[nodiscard]] virtual std::expected<std::vector<FontInfo>, EngineError> getFonts() const = 0;
|
||||
[[nodiscard]] virtual std::expected<std::vector<std::string>, EngineError> extractAnnotationsText() const = 0;
|
||||
|
||||
[[nodiscard]] virtual DevicePoint pageToDevice(const Point2D& pagePoint, int deviceWidth, int deviceHeight, int rotate = 0) const noexcept = 0;
|
||||
@@ -88,6 +113,9 @@ public:
|
||||
[[nodiscard]] virtual std::expected<std::shared_ptr<PdfPage>, EngineError>
|
||||
getPage(int pageIndex) = 0;
|
||||
|
||||
[[nodiscard]] virtual std::expected<std::vector<FontInfo>, EngineError>
|
||||
getFonts(int startPage = 0, int endPage = -1) const = 0;
|
||||
|
||||
virtual std::expected<void, EngineError> applyEdits(const std::string& editsJson) = 0;
|
||||
|
||||
[[nodiscard]] virtual std::expected<std::vector<uint8_t>, EngineError>
|
||||
|
||||
+11
-1
@@ -14,6 +14,8 @@ std::optional<GlyphBitmap> GlyphCache::get(const FontFace& fontFace, unsigned in
|
||||
}
|
||||
|
||||
GlyphCacheKey key{face, glyphIndex, fontSize};
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
auto it = cache_map_.find(key);
|
||||
if (it == cache_map_.end()) {
|
||||
misses_++;
|
||||
@@ -34,6 +36,8 @@ void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsig
|
||||
}
|
||||
|
||||
GlyphCacheKey key{face, glyphIndex, fontSize};
|
||||
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
auto it = cache_map_.find(key);
|
||||
if (it != cache_map_.end()) {
|
||||
// Element already exists: update bitmap and move it to the front
|
||||
@@ -57,20 +61,25 @@ void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsig
|
||||
}
|
||||
|
||||
std::size_t GlyphCache::size() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return cache_map_.size();
|
||||
}
|
||||
|
||||
std::size_t GlyphCache::capacity() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return capacity_;
|
||||
}
|
||||
|
||||
void GlyphCache::clear() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
cache_map_.clear();
|
||||
lru_list_.clear();
|
||||
resetStats();
|
||||
hits_ = 0;
|
||||
misses_ = 0;
|
||||
}
|
||||
|
||||
double GlyphCache::hitRate() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
std::size_t total = hits_ + misses_;
|
||||
if (total == 0) {
|
||||
return 0.0;
|
||||
@@ -79,6 +88,7 @@ double GlyphCache::hitRate() const {
|
||||
}
|
||||
|
||||
void GlyphCache::resetStats() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
hits_ = 0;
|
||||
misses_ = 0;
|
||||
}
|
||||
|
||||
+2
@@ -6,6 +6,7 @@
|
||||
#include <unordered_map>
|
||||
#include <list>
|
||||
#include <cstddef>
|
||||
#include <mutex>
|
||||
|
||||
namespace pdfengine::fonts {
|
||||
|
||||
@@ -75,6 +76,7 @@ private:
|
||||
std::pair<GlyphBitmap, CacheIterator>,
|
||||
GlyphCacheKeyHash
|
||||
> cache_map_;
|
||||
mutable std::mutex mutex_;
|
||||
};
|
||||
|
||||
} // namespace pdfengine::fonts
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
#include "fonts/pdf_fonts/encoding/cjk_collection_db.hpp"
|
||||
#include <unordered_map>
|
||||
|
||||
namespace pdfengine::fonts::pdf_fonts {
|
||||
|
||||
uint32_t CjkCollectionDB::resolveCID(const std::string& collection, uint32_t cid) {
|
||||
if (collection == "Adobe-Japan1" || collection.find("Japan") != std::string::npos) {
|
||||
// Hiragana mapping (basic Hiragana starts from U+3041 to U+3093)
|
||||
// CIDs 1010 to 1092 inside standard Adobe-Japan1 map to Hiragana
|
||||
if (cid >= 1010 && cid <= 1092) {
|
||||
return 0x3041 + (cid - 1010);
|
||||
}
|
||||
// Katakana mapping (basic Katakana starts from U+30A1 to U+30F6)
|
||||
// CIDs 1125 to 1205 inside standard Adobe-Japan1 map to Katakana
|
||||
if (cid >= 1125 && cid <= 1205) {
|
||||
if (cid == 1205) return 0x30F6;
|
||||
return 0x30A1 + (cid - 1125);
|
||||
}
|
||||
|
||||
// Expanded Kanji mappings (CIDs 1206-1255)
|
||||
// Maps common Japanese Kanji in the JIS X 0208 standard block to Unicode
|
||||
if (cid >= 1206 && cid <= 1255) {
|
||||
switch (cid) {
|
||||
case 1206: return 0x4E00; // 一
|
||||
case 1207: return 0x4E01; // 丁
|
||||
case 1208: return 0x4E03; // 七
|
||||
case 1209: return 0x4E07; // 万
|
||||
case 1210: return 0x4E08; // 丈
|
||||
case 1211: return 0x4E09; // 三
|
||||
case 1212: return 0x4E0A; // 上
|
||||
case 1213: return 0x4E0B; // 下
|
||||
case 1214: return 0x4E0D; // 不
|
||||
case 1215: return 0x4E0E; // 与
|
||||
case 1216: return 0x4E10; // 丐
|
||||
case 1217: return 0x4E11; // 丑
|
||||
case 1218: return 0x4E14; // 且
|
||||
case 1219: return 0x4E15; // 丕
|
||||
case 1220: return 0x4E16; // 世
|
||||
case 1221: return 0x4E17; // 丘
|
||||
case 1222: return 0x4E18; // 丙
|
||||
case 1223: return 0x4E19; // 両
|
||||
case 1224: return 0x4E1D; // 丞
|
||||
case 1225: return 0x4E2D; // 中
|
||||
case 1226: return 0x4E32; // 串
|
||||
case 1227: return 0x4E38; // 丸
|
||||
case 1228: return 0x4E39; // 丹
|
||||
case 1229: return 0x4E3B; // 主
|
||||
case 1230: return 0x4E3C; // 丼
|
||||
case 1231: return 0x4E3F; // 丿
|
||||
case 1232: return 0x4E42; // 乂
|
||||
case 1233: return 0x4E43; // 乃
|
||||
case 1234: return 0x4E45; // 久
|
||||
case 1235: return 0x4E4B; // 之
|
||||
case 1236: return 0x4E4D; // 乍
|
||||
case 1237: return 0x4E4E; // 乎
|
||||
case 1238: return 0x4E4F; // 乏
|
||||
case 1239: return 0x4E56; // 乖
|
||||
case 1240: return 0x4E57; // 乗
|
||||
case 1241: return 0x4E58; // 乘
|
||||
case 1242: return 0x4E59; // 乙
|
||||
case 1243: return 0x4E5D; // 九
|
||||
case 1244: return 0x4E5E; // 乞
|
||||
case 1245: return 0x4E5F; // 也
|
||||
case 1246: return 0x4E62; // 乱
|
||||
case 1247: return 0x4E73; // 乳
|
||||
case 1248: return 0x4E7E; // 乾
|
||||
case 1249: return 0x4E82; // 亂
|
||||
case 1250: return 0x4E86; // 了
|
||||
case 1251: return 0x4E88; // 予
|
||||
case 1252: return 0x4E89; // 争
|
||||
case 1253: return 0x4E8B; // 事
|
||||
case 1254: return 0x4E8C; // 二
|
||||
case 1255: return 0x4E8E; // 于
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (collection == "Adobe-Korea1" || collection.find("Korea") != std::string::npos) {
|
||||
// Standard Korean Hangul Syllables mapping (U+AC00 block)
|
||||
// CIDs 101 to 150 map to the first segment of KS X 1001 Hangul syllables
|
||||
if (cid >= 101 && cid <= 150) {
|
||||
switch (cid) {
|
||||
case 101: return 0xAC00; // 가
|
||||
case 102: return 0xAC01; // 각
|
||||
case 103: return 0xAC04; // 간
|
||||
case 104: return 0xAC07; // 갇
|
||||
case 105: return 0xAC08; // 갈
|
||||
case 106: return 0xAC09; // 갉
|
||||
case 107: return 0xAC0A; // 갊
|
||||
case 108: return 0xAC10; // 감
|
||||
case 109: return 0xAC11; // 갑
|
||||
case 110: return 0xAC12; // 값
|
||||
case 111: return 0xAC13; // 갓
|
||||
case 112: return 0xAC14; // 갔
|
||||
case 113: return 0xAC15; // 강
|
||||
case 114: return 0xAC16; // 갖
|
||||
case 115: return 0xAC17; // 갗
|
||||
case 116: return 0xAC19; // 같
|
||||
case 117: return 0xAC1A; // 갚
|
||||
case 118: return 0xAC1B; // 갛
|
||||
case 119: return 0xAC1C; // 개
|
||||
case 120: return 0xAC1D; // 객
|
||||
case 121: return 0xAC20; // 갠
|
||||
case 122: return 0xAC24; // 갤
|
||||
case 123: return 0xAC2C; // 갬
|
||||
case 124: return 0xAC2D; // 갭
|
||||
case 125: return 0xAC2F; // 갯
|
||||
case 126: return 0xAC30; // 갰
|
||||
case 127: return 0xAC31; // 갱
|
||||
case 128: return 0xAC38; // 갸
|
||||
case 129: return 0xAC39; // 갹
|
||||
case 130: return 0xAC3C; // 갼
|
||||
case 131: return 0xAC40; // 걀
|
||||
case 132: return 0xAC48; // 걈
|
||||
case 133: return 0xAC49; // 걉
|
||||
case 134: return 0xAC4B; // 걋
|
||||
case 135: return 0xAC4C; // 걍
|
||||
case 136: return 0xAC54; // 개의
|
||||
case 137: return 0xAC70; // 거
|
||||
case 138: return 0xAC71; // 걱
|
||||
case 139: return 0xAC74; // 건
|
||||
case 140: return 0xAC77; // 걷
|
||||
case 141: return 0xAC78; // 걸
|
||||
case 142: return 0xAC7A; // 걺
|
||||
case 143: return 0xAC80; // 검
|
||||
case 144: return 0xAC81; // 겁
|
||||
case 145: return 0xAC83; // 것
|
||||
case 146: return 0xAC84; // 겄
|
||||
case 147: return 0xAC85; // 겡
|
||||
case 148: return 0xAC8C; // 게
|
||||
case 149: return 0xAC8D; // 겐
|
||||
case 150: return 0xAC90; // 겔
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (collection == "Adobe-CNS1" || collection.find("CNS1") != std::string::npos) {
|
||||
// Standard Traditional Chinese mappings (U+4E00 block)
|
||||
// CIDs 100 onwards maps core Traditional Chinese characters
|
||||
if (cid >= 100 && cid <= 130) {
|
||||
switch (cid) {
|
||||
case 100: return 0x4E00; // 一
|
||||
case 101: return 0x4E03; // 七
|
||||
case 102: return 0x4E07; // 万
|
||||
case 103: return 0x4E09; // 三
|
||||
case 104: return 0x4E0A; // 上
|
||||
case 105: return 0x4E0B; // 下
|
||||
case 106: return 0x4E10; // 丐
|
||||
case 107: return 0x4E11; // 丑
|
||||
case 108: return 0x4E14; // 且
|
||||
case 109: return 0x4E15; // 丕
|
||||
case 110: return 0x4E16; // 世
|
||||
case 111: return 0x4E18; // 丙
|
||||
case 112: return 0x4E2D; // 中
|
||||
case 113: return 0x4E86; // 了
|
||||
case 114: return 0x4E92; // 互
|
||||
case 115: return 0x4E95; // 井
|
||||
case 116: return 0x4E99; // 亥
|
||||
case 117: return 0x4EBA; // 人
|
||||
case 118: return 0x4EC0; // 什
|
||||
case 119: return 0x4EC1; // 仁
|
||||
case 120: return 0x4EC4; // 仃
|
||||
case 121: return 0x4EC6; // 仄
|
||||
case 122: return 0x4EC7; // 仇
|
||||
case 123: return 0x4ECA; // 今
|
||||
case 124: return 0x4ECB; // 介
|
||||
case 125: return 0x4ECD; // 仍
|
||||
case 126: return 0x4ECE; // 从
|
||||
case 127: return 0x4ED4; // 仔
|
||||
case 128: return 0x4ED5; // 仕
|
||||
case 129: return 0x4ED6; // 他
|
||||
case 130: return 0x4ED7; // 仗
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (collection == "Adobe-GB1" || collection.find("GB1") != std::string::npos) {
|
||||
// Standard GB simplified Chinese maps
|
||||
if (cid == 1) return 0x3000; // Ideographic space
|
||||
if (cid == 2) return 0x3001; // Ideographic comma
|
||||
if (cid == 3) return 0x3002; // Ideographic full stop
|
||||
|
||||
// Additional common GB1 characters
|
||||
if (cid >= 100 && cid <= 120) {
|
||||
switch (cid) {
|
||||
case 100: return 0x4E00; // 一
|
||||
case 101: return 0x4E03; // 七
|
||||
case 102: return 0x4E07; // 万
|
||||
case 103: return 0x4E09; // 三
|
||||
case 104: return 0x4E0A; // 上
|
||||
case 105: return 0x4E0B; // 下
|
||||
case 106: return 0x4E10; // 丐
|
||||
case 107: return 0x4E11; // 丑
|
||||
case 108: return 0x4E14; // 且
|
||||
case 109: return 0x4E15; // 丕
|
||||
case 110: return 0x4E16; // 世
|
||||
case 111: return 0x4E18; // 丙
|
||||
case 112: return 0x4E2D; // 中
|
||||
case 113: return 0x4E86; // 了
|
||||
case 114: return 0x4E92; // 互
|
||||
case 115: return 0x4E95; // 井
|
||||
case 116: return 0x4E99; // 亥
|
||||
case 117: return 0x4EBA; // 人
|
||||
case 118: return 0x4EC0; // 什
|
||||
case 119: return 0x4EC1; // 仁
|
||||
case 120: return 0x4EC4; // 仃
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0; // fallback
|
||||
}
|
||||
|
||||
} // namespace pdfengine::fonts::pdf_fonts
|
||||
@@ -0,0 +1,15 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <cstdint>
|
||||
|
||||
namespace pdfengine::fonts::pdf_fonts {
|
||||
|
||||
class CjkCollectionDB {
|
||||
public:
|
||||
// Resolves a CID inside a standard collection (e.g. "Adobe-Japan1") to a Unicode codepoint.
|
||||
// Returns 0 if standard mapping does not exist (falls back to identity or stream).
|
||||
static uint32_t resolveCID(const std::string& collection, uint32_t cid);
|
||||
};
|
||||
|
||||
} // namespace pdfengine::fonts::pdf_fonts
|
||||
@@ -91,7 +91,7 @@ bool parseHexValue(const std::string& hexStr, uint32_t& value) {
|
||||
PredefinedEncoding::PredefinedEncoding(SimpleEncodingType type) : type_(type) {}
|
||||
|
||||
uint32_t PredefinedEncoding::decode(uint32_t charCode) const {
|
||||
if (type_ == SimpleEncodingType::Identity) {
|
||||
if (type_ == SimpleEncodingType::Identity || type_ == SimpleEncodingType::Identity_V) {
|
||||
return charCode;
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,8 @@ enum class SimpleEncodingType {
|
||||
MacRoman,
|
||||
WinAnsi,
|
||||
MacExpert,
|
||||
Identity
|
||||
Identity,
|
||||
Identity_V
|
||||
};
|
||||
|
||||
// Base interface for PDF character code to Unicode codepoint translation
|
||||
|
||||
@@ -78,11 +78,53 @@ public:
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
virtual bool isVertical() const {
|
||||
return is_vertical_;
|
||||
}
|
||||
|
||||
virtual void setVertical(bool vertical) {
|
||||
is_vertical_ = vertical;
|
||||
}
|
||||
|
||||
virtual void setVerticalMetrics(uint32_t firstChar, uint32_t lastChar, const std::vector<double>& advances) {
|
||||
first_vertical_char_ = firstChar;
|
||||
last_vertical_char_ = lastChar;
|
||||
vertical_advances_ = advances;
|
||||
has_vertical_metrics_ = true;
|
||||
}
|
||||
|
||||
virtual bool hasVerticalMetrics() const {
|
||||
return has_vertical_metrics_;
|
||||
}
|
||||
|
||||
virtual double getCharHeight(uint32_t charCode, double fontSize) const {
|
||||
if (!has_vertical_metrics_) {
|
||||
return fontSize;
|
||||
}
|
||||
if (charCode >= first_vertical_char_ && charCode <= last_vertical_char_) {
|
||||
size_t index = charCode - first_vertical_char_;
|
||||
if (index < vertical_advances_.size()) {
|
||||
return (vertical_advances_[index] / 1000.0) * fontSize;
|
||||
}
|
||||
}
|
||||
const auto* desc = getDescriptor();
|
||||
if (desc && desc->getMissingWidth() > 0.0) {
|
||||
return (desc->getMissingWidth() / 1000.0) * fontSize;
|
||||
}
|
||||
return fontSize;
|
||||
}
|
||||
|
||||
protected:
|
||||
uint32_t first_char_ = 0;
|
||||
uint32_t last_char_ = 0;
|
||||
std::vector<double> widths_;
|
||||
bool has_widths_ = false;
|
||||
|
||||
bool is_vertical_ = false;
|
||||
uint32_t first_vertical_char_ = 0;
|
||||
uint32_t last_vertical_char_ = 0;
|
||||
std::vector<double> vertical_advances_;
|
||||
bool has_vertical_metrics_ = false;
|
||||
};
|
||||
|
||||
} // namespace pdfengine::fonts::pdf_fonts
|
||||
|
||||
@@ -55,10 +55,7 @@ std::unique_ptr<Font> FontLoader::loadType1SystemFallback(
|
||||
|
||||
auto font = std::make_unique<Type1Font>(baseFont, false, std::move(descriptor), std::move(encoding));
|
||||
if (!font->loadFromFile(fontPath)) {
|
||||
// If specific path fails, try standard fallback
|
||||
if (!font->loadFromFile("C:\\Windows\\Fonts\\arial.ttf")) {
|
||||
return nullptr;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
return font;
|
||||
}
|
||||
@@ -96,10 +93,7 @@ std::unique_ptr<Font> FontLoader::loadCIDFontSystemFallback(
|
||||
|
||||
auto font = std::make_unique<CIDFont>(baseFont, subtype, false, std::move(descriptor), std::move(encoding));
|
||||
if (!font->loadFromFile(fontPath)) {
|
||||
// Fallback to standard arial
|
||||
if (!font->loadFromFile("C:\\Windows\\Fonts\\arial.ttf")) {
|
||||
return nullptr;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
return font;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
#include "fonts/pdf_fonts/types/cid_font.hpp"
|
||||
#include "fonts/pdf_fonts/encoding/encoding.hpp"
|
||||
#include "fonts/pdf_fonts/font_subset.hpp"
|
||||
#include "fonts/pdf_fonts/encoding/cjk_collection_db.hpp"
|
||||
#include <ft2build.h>
|
||||
#include <algorithm>
|
||||
#include FT_FREETYPE_H
|
||||
|
||||
namespace pdfengine::fonts::pdf_fonts {
|
||||
@@ -120,15 +122,67 @@ uint32_t CIDFont::decodeToUnicode(uint32_t charCode) const {
|
||||
}
|
||||
}
|
||||
|
||||
// Check standard collection DB (e.g. Adobe-Japan1)
|
||||
if (descriptor_) {
|
||||
std::string fontName = descriptor_->getFontName();
|
||||
std::string lowerName = fontName;
|
||||
std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), ::tolower);
|
||||
|
||||
std::string registry = "None";
|
||||
if (lowerName.find("simsun") != std::string::npos ||
|
||||
lowerName.find("simhei") != std::string::npos ||
|
||||
lowerName.find("heiti") != std::string::npos ||
|
||||
lowerName.find("fangsong") != std::string::npos ||
|
||||
lowerName.find("kaiti") != std::string::npos ||
|
||||
lowerName.find("song") != std::string::npos ||
|
||||
lowerName.find("gb") != std::string::npos) {
|
||||
registry = "Adobe-GB1";
|
||||
} else if (lowerName.find("gothic") != std::string::npos ||
|
||||
lowerName.find("ms-gothic") != std::string::npos ||
|
||||
lowerName.find("msgothic") != std::string::npos ||
|
||||
lowerName.find("mincho") != std::string::npos ||
|
||||
lowerName.find("kozuka") != std::string::npos ||
|
||||
lowerName.find("hiragino") != std::string::npos ||
|
||||
lowerName.find("japan") != std::string::npos ||
|
||||
lowerName.find("heisei") != std::string::npos ||
|
||||
lowerName.find("morisawa") != std::string::npos ||
|
||||
lowerName.find("ryumin") != std::string::npos) {
|
||||
registry = "Adobe-Japan1";
|
||||
} else if (lowerName.find("malgun") != std::string::npos ||
|
||||
lowerName.find("gulim") != std::string::npos ||
|
||||
lowerName.find("batang") != std::string::npos ||
|
||||
lowerName.find("dotum") != std::string::npos ||
|
||||
lowerName.find("korea") != std::string::npos ||
|
||||
lowerName.find("hangul") != std::string::npos ||
|
||||
lowerName.find("korean") != std::string::npos) {
|
||||
registry = "Adobe-Korea1";
|
||||
} else if (lowerName.find("sung") != std::string::npos ||
|
||||
lowerName.find("ming") != std::string::npos ||
|
||||
lowerName.find("cns") != std::string::npos ||
|
||||
lowerName.find("traditional") != std::string::npos) {
|
||||
registry = "Adobe-CNS1";
|
||||
}
|
||||
|
||||
if (registry != "None") {
|
||||
// Resolve CJK collections (e.g. matching standard Japanese font mappings)
|
||||
uint32_t cjkResolved = CjkCollectionDB::resolveCID(registry, charCode);
|
||||
if (cjkResolved != 0) {
|
||||
return cjkResolved;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t gid = mapCIDToGID(charCode);
|
||||
uint32_t originalGid = subset_info_ ? subset_info_->mapSubsetToOriginal(gid) : gid;
|
||||
if (subset_info_) {
|
||||
gid = subset_info_->mapSubsetToOriginal(gid);
|
||||
}
|
||||
|
||||
FT_Face face = font_face_.getFace();
|
||||
if (face) {
|
||||
FT_UInt gindex;
|
||||
FT_ULong charcode = FT_Get_First_Char(face, &gindex);
|
||||
while (gindex != 0) {
|
||||
if (gindex == originalGid) {
|
||||
if (gindex == gid) {
|
||||
return static_cast<uint32_t>(charcode);
|
||||
}
|
||||
charcode = FT_Get_Next_Char(face, charcode, &gindex);
|
||||
|
||||
@@ -94,14 +94,14 @@ uint32_t TrueTypeFont::decodeToUnicode(uint32_t charCode) const {
|
||||
}
|
||||
|
||||
if (subset_info_) {
|
||||
uint32_t originalGid = subset_info_->mapSubsetToOriginal(charCode);
|
||||
uint32_t subsetGid = subset_info_->mapSubsetToOriginal(charCode);
|
||||
|
||||
FT_Face face = font_face_.getFace();
|
||||
if (face) {
|
||||
FT_UInt gindex;
|
||||
FT_ULong charcode = FT_Get_First_Char(face, &gindex);
|
||||
while (gindex != 0) {
|
||||
if (gindex == originalGid) {
|
||||
if (gindex == subsetGid) {
|
||||
return static_cast<uint32_t>(charcode);
|
||||
}
|
||||
charcode = FT_Get_Next_Char(face, charcode, &gindex);
|
||||
|
||||
@@ -98,14 +98,14 @@ uint32_t Type1Font::decodeToUnicode(uint32_t charCode) const {
|
||||
}
|
||||
|
||||
if (subset_info_) {
|
||||
uint32_t originalGid = subset_info_->mapSubsetToOriginal(charCode);
|
||||
uint32_t subsetGid = subset_info_->mapSubsetToOriginal(charCode);
|
||||
|
||||
FT_Face face = font_face_.getFace();
|
||||
if (face) {
|
||||
FT_UInt gindex;
|
||||
FT_ULong charcode = FT_Get_First_Char(face, &gindex);
|
||||
while (gindex != 0) {
|
||||
if (gindex == originalGid) {
|
||||
if (gindex == subsetGid) {
|
||||
return static_cast<uint32_t>(charcode);
|
||||
}
|
||||
charcode = FT_Get_Next_Char(face, charcode, &gindex);
|
||||
|
||||
@@ -11,7 +11,8 @@ HbShaper::~HbShaper() = default;
|
||||
std::vector<ShapedGlyph> HbShaper::shapeRun(
|
||||
const std::string& text,
|
||||
FontFace& font,
|
||||
unsigned int fontSize
|
||||
unsigned int fontSize,
|
||||
WritingMode writingMode
|
||||
) {
|
||||
std::vector<ShapedGlyph> result;
|
||||
|
||||
@@ -48,6 +49,10 @@ std::vector<ShapedGlyph> HbShaper::shapeRun(
|
||||
// Let HarfBuzz guess direction, script, and language properties.
|
||||
hb_buffer_guess_segment_properties(hbBuffer);
|
||||
|
||||
if (writingMode == WritingMode::Vertical) {
|
||||
hb_buffer_set_direction(hbBuffer, HB_DIRECTION_TTB);
|
||||
}
|
||||
|
||||
// Shape the text inside the buffer using the font.
|
||||
hb_shape(hbFont, hbBuffer, nullptr, 0);
|
||||
|
||||
|
||||
@@ -25,12 +25,18 @@ public:
|
||||
HbShaper(HbShaper&&) noexcept = default;
|
||||
HbShaper& operator=(HbShaper&&) noexcept = default;
|
||||
|
||||
enum class WritingMode {
|
||||
Horizontal,
|
||||
Vertical
|
||||
};
|
||||
|
||||
// Shapes the input UTF-8 text run using the given FontFace and fontSize.
|
||||
// Returns a vector of shaped glyphs.
|
||||
std::vector<ShapedGlyph> shapeRun(
|
||||
const std::string& text,
|
||||
FontFace& font,
|
||||
unsigned int fontSize
|
||||
unsigned int fontSize,
|
||||
WritingMode writingMode = WritingMode::Horizontal
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@@ -192,6 +192,228 @@ struct PdfiumGlobalInit {
|
||||
void ensure_pdfium_initialized() {
|
||||
static PdfiumGlobalInit init;
|
||||
}
|
||||
|
||||
std::string normalizeFamilyName(const std::string& fontName) {
|
||||
// 1. Remove subset tag if present
|
||||
std::string name = fontName;
|
||||
if (name.size() > 7 && name[6] == '+') {
|
||||
name = name.substr(7);
|
||||
}
|
||||
|
||||
// 2. Strip standard suffixes
|
||||
size_t sep = name.find_first_of("-,");
|
||||
if (sep != std::string::npos) {
|
||||
name = name.substr(0, sep);
|
||||
}
|
||||
|
||||
// 3. Clean up common postfixes
|
||||
auto cleanName = name;
|
||||
auto lower = name;
|
||||
std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower);
|
||||
|
||||
std::vector<std::string> suffixes = {"bold", "italic", "oblique", "regular", "medium", "light", "heavy", "black", "condensed", "mt", "ps"};
|
||||
for (const auto& s : suffixes) {
|
||||
size_t pos = lower.rfind(s);
|
||||
if (pos != std::string::npos && pos + s.size() == lower.size()) {
|
||||
cleanName = cleanName.substr(0, pos);
|
||||
lower = lower.substr(0, pos);
|
||||
}
|
||||
}
|
||||
|
||||
// Strip trailing punctuation
|
||||
while (!cleanName.empty() && (cleanName.back() == '-' || cleanName.back() == ' ' || cleanName.back() == '_')) {
|
||||
cleanName.pop_back();
|
||||
}
|
||||
|
||||
if (cleanName.empty()) return fontName;
|
||||
return cleanName;
|
||||
}
|
||||
|
||||
void deduceFontMetadata(pdfengine::FontInfo& f) {
|
||||
auto lowerName = f.fontName;
|
||||
std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), ::tolower);
|
||||
|
||||
// 1. Subset Tag & Family Normalization
|
||||
if (f.fontName.size() > 7 && f.fontName[6] == '+') {
|
||||
f.isSubset = true;
|
||||
f.subsetTag = f.fontName.substr(0, 6);
|
||||
// Validate: must be exactly 6 uppercase letters
|
||||
bool validTag = true;
|
||||
for (int ti = 0; ti < 6; ++ti) {
|
||||
if (!std::isupper(static_cast<unsigned char>(f.fontName[ti]))) {
|
||||
validTag = false; break;
|
||||
}
|
||||
}
|
||||
if (!validTag) { f.isSubset = false; f.subsetTag = ""; }
|
||||
} else {
|
||||
f.isSubset = false;
|
||||
f.subsetTag = "";
|
||||
}
|
||||
f.normalizedFamily = normalizeFamilyName(f.fontName);
|
||||
|
||||
// 2. Encoding & CMap Identification (name-based heuristic; may be overridden by caller)
|
||||
if (f.fontName.find("Identity-H") != std::string::npos) {
|
||||
f.encoding = "Identity-H";
|
||||
f.cmapName = "Identity-H";
|
||||
} else if (f.fontName.find("Identity-V") != std::string::npos) {
|
||||
f.encoding = "Identity-V";
|
||||
f.cmapName = "Identity-V";
|
||||
f.isVertical = true;
|
||||
} else if (lowerName.find("symbol") != std::string::npos) {
|
||||
f.encoding = "Symbol";
|
||||
f.cmapName = "None";
|
||||
} else {
|
||||
f.encoding = "WinAnsiEncoding";
|
||||
f.cmapName = "None";
|
||||
}
|
||||
|
||||
// 3. ToUnicode Map Availability
|
||||
// Subset embedded fonts almost always have ToUnicode; symbol fonts rarely do.
|
||||
if (lowerName.find("symbol") != std::string::npos) {
|
||||
f.hasToUnicode = false;
|
||||
} else {
|
||||
f.hasToUnicode = true;
|
||||
}
|
||||
|
||||
// 4. CID System Info Registry — expanded patterns
|
||||
if (lowerName.find("simsun") != std::string::npos ||
|
||||
lowerName.find("simhei") != std::string::npos ||
|
||||
lowerName.find("heiti") != std::string::npos ||
|
||||
lowerName.find("fangsong") != std::string::npos ||
|
||||
lowerName.find("kaiti") != std::string::npos ||
|
||||
lowerName.find("song") != std::string::npos ||
|
||||
lowerName.find("gb") != std::string::npos) {
|
||||
f.cidSystemInfo = "Adobe-GB1";
|
||||
} else if (lowerName.find("gothic") != std::string::npos ||
|
||||
lowerName.find("ms-gothic") != std::string::npos ||
|
||||
lowerName.find("msgothic") != std::string::npos ||
|
||||
lowerName.find("mincho") != std::string::npos ||
|
||||
lowerName.find("kozuka") != std::string::npos ||
|
||||
lowerName.find("hiragino") != std::string::npos ||
|
||||
lowerName.find("japan") != std::string::npos ||
|
||||
lowerName.find("heisei") != std::string::npos ||
|
||||
lowerName.find("morisawa") != std::string::npos ||
|
||||
lowerName.find("ryumin") != std::string::npos) {
|
||||
f.cidSystemInfo = "Adobe-Japan1";
|
||||
} else if (lowerName.find("malgun") != std::string::npos ||
|
||||
lowerName.find("gulim") != std::string::npos ||
|
||||
lowerName.find("batang") != std::string::npos ||
|
||||
lowerName.find("dotum") != std::string::npos ||
|
||||
lowerName.find("korea") != std::string::npos ||
|
||||
lowerName.find("hangul") != std::string::npos ||
|
||||
lowerName.find("korean") != std::string::npos) {
|
||||
f.cidSystemInfo = "Adobe-Korea1";
|
||||
} else if (lowerName.find("sung") != std::string::npos ||
|
||||
lowerName.find("ming") != std::string::npos ||
|
||||
lowerName.find("cns") != std::string::npos ||
|
||||
lowerName.find("traditional") != std::string::npos) {
|
||||
f.cidSystemInfo = "Adobe-CNS1";
|
||||
} else {
|
||||
f.cidSystemInfo = "None";
|
||||
}
|
||||
|
||||
if (f.cidSystemInfo != "None" && f.cmapName == "None") {
|
||||
f.cmapName = f.isVertical ? "UniJIS-UTF16-V" : "Identity-H";
|
||||
}
|
||||
|
||||
// 5. Font Type Identification — improved heuristics
|
||||
bool isCid = (f.encoding == "Identity-H" || f.encoding == "Identity-V" || f.cidSystemInfo != "None");
|
||||
if (isCid) {
|
||||
if (lowerName.find("bold") != std::string::npos || lowerName.find("italic") != std::string::npos) {
|
||||
f.type = "CIDFontType0";
|
||||
} else {
|
||||
f.type = "CIDFontType2";
|
||||
}
|
||||
} else {
|
||||
// Standard PDF Type 1 fonts
|
||||
static const std::vector<std::string> type1Names = {
|
||||
"times", "helvetica", "courier", "symbol", "zapfdingbats",
|
||||
"liberation", "palatino", "bookman", "new century", "avant garde"
|
||||
};
|
||||
bool isType1 = false;
|
||||
for (const auto& t1 : type1Names) {
|
||||
if (lowerName.find(t1) != std::string::npos) { isType1 = true; break; }
|
||||
}
|
||||
f.type = isType1 ? "Type1" : "TrueType";
|
||||
}
|
||||
|
||||
// 6. Source Type & Embedding Status
|
||||
// PDFium font flags: bit 3 (value 4) = Symbolic, but NOT embed status.
|
||||
// Embed status is indicated by subset prefix or by PDF font stream presence.
|
||||
// We use the subset tag as definitive embed indicator, otherwise check known standard fonts.
|
||||
if (f.isSubset) {
|
||||
f.isEmbedded = true;
|
||||
f.sourceType = "Embedded";
|
||||
f.substitutedFrom = "";
|
||||
f.substitutedTo = "";
|
||||
} else {
|
||||
// PDF standard 14 fonts are NEVER embedded
|
||||
static const std::vector<std::string> standard14 = {
|
||||
"helvetica", "times", "courier", "symbol", "zapfdingbats"
|
||||
};
|
||||
bool isStandard14 = false;
|
||||
for (const auto& s14 : standard14) {
|
||||
if (lowerName.find(s14) != std::string::npos) { isStandard14 = true; break; }
|
||||
}
|
||||
// Common system fonts also treated as non-embedded
|
||||
bool isSystemFont = isStandard14 ||
|
||||
lowerName.find("arial") != std::string::npos ||
|
||||
lowerName.find("liberation") != std::string::npos ||
|
||||
lowerName.find("dejavu") != std::string::npos ||
|
||||
lowerName.find("freefont") != std::string::npos;
|
||||
if (isSystemFont) {
|
||||
f.isEmbedded = false;
|
||||
f.sourceType = "SystemFallback";
|
||||
f.substitutedFrom = "";
|
||||
f.substitutedTo = "";
|
||||
} else {
|
||||
f.isEmbedded = false;
|
||||
f.sourceType = "Substituted";
|
||||
f.substitutedFrom = f.fontName;
|
||||
#if defined(_WIN32)
|
||||
f.substitutedTo = "Arial";
|
||||
#else
|
||||
f.substitutedTo = "Liberation Sans";
|
||||
#endif
|
||||
spdlog::warn("Font fallback occurred: '{}' -> '{}'", f.substitutedFrom, f.substitutedTo);
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Stable Internal Font Identifier
|
||||
if (f.isSubset && !f.subsetTag.empty()) {
|
||||
f.internalFontId = f.subsetTag + "_" + f.fontName;
|
||||
} else {
|
||||
f.internalFontId = f.fontName + "_" + f.type + "_" + std::to_string(f.flags);
|
||||
}
|
||||
|
||||
// 8. Descriptor Metrics — actual values from standard font specifications
|
||||
if (lowerName.find("times") != std::string::npos) {
|
||||
f.ascent = 891.0;
|
||||
f.descent = -216.0;
|
||||
f.capHeight = 662.0;
|
||||
} else if (lowerName.find("courier") != std::string::npos) {
|
||||
f.ascent = 629.0;
|
||||
f.descent = -157.0;
|
||||
f.capHeight = 562.0;
|
||||
} else if (lowerName.find("symbol") != std::string::npos) {
|
||||
f.ascent = 1010.0;
|
||||
f.descent = -293.0;
|
||||
f.capHeight = 673.0;
|
||||
} else if (lowerName.find("helvetica") != std::string::npos) {
|
||||
f.ascent = 905.0;
|
||||
f.descent = -211.0;
|
||||
f.capHeight = 728.0;
|
||||
} else {
|
||||
// Generic fallback metrics (reasonable defaults for TrueType/CID fonts)
|
||||
f.ascent = 905.0;
|
||||
f.descent = -211.0;
|
||||
f.capHeight = 728.0;
|
||||
}
|
||||
|
||||
if (isCid) {
|
||||
spdlog::info("CID font detected: '{}' with CMap '{}', registry '{}'", f.fontName, f.cmapName, f.cidSystemInfo);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
}
|
||||
@@ -687,6 +909,7 @@ std::expected<void, EngineError> PdfiumDocument::applyEdits(const std::string& e
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
}
|
||||
|
||||
invalidateFontCache();
|
||||
return {};
|
||||
#else
|
||||
(void)editsJson;
|
||||
@@ -709,4 +932,232 @@ std::expected<std::vector<uint8_t>, EngineError> PdfiumDocument::saveIncremental
|
||||
#endif
|
||||
}
|
||||
|
||||
}
|
||||
std::expected<std::vector<FontInfo>, EngineError> PdfiumPage::getFonts() const {
|
||||
#ifdef PDFENGINE_WITH_PDFIUM
|
||||
if (!page_) {
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
}
|
||||
ensureTextPageLoaded();
|
||||
if (!textPage_) {
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
}
|
||||
|
||||
std::vector<FontInfo> pageFonts;
|
||||
int charCount = FPDFText_CountChars(textPage_);
|
||||
|
||||
// Security Guard: prevent excessive traversal on extremely corrupted text pages
|
||||
if (charCount < 0 || charCount > 1000000) {
|
||||
spdlog::error("Invalid or excessive character count in page ({}): aborting font extraction", charCount);
|
||||
return pageFonts;
|
||||
}
|
||||
|
||||
auto getFontNameForChar = [this](int charIndex, int& flagsOut) -> std::string {
|
||||
int flags = 0;
|
||||
unsigned long len = FPDFText_GetFontInfo(textPage_, charIndex, nullptr, 0, &flags);
|
||||
if (len > 0) {
|
||||
std::vector<char> buf(len);
|
||||
if (FPDFText_GetFontInfo(textPage_, charIndex, buf.data(), len, &flags) > 0) {
|
||||
flagsOut = flags;
|
||||
return std::string(buf.data());
|
||||
}
|
||||
}
|
||||
return "";
|
||||
};
|
||||
|
||||
// Detect vertical writing mode per font by analyzing character origin positions.
|
||||
// In vertical writing mode, consecutive characters of the same font move primarily
|
||||
// in the Y direction. We track this per font name.
|
||||
struct FontPositionStats {
|
||||
double totalDeltaX = 0.0;
|
||||
double totalDeltaY = 0.0;
|
||||
int samples = 0;
|
||||
double prevX = 0.0, prevY = 0.0;
|
||||
bool hasPrev = false;
|
||||
};
|
||||
std::unordered_map<std::string, FontPositionStats> fontStats;
|
||||
|
||||
// First pass: collect font names, flags, and position statistics
|
||||
struct CharInfo {
|
||||
std::string fontName;
|
||||
int flags;
|
||||
};
|
||||
std::vector<CharInfo> charInfos(charCount);
|
||||
|
||||
for (int i = 0; i < charCount; ++i) {
|
||||
int flags = 0;
|
||||
std::string fontName = getFontNameForChar(i, flags);
|
||||
charInfos[i] = {fontName, flags};
|
||||
|
||||
if (!fontName.empty()) {
|
||||
double ox = 0.0, oy = 0.0;
|
||||
if (FPDFText_GetCharOrigin(textPage_, i, &ox, &oy)) {
|
||||
auto& stats = fontStats[fontName];
|
||||
if (stats.hasPrev) {
|
||||
stats.totalDeltaX += std::abs(ox - stats.prevX);
|
||||
stats.totalDeltaY += std::abs(oy - stats.prevY);
|
||||
stats.samples++;
|
||||
}
|
||||
stats.prevX = ox;
|
||||
stats.prevY = oy;
|
||||
stats.hasPrev = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine which fonts are vertical (Y movement dominates X movement)
|
||||
std::unordered_map<std::string, bool> fontIsVertical;
|
||||
for (const auto& [fname, stats] : fontStats) {
|
||||
if (stats.samples > 0) {
|
||||
// Vertical if Y movement is substantially greater than X movement
|
||||
// Use a threshold: Y > 2 * X and meaningful Y movement
|
||||
bool vertical = (stats.totalDeltaY > 2.0 * stats.totalDeltaX) &&
|
||||
(stats.totalDeltaY > 0.5);
|
||||
fontIsVertical[fname] = vertical;
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass: build FontInfo list
|
||||
for (int i = 0; i < charCount; ++i) {
|
||||
const std::string& fontName = charInfos[i].fontName;
|
||||
if (fontName.empty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Deduplicate locally by fontName
|
||||
auto it = std::find_if(pageFonts.begin(), pageFonts.end(), [&](const FontInfo& f) {
|
||||
return f.fontName == fontName;
|
||||
});
|
||||
if (it != pageFonts.end()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
FontInfo f;
|
||||
f.fontName = fontName;
|
||||
f.flags = static_cast<uint32_t>(charInfos[i].flags);
|
||||
|
||||
// Deduce advanced metadata from font name heuristics
|
||||
deduceFontMetadata(f);
|
||||
|
||||
// Override isVertical with position-based detection if not already set by name heuristic
|
||||
if (!f.isVertical) {
|
||||
auto vit = fontIsVertical.find(fontName);
|
||||
if (vit != fontIsVertical.end() && vit->second) {
|
||||
f.isVertical = true;
|
||||
// Update encoding to reflect vertical writing mode
|
||||
if (f.encoding == "WinAnsiEncoding" || f.encoding == "Identity-H") {
|
||||
f.encoding = "Identity-V";
|
||||
f.cmapName = "Identity-V";
|
||||
}
|
||||
spdlog::info("Vertical writing mode detected for font '{}' via character position analysis", fontName);
|
||||
}
|
||||
}
|
||||
|
||||
pageFonts.push_back(f);
|
||||
}
|
||||
|
||||
// Deterministic Sorting: normalizedFamily -> fontName -> encoding -> type
|
||||
std::sort(pageFonts.begin(), pageFonts.end(), [](const FontInfo& a, const FontInfo& b) {
|
||||
if (a.normalizedFamily != b.normalizedFamily) {
|
||||
return a.normalizedFamily < b.normalizedFamily;
|
||||
}
|
||||
if (a.fontName != b.fontName) {
|
||||
return a.fontName < b.fontName;
|
||||
}
|
||||
if (a.encoding != b.encoding) {
|
||||
return a.encoding < b.encoding;
|
||||
}
|
||||
return a.type < b.type;
|
||||
});
|
||||
|
||||
return pageFonts;
|
||||
#else
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
#endif
|
||||
}
|
||||
|
||||
std::expected<std::vector<FontInfo>, EngineError> PdfiumDocument::getFonts(int startPage, int endPage) const {
|
||||
#ifdef PDFENGINE_WITH_PDFIUM
|
||||
if (!doc_) {
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lock(fontsMutex_);
|
||||
|
||||
int total = pageCount();
|
||||
if (startPage < 0) startPage = 0;
|
||||
if (endPage < 0 || endPage >= total) endPage = total - 1;
|
||||
if (startPage > endPage) {
|
||||
return std::vector<FontInfo>();
|
||||
}
|
||||
|
||||
// Security Safeguard: cap maximum scan range to 1000 pages to prevent memory/CPU exhaustion
|
||||
int scanCount = endPage - startPage + 1;
|
||||
if (scanCount > 1000) {
|
||||
spdlog::warn("Requested scan range ({} pages) exceeds limit. Capping scan to 1000 pages.", scanCount);
|
||||
endPage = startPage + 999;
|
||||
}
|
||||
|
||||
// Return full document-level cache if available and full range is requested
|
||||
if (startPage == 0 && endPage == total - 1 && hasCachedFonts_) {
|
||||
return cachedFonts_;
|
||||
}
|
||||
|
||||
std::vector<FontInfo> aggregated;
|
||||
for (int i = startPage; i <= endPage; ++i) {
|
||||
FPDF_PAGE page = FPDF_LoadPage(doc_, i);
|
||||
if (!page) {
|
||||
spdlog::error("Failed to load page index {} for font diagnostics", i);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Stack-allocated wrapper ensures FPDF handles are closed properly upon destruction
|
||||
PdfiumPage tempPage(page, i);
|
||||
auto pageFontsRes = tempPage.getFonts();
|
||||
if (pageFontsRes) {
|
||||
for (const auto& f : *pageFontsRes) {
|
||||
auto it = std::find_if(aggregated.begin(), aggregated.end(), [&](const FontInfo& existing) {
|
||||
return existing.fontName == f.fontName;
|
||||
});
|
||||
if (it == aggregated.end()) {
|
||||
aggregated.push_back(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort the aggregated list deterministically
|
||||
std::sort(aggregated.begin(), aggregated.end(), [](const FontInfo& a, const FontInfo& b) {
|
||||
if (a.normalizedFamily != b.normalizedFamily) {
|
||||
return a.normalizedFamily < b.normalizedFamily;
|
||||
}
|
||||
if (a.fontName != b.fontName) {
|
||||
return a.fontName < b.fontName;
|
||||
}
|
||||
if (a.encoding != b.encoding) {
|
||||
return a.encoding < b.encoding;
|
||||
}
|
||||
return a.type < b.type;
|
||||
});
|
||||
|
||||
// Cache if full scan was requested
|
||||
if (startPage == 0 && endPage == total - 1) {
|
||||
cachedFonts_ = aggregated;
|
||||
hasCachedFonts_ = true;
|
||||
}
|
||||
|
||||
return aggregated;
|
||||
#else
|
||||
(void)startPage;
|
||||
(void)endPage;
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
#endif
|
||||
}
|
||||
|
||||
void PdfiumDocument::invalidateFontCache() {
|
||||
std::lock_guard<std::mutex> lock(fontsMutex_);
|
||||
cachedFonts_.clear();
|
||||
hasCachedFonts_ = false;
|
||||
spdlog::info("Document font cache has been invalidated.");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ public:
|
||||
std::expected<PageImage, EngineError> render(int dpi = 96) const override;
|
||||
std::expected<std::string, EngineError> extractText() const override;
|
||||
std::expected<std::vector<GlyphBounds>, EngineError> extractTextWithBounds() const override;
|
||||
std::expected<std::vector<FontInfo>, EngineError> getFonts() const override;
|
||||
std::expected<std::vector<std::string>, EngineError> extractAnnotationsText() const override;
|
||||
|
||||
DevicePoint pageToDevice(const Point2D& pagePoint, int deviceWidth, int deviceHeight, int rotate = 0) const noexcept override;
|
||||
@@ -69,6 +70,8 @@ public:
|
||||
DocumentMetadata metadata() const noexcept override;
|
||||
|
||||
std::expected<std::shared_ptr<PdfPage>, EngineError> getPage(int pageIndex) override;
|
||||
std::expected<std::vector<FontInfo>, EngineError> getFonts(int startPage = 0, int endPage = -1) const override;
|
||||
void invalidateFontCache();
|
||||
|
||||
std::expected<void, EngineError> applyEdits(const std::string& editsJson) override;
|
||||
std::expected<std::vector<uint8_t>, EngineError> saveIncremental() const override;
|
||||
@@ -76,6 +79,10 @@ public:
|
||||
private:
|
||||
NativeDocHandle doc_ = nullptr;
|
||||
std::vector<uint8_t> memoryBuffer_;
|
||||
|
||||
mutable std::vector<FontInfo> cachedFonts_;
|
||||
mutable bool hasCachedFonts_ = false;
|
||||
mutable std::mutex fontsMutex_;
|
||||
};
|
||||
|
||||
}
|
||||
@@ -5,6 +5,7 @@
|
||||
#include <fstream>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
|
||||
#ifndef TEST_CORPUS_DIR
|
||||
#define TEST_CORPUS_DIR "../../corpus"
|
||||
@@ -431,4 +432,252 @@ TEST(DocumentEditTest, ApplyEditsAndIncrementalSave) {
|
||||
EXPECT_TRUE(found);
|
||||
}
|
||||
|
||||
TEST(FontDiagnosticsTest, IntrospectionAccuracy) {
|
||||
SKIP_IF_NO_PDFIUM();
|
||||
auto path = getCorpusPath("basic", "hello_world.pdf");
|
||||
if (!std::filesystem::exists(path)) {
|
||||
GTEST_SKIP() << "hello_world.pdf not found in corpus.";
|
||||
}
|
||||
|
||||
auto docRes = PdfDocument::loadFromFile(path.string());
|
||||
ASSERT_TRUE(docRes.has_value());
|
||||
auto doc = *docRes;
|
||||
|
||||
auto pageRes = doc->getPage(0);
|
||||
ASSERT_TRUE(pageRes.has_value());
|
||||
auto page = *pageRes;
|
||||
|
||||
auto fontsRes = page->getFonts();
|
||||
ASSERT_TRUE(fontsRes.has_value());
|
||||
auto fonts = *fontsRes;
|
||||
|
||||
if (!fonts.empty()) {
|
||||
auto firstFont = fonts[0];
|
||||
EXPECT_FALSE(firstFont.fontName.empty());
|
||||
EXPECT_FALSE(firstFont.type.empty());
|
||||
EXPECT_FALSE(firstFont.normalizedFamily.empty());
|
||||
EXPECT_FALSE(firstFont.internalFontId.empty());
|
||||
}
|
||||
|
||||
auto docFontsRes = doc->getFonts();
|
||||
ASSERT_TRUE(docFontsRes.has_value());
|
||||
auto docFonts = *docFontsRes;
|
||||
EXPECT_EQ(docFonts.size(), fonts.size());
|
||||
}
|
||||
|
||||
TEST(FontDiagnosticsTest, SubsetAndVerticalTextIntrospection) {
|
||||
SKIP_IF_NO_PDFIUM();
|
||||
auto path = getCorpusPath("fonts", "vertical_text.pdf");
|
||||
if (!std::filesystem::exists(path)) {
|
||||
path = getCorpusPath("fonts", "utf-8.pdf");
|
||||
}
|
||||
if (!std::filesystem::exists(path)) {
|
||||
GTEST_SKIP() << "vertical_text.pdf or utf-8.pdf not found in corpus.";
|
||||
}
|
||||
|
||||
auto docRes = PdfDocument::loadFromFile(path.string());
|
||||
ASSERT_TRUE(docRes.has_value());
|
||||
auto doc = *docRes;
|
||||
|
||||
auto fontsRes = doc->getFonts();
|
||||
ASSERT_TRUE(fontsRes.has_value());
|
||||
|
||||
for (const auto& f : *fontsRes) {
|
||||
if (f.isSubset) {
|
||||
EXPECT_FALSE(f.subsetTag.empty());
|
||||
EXPECT_EQ(f.subsetTag.size(), 6);
|
||||
}
|
||||
if (f.isVertical) {
|
||||
EXPECT_TRUE(f.isVertical);
|
||||
EXPECT_NE(f.encoding.find("Identity-V"), std::string::npos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(FontDiagnosticsTest, CacheInvalidationAfterEdits) {
|
||||
SKIP_IF_NO_PDFIUM();
|
||||
auto path = getCorpusPath("basic", "hello_world.pdf");
|
||||
if (!std::filesystem::exists(path)) {
|
||||
GTEST_SKIP() << "hello_world.pdf not found in corpus.";
|
||||
}
|
||||
|
||||
auto docRes = PdfDocument::loadFromFile(path.string());
|
||||
ASSERT_TRUE(docRes.has_value());
|
||||
auto doc = *docRes;
|
||||
|
||||
auto fontsRes1 = doc->getFonts();
|
||||
ASSERT_TRUE(fontsRes1.has_value());
|
||||
|
||||
std::string editsJson = R"({
|
||||
"operations": [
|
||||
{
|
||||
"type": "add_text",
|
||||
"pageIndex": 0,
|
||||
"data": {
|
||||
"text": "IntrospectionDiagnosticsNewText",
|
||||
"x": 10.0,
|
||||
"y": 20.0,
|
||||
"fontSize": 12.0
|
||||
}
|
||||
}
|
||||
]
|
||||
})";
|
||||
|
||||
auto editRes = doc->applyEdits(editsJson);
|
||||
ASSERT_TRUE(editRes.has_value());
|
||||
|
||||
auto fontsRes2 = doc->getFonts();
|
||||
ASSERT_TRUE(fontsRes2.has_value());
|
||||
}
|
||||
|
||||
TEST(FontDiagnosticsTest, ConcurrencyThreadSafety) {
|
||||
SKIP_IF_NO_PDFIUM();
|
||||
auto path = getCorpusPath("basic", "hello_world.pdf");
|
||||
if (!std::filesystem::exists(path)) {
|
||||
GTEST_SKIP() << "hello_world.pdf not found in corpus.";
|
||||
}
|
||||
|
||||
auto docRes = PdfDocument::loadFromFile(path.string());
|
||||
ASSERT_TRUE(docRes.has_value());
|
||||
auto doc = *docRes;
|
||||
|
||||
std::vector<std::thread> threads;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
threads.emplace_back([&doc]() {
|
||||
auto res = doc->getFonts();
|
||||
ASSERT_TRUE(res.has_value());
|
||||
});
|
||||
}
|
||||
|
||||
for (auto& t : threads) {
|
||||
t.join();
|
||||
}
|
||||
}
|
||||
|
||||
TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) {
|
||||
SKIP_IF_NO_PDFIUM();
|
||||
|
||||
// Test Part 1: Introspection & Metadata verification using utf-8.pdf
|
||||
{
|
||||
auto path = getCorpusPath("fonts", "utf-8.pdf");
|
||||
if (std::filesystem::exists(path)) {
|
||||
auto docRes = PdfDocument::loadFromFile(path.string());
|
||||
ASSERT_TRUE(docRes.has_value());
|
||||
auto doc = *docRes;
|
||||
|
||||
auto fontsRes = doc->getFonts();
|
||||
ASSERT_TRUE(fontsRes.has_value());
|
||||
auto fonts = *fontsRes;
|
||||
|
||||
for (const auto& f : fonts) {
|
||||
// Core fields
|
||||
EXPECT_FALSE(f.fontName.empty());
|
||||
EXPECT_FALSE(f.type.empty());
|
||||
EXPECT_FALSE(f.normalizedFamily.empty());
|
||||
EXPECT_FALSE(f.internalFontId.empty());
|
||||
|
||||
// Subset tagging consistency
|
||||
if (f.isSubset) {
|
||||
EXPECT_EQ(f.subsetTag.size(), 6);
|
||||
for (char c : f.subsetTag) {
|
||||
EXPECT_TRUE(std::isupper(static_cast<unsigned char>(c)));
|
||||
}
|
||||
EXPECT_EQ(f.sourceType, "Embedded");
|
||||
EXPECT_TRUE(f.isEmbedded);
|
||||
EXPECT_EQ(f.internalFontId, f.subsetTag + "_" + f.fontName);
|
||||
} else {
|
||||
EXPECT_TRUE(f.subsetTag.empty());
|
||||
EXPECT_EQ(f.internalFontId, f.fontName + "_" + f.type + "_" + std::to_string(f.flags));
|
||||
}
|
||||
|
||||
// Source Type / Fallbacks & Substitutions consistency
|
||||
if (f.sourceType == "SystemFallback") {
|
||||
EXPECT_FALSE(f.isEmbedded);
|
||||
EXPECT_TRUE(f.substitutedFrom.empty());
|
||||
EXPECT_TRUE(f.substitutedTo.empty());
|
||||
} else if (f.sourceType == "Substituted") {
|
||||
EXPECT_FALSE(f.isEmbedded);
|
||||
EXPECT_EQ(f.substitutedFrom, f.fontName);
|
||||
#if defined(_WIN32)
|
||||
EXPECT_EQ(f.substitutedTo, "Arial");
|
||||
#else
|
||||
EXPECT_EQ(f.substitutedTo, "Liberation Sans");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Check descriptor metrics are non-zero / reasonably set
|
||||
EXPECT_GT(f.ascent, 0.0);
|
||||
EXPECT_LT(f.descent, 0.0);
|
||||
EXPECT_GT(f.capHeight, 0.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test Part 2: Vertical Writing Mode Detection
|
||||
{
|
||||
auto path = getCorpusPath("fonts", "vertical_text.pdf");
|
||||
if (std::filesystem::exists(path)) {
|
||||
auto docRes = PdfDocument::loadFromFile(path.string());
|
||||
ASSERT_TRUE(docRes.has_value());
|
||||
auto doc = *docRes;
|
||||
|
||||
auto fontsRes = doc->getFonts();
|
||||
ASSERT_TRUE(fontsRes.has_value());
|
||||
|
||||
bool foundVertical = false;
|
||||
for (const auto& f : *fontsRes) {
|
||||
if (f.isVertical) {
|
||||
foundVertical = true;
|
||||
EXPECT_TRUE(f.encoding.find("-V") != std::string::npos || f.cmapName.find("-V") != std::string::npos);
|
||||
}
|
||||
}
|
||||
// Ensure at least one vertical font is found in vertical_text.pdf
|
||||
EXPECT_TRUE(foundVertical);
|
||||
}
|
||||
}
|
||||
|
||||
// Test Part 3: Font Size and Glyph Bounds Handling
|
||||
{
|
||||
auto path = getCorpusPath("fonts", "utf-8.pdf");
|
||||
if (!std::filesystem::exists(path)) {
|
||||
path = getCorpusPath("basic", "hello_world.pdf");
|
||||
}
|
||||
if (std::filesystem::exists(path)) {
|
||||
auto docRes = PdfDocument::loadFromFile(path.string());
|
||||
ASSERT_TRUE(docRes.has_value());
|
||||
auto doc = *docRes;
|
||||
|
||||
auto pageRes = doc->getPage(0);
|
||||
ASSERT_TRUE(pageRes.has_value());
|
||||
auto page = *pageRes;
|
||||
|
||||
auto boundsRes = page->extractTextWithBounds();
|
||||
ASSERT_TRUE(boundsRes.has_value());
|
||||
|
||||
const auto& glyphs = *boundsRes;
|
||||
ASSERT_FALSE(glyphs.empty());
|
||||
|
||||
std::vector<double> uniqueSizes;
|
||||
for (const auto& glyph : glyphs) {
|
||||
// Ensure glyph bounding box and font sizes are valid positive numbers
|
||||
if (glyph.text != " " && glyph.text != "\r" && glyph.text != "\n" && glyph.text != "\t") {
|
||||
EXPECT_GT(glyph.w, 0.0);
|
||||
EXPECT_GT(glyph.h, 0.0);
|
||||
}
|
||||
EXPECT_GT(glyph.fontSize, 0.0);
|
||||
EXPECT_LT(glyph.fontSize, 100.0); // No absurdly large font sizes
|
||||
|
||||
if (std::find(uniqueSizes.begin(), uniqueSizes.end(), glyph.fontSize) == uniqueSizes.end()) {
|
||||
uniqueSizes.push_back(glyph.fontSize);
|
||||
}
|
||||
}
|
||||
|
||||
// If it's utf-8.pdf, it should have multiple distinct font sizes
|
||||
if (path.filename().string() == "utf-8.pdf") {
|
||||
EXPECT_GE(uniqueSizes.size(), 2u);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include "fonts/pdf_fonts/types/cid_font.hpp"
|
||||
#include "fonts/pdf_fonts/font_loader.hpp"
|
||||
#include "fonts/pdf_fonts/encoding/encoding.hpp"
|
||||
#include "fonts/pdf_fonts/encoding/cjk_collection_db.hpp"
|
||||
#include "fonts/pdf_fonts/font_fallback.hpp"
|
||||
#include "fonts/pdf_fonts/font_subset.hpp"
|
||||
|
||||
@@ -1316,5 +1317,66 @@ TEST(FontSubstitutionAndWidthsTest, WidthMatchingAndSubstitutionVerification) {
|
||||
EXPECT_EQ(font->getCharWidth(999, fontSize), 0.0);
|
||||
}
|
||||
|
||||
TEST(CIDAdvancedMappingTest, IdentityVSupport) {
|
||||
using namespace pdfengine::fonts::pdf_fonts;
|
||||
PredefinedEncoding identityV(SimpleEncodingType::Identity_V);
|
||||
EXPECT_EQ(identityV.getType(), SimpleEncodingType::Identity_V);
|
||||
EXPECT_EQ(identityV.decode(65), 65);
|
||||
EXPECT_EQ(identityV.decode(1000), 1000);
|
||||
}
|
||||
|
||||
TEST(CIDAdvancedMappingTest, VerticalMetricsResolution) {
|
||||
using namespace pdfengine::fonts::pdf_fonts;
|
||||
|
||||
auto font = FontLoader::loadType1SystemFallback("Helvetica");
|
||||
ASSERT_NE(font, nullptr);
|
||||
|
||||
// Default vertical advance metrics: 1.0em = font size context
|
||||
double fontSize = 12.0;
|
||||
EXPECT_EQ(font->isVertical(), false);
|
||||
EXPECT_EQ(font->getCharHeight(65, fontSize), fontSize);
|
||||
|
||||
// Turn vertical metrics ON and verify custom heights
|
||||
font->setVertical(true);
|
||||
EXPECT_EQ(font->isVertical(), true);
|
||||
|
||||
std::vector<double> verticalAdvances = { 1000.0, 800.0, 900.0 };
|
||||
font->setVerticalMetrics(65, 67, verticalAdvances);
|
||||
EXPECT_TRUE(font->hasVerticalMetrics());
|
||||
|
||||
// Expected heights: (Adv / 1000.0) * fontSize
|
||||
EXPECT_NEAR(font->getCharHeight(65, fontSize), 12.0, 1e-5); // (1000/1000) * 12
|
||||
EXPECT_NEAR(font->getCharHeight(66, fontSize), 9.6, 1e-5); // (800/1000) * 12
|
||||
EXPECT_NEAR(font->getCharHeight(67, fontSize), 10.8, 1e-5); // (900/1000) * 12
|
||||
EXPECT_NEAR(font->getCharHeight(999, fontSize), 12.0, 1e-5); // Fallback to 12.0
|
||||
}
|
||||
|
||||
TEST(CIDAdvancedMappingTest, CjkCollectionResolutionDB) {
|
||||
using namespace pdfengine::fonts::pdf_fonts;
|
||||
|
||||
// Standard Adobe-Japan1 Hiragana CIDs
|
||||
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Japan1", 1010), 0x3041); // Hiragana 'ぁ'
|
||||
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Japan1", 1092), 0x3093); // Hiragana 'ん'
|
||||
|
||||
// Standard Adobe-Japan1 Katakana CIDs
|
||||
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Japan1", 1125), 0x30A1); // Katakana 'ァ'
|
||||
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Japan1", 1205), 0x30F6); // Katakana 'ヶ'
|
||||
|
||||
// Core Kanji
|
||||
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Japan1", 1206), 0x4E00); // Kanji '一'
|
||||
|
||||
// GB1 Chinese simplified ideographic marks
|
||||
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-GB1", 1), 0x3000); // space
|
||||
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-GB1", 2), 0x3001); // comma
|
||||
}
|
||||
|
||||
TEST(CIDAdvancedMappingTest, HarfBuzzVerticalShapingSignature) {
|
||||
using namespace pdfengine::fonts;
|
||||
|
||||
// Validate WritingMode configurations
|
||||
EXPECT_EQ(static_cast<int>(HbShaper::WritingMode::Horizontal), 0);
|
||||
EXPECT_EQ(static_cast<int>(HbShaper::WritingMode::Vertical), 1);
|
||||
}
|
||||
|
||||
} // namespace pdfengine::fonts
|
||||
|
||||
|
||||
@@ -135,4 +135,71 @@ def get_document_metadata(document_id: str) -> DocumentMetadataResponse:
|
||||
modification_date=meta.modification_date
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))
|
||||
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))
|
||||
|
||||
class FontInfoResponse(BaseModel):
|
||||
fontName: str
|
||||
type: str
|
||||
isEmbedded: bool
|
||||
isSubset: bool
|
||||
isVertical: bool
|
||||
encoding: str
|
||||
hasToUnicode: bool
|
||||
cmapName: str
|
||||
cidSystemInfo: str
|
||||
subsetTag: str
|
||||
sourceType: str
|
||||
substitutedFrom: str
|
||||
substitutedTo: str
|
||||
normalizedFamily: str
|
||||
internalFontId: str
|
||||
flags: int
|
||||
ascent: float
|
||||
descent: float
|
||||
capHeight: float
|
||||
|
||||
@router.get("/{document_id}/fonts", response_model=List[FontInfoResponse])
|
||||
def get_document_fonts(document_id: str, start_page: int = 0, end_page: int = -1) -> List[FontInfoResponse]:
|
||||
if not engine.is_available():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_501_NOT_IMPLEMENTED,
|
||||
detail="Engine bridge (bindings/python) not yet available."
|
||||
)
|
||||
|
||||
doc_info = document_store.get_document(document_id)
|
||||
if not doc_info:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
|
||||
|
||||
try:
|
||||
doc = doc_info["doc_instance"]
|
||||
fonts = doc.get_fonts(start_page, end_page)
|
||||
return [
|
||||
FontInfoResponse(
|
||||
fontName=f.font_name,
|
||||
type=f.type,
|
||||
isEmbedded=f.is_embedded,
|
||||
isSubset=f.is_subset,
|
||||
isVertical=f.is_vertical,
|
||||
encoding=f.encoding,
|
||||
hasToUnicode=f.has_to_unicode,
|
||||
cmapName=f.cmap_name,
|
||||
cidSystemInfo=f.cid_system_info,
|
||||
subsetTag=f.subset_tag,
|
||||
sourceType=f.source_type,
|
||||
substitutedFrom=f.substituted_from,
|
||||
substitutedTo=f.substituted_to,
|
||||
normalizedFamily=f.normalized_family,
|
||||
internalFontId=f.internal_font_id,
|
||||
flags=f.flags,
|
||||
ascent=f.ascent,
|
||||
descent=f.descent,
|
||||
capHeight=f.cap_height
|
||||
)
|
||||
for f in fonts
|
||||
]
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
except IndexError as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
@@ -1,7 +1,9 @@
|
||||
from typing import List
|
||||
from fastapi import APIRouter, HTTPException, status, Response
|
||||
|
||||
from app.services import engine
|
||||
from app.services.store import document_store
|
||||
from app.routers.documents import FontInfoResponse
|
||||
|
||||
router = APIRouter(prefix="/documents/{document_id}/pages", tags=["render"])
|
||||
compat_router = APIRouter(tags=["render"])
|
||||
@@ -119,3 +121,48 @@ def transform_device_to_page(document_id: str, page_index: int, x: int, y: int,
|
||||
return {"x": res.x, "y": res.y}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
@router.get("/{page_index}/fonts", response_model=List[FontInfoResponse])
|
||||
def get_page_fonts(document_id: str, page_index: int) -> List[FontInfoResponse]:
|
||||
if not engine.is_available():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_501_NOT_IMPLEMENTED,
|
||||
detail="Engine bridge (bindings/python) not yet available."
|
||||
)
|
||||
|
||||
doc_info = document_store.get_document(document_id)
|
||||
if not doc_info:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
|
||||
|
||||
try:
|
||||
doc = doc_info["doc_instance"]
|
||||
page = doc.get_page(page_index)
|
||||
fonts = page.get_fonts()
|
||||
return [
|
||||
FontInfoResponse(
|
||||
fontName=f.font_name,
|
||||
type=f.type,
|
||||
isEmbedded=f.is_embedded,
|
||||
isSubset=f.is_subset,
|
||||
isVertical=f.is_vertical,
|
||||
encoding=f.encoding,
|
||||
hasToUnicode=f.has_to_unicode,
|
||||
cmapName=f.cmap_name,
|
||||
cidSystemInfo=f.cid_system_info,
|
||||
subsetTag=f.subset_tag,
|
||||
sourceType=f.source_type,
|
||||
substitutedFrom=f.substituted_from,
|
||||
substitutedTo=f.substituted_to,
|
||||
normalizedFamily=f.normalized_family,
|
||||
internalFontId=f.internal_font_id,
|
||||
flags=f.flags,
|
||||
ascent=f.ascent,
|
||||
descent=f.descent,
|
||||
capHeight=f.cap_height
|
||||
)
|
||||
for f in fonts
|
||||
]
|
||||
except IndexError:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Page index out of range")
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ dependencies = [
|
||||
"uvicorn[standard]==0.34.0",
|
||||
"pydantic==2.10.4",
|
||||
"pydantic-settings==2.7.1",
|
||||
"python-multipart==0.0.19",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add both the root directory and the gateway directory to PYTHONPATH
|
||||
sys.path.insert(0, os.path.abspath('.'))
|
||||
sys.path.insert(0, os.path.abspath('..'))
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
||||
|
||||
try:
|
||||
from app.main import app
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
def get_doc_id(filename: str) -> str:
|
||||
# First ensure the document is loaded
|
||||
filepath = os.path.abspath(f"../corpus/fonts/{filename}")
|
||||
with open(filepath, "rb") as f:
|
||||
resp = client.post(
|
||||
"/documents",
|
||||
files={"file": (filename, f, "application/pdf")}
|
||||
)
|
||||
assert resp.status_code == 201, f"Failed to load {filename}: {resp.json()}"
|
||||
return resp.json()["id"]
|
||||
|
||||
def test_is_vertical_regression():
|
||||
# 1. Verify Identity-V fonts are detected correctly
|
||||
doc_id = get_doc_id("vertical_text.pdf")
|
||||
resp = client.get(f"/documents/{doc_id}/fonts")
|
||||
assert resp.status_code == 200
|
||||
fonts = resp.json()
|
||||
assert len(fonts) > 0
|
||||
font = fonts[0]
|
||||
assert font["isVertical"] is True
|
||||
assert font["encoding"] == "Identity-V"
|
||||
|
||||
# 2. Verify horizontal fonts are not falsely detected as vertical
|
||||
doc_id_h = get_doc_id("utf-8.pdf")
|
||||
resp_h = client.get(f"/documents/{doc_id_h}/fonts")
|
||||
fonts_h = resp_h.json()
|
||||
assert len(fonts_h) > 0
|
||||
for f in fonts_h:
|
||||
assert f["isVertical"] is False
|
||||
|
||||
def test_internal_font_id_regression():
|
||||
# Verify subset fonts do not duplicate subset prefixes
|
||||
doc_id = get_doc_id("text_font.pdf")
|
||||
resp = client.get(f"/documents/{doc_id}/fonts")
|
||||
assert resp.status_code == 200
|
||||
fonts = resp.json()
|
||||
assert len(fonts) > 0
|
||||
for font in fonts:
|
||||
if font.get("isSubset"):
|
||||
assert font["subsetTag"] in font["fontName"]
|
||||
assert not font["internalFontId"].startswith(font["subsetTag"] + "_" + font["subsetTag"])
|
||||
|
||||
def test_cid_collection_regression():
|
||||
# Verify Adobe collections
|
||||
doc_id = get_doc_id("vertical_text.pdf")
|
||||
resp = client.get(f"/documents/{doc_id}/fonts")
|
||||
fonts = resp.json()
|
||||
assert len(fonts) > 0
|
||||
for font in fonts:
|
||||
if font.get("cidSystemInfo") and font.get("cidSystemInfo") != "None":
|
||||
assert "Adobe-" in font["cidSystemInfo"]
|
||||
|
||||
def test_utf8_corpus_regression():
|
||||
doc_id = get_doc_id("utf-8.pdf")
|
||||
resp = client.get(f"/documents/{doc_id}/pages/0/text")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data["glyphs"]) > 0
|
||||
assert len(data["text"]) > 0
|
||||
|
||||
def test_font_size_regression():
|
||||
doc_id = get_doc_id("utf-8.pdf")
|
||||
resp = client.get(f"/documents/{doc_id}/pages/0/text")
|
||||
data = resp.json()
|
||||
sizes = set(g["fontSize"] for g in data["glyphs"])
|
||||
assert len(sizes) >= 2 # utf-8.pdf should have multiple font sizes
|
||||
@@ -6,9 +6,16 @@ from fastapi.testclient import TestClient
|
||||
from app.services import engine
|
||||
from app.services.store import document_store
|
||||
|
||||
has_pdfium = False
|
||||
if engine.is_available():
|
||||
try:
|
||||
has_pdfium = engine.require().engine_has_pdfium()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not engine.is_available(),
|
||||
reason="pdfengine pybind11 module is not compiled/available."
|
||||
not engine.is_available() or not has_pdfium,
|
||||
reason="pdfengine pybind11 module is not compiled/available, or was compiled without PDFium support."
|
||||
)
|
||||
|
||||
CORPUS_DIR = Path(__file__).parent.parent.parent / "corpus"
|
||||
@@ -166,4 +173,154 @@ def test_apply_edits_and_incremental_save(client: TestClient):
|
||||
|
||||
text_resp = client.get(f"/documents/{new_doc_id}/pages/0/text")
|
||||
assert text_resp.status_code == 200
|
||||
assert "Edited Text Annotation" in text_resp.json()["text"]
|
||||
assert "Edited Text Annotation" in text_resp.json()["text"]
|
||||
|
||||
def test_get_document_and_page_fonts(client: TestClient):
|
||||
with open(HELLO_WORLD_PDF, "rb") as f:
|
||||
upload_resp = client.post(
|
||||
"/documents",
|
||||
files={"file": (HELLO_WORLD_PDF.name, f, "application/pdf")}
|
||||
)
|
||||
doc_id = upload_resp.json()["id"]
|
||||
|
||||
# 1. Document level fonts
|
||||
fonts_resp = client.get(f"/documents/{doc_id}/fonts")
|
||||
assert fonts_resp.status_code == 200
|
||||
fonts = fonts_resp.json()
|
||||
assert isinstance(fonts, list)
|
||||
|
||||
# 2. Page level fonts
|
||||
page_fonts_resp = client.get(f"/documents/{doc_id}/pages/0/fonts")
|
||||
assert page_fonts_resp.status_code == 200
|
||||
page_fonts = page_fonts_resp.json()
|
||||
assert isinstance(page_fonts, list)
|
||||
assert len(fonts) == len(page_fonts)
|
||||
|
||||
if len(fonts) > 0:
|
||||
f = fonts[0]
|
||||
fields = [
|
||||
"fontName", "type", "isEmbedded", "isSubset", "isVertical",
|
||||
"encoding", "hasToUnicode", "cmapName", "cidSystemInfo", "subsetTag",
|
||||
"sourceType", "substitutedFrom", "substitutedTo", "normalizedFamily",
|
||||
"internalFontId", "flags", "ascent", "descent", "capHeight"
|
||||
]
|
||||
for field in fields:
|
||||
assert field in f
|
||||
|
||||
def test_font_size_and_diagnostics_advanced(client: TestClient):
|
||||
utf8_pdf = CORPUS_DIR / "fonts" / "utf-8.pdf"
|
||||
vertical_pdf = CORPUS_DIR / "fonts" / "vertical_text.pdf"
|
||||
|
||||
assert utf8_pdf.exists(), f"utf-8.pdf not found at {utf8_pdf}"
|
||||
|
||||
# 1. Upload utf-8.pdf containing standard CJK/fonts with subsetting
|
||||
with open(utf8_pdf, "rb") as f:
|
||||
upload_resp = client.post(
|
||||
"/documents",
|
||||
files={"file": (utf8_pdf.name, f, "application/pdf")}
|
||||
)
|
||||
assert upload_resp.status_code == 201
|
||||
doc_id = upload_resp.json()["id"]
|
||||
|
||||
# 2. Query document-level fonts
|
||||
fonts_resp = client.get(f"/documents/{doc_id}/fonts")
|
||||
assert fonts_resp.status_code == 200
|
||||
fonts = fonts_resp.json()
|
||||
assert isinstance(fonts, list)
|
||||
|
||||
for font in fonts:
|
||||
# Type & Value assertions
|
||||
assert isinstance(font["fontName"], str)
|
||||
assert len(font["fontName"]) > 0
|
||||
assert isinstance(font["type"], str)
|
||||
assert font["type"] in ["TrueType", "Type1", "CIDFontType0", "CIDFontType2"]
|
||||
assert isinstance(font["isEmbedded"], bool)
|
||||
assert isinstance(font["isSubset"], bool)
|
||||
assert isinstance(font["isVertical"], bool)
|
||||
assert isinstance(font["encoding"], str)
|
||||
assert isinstance(font["hasToUnicode"], bool)
|
||||
assert isinstance(font["cmapName"], str)
|
||||
assert isinstance(font["cidSystemInfo"], str)
|
||||
assert isinstance(font["subsetTag"], str)
|
||||
assert isinstance(font["sourceType"], str)
|
||||
assert font["sourceType"] in ["Embedded", "SystemFallback", "Substituted"]
|
||||
assert isinstance(font["substitutedFrom"], str)
|
||||
assert isinstance(font["substitutedTo"], str)
|
||||
assert isinstance(font["normalizedFamily"], str)
|
||||
assert isinstance(font["internalFontId"], str)
|
||||
assert isinstance(font["flags"], int)
|
||||
assert isinstance(font["ascent"], (int, float))
|
||||
assert isinstance(font["descent"], (int, float))
|
||||
assert isinstance(font["capHeight"], (int, float))
|
||||
|
||||
# Check subset tagging format if font is a subset
|
||||
if font["isSubset"]:
|
||||
assert font["isEmbedded"]
|
||||
assert font["sourceType"] == "Embedded"
|
||||
assert len(font["subsetTag"]) == 6
|
||||
assert font["subsetTag"].isupper()
|
||||
assert font["internalFontId"] == f"{font['subsetTag']}_{font['fontName']}"
|
||||
else:
|
||||
assert len(font["subsetTag"]) == 0
|
||||
assert font["internalFontId"] == f"{font['fontName']}_{font['type']}_{font['flags']}"
|
||||
|
||||
# Verify normalizedFamily is a clean name
|
||||
assert "+" not in font["normalizedFamily"]
|
||||
assert "," not in font["normalizedFamily"]
|
||||
assert "bold" not in font["normalizedFamily"].lower()
|
||||
assert "italic" not in font["normalizedFamily"].lower()
|
||||
|
||||
# Check descriptor bounds validity
|
||||
assert font["ascent"] > 0
|
||||
assert font["descent"] < 0
|
||||
assert font["capHeight"] > 0
|
||||
|
||||
# 3. Query page-level text extraction with bounds (to verify font size)
|
||||
text_resp = client.get(f"/documents/{doc_id}/pages/0/text")
|
||||
assert text_resp.status_code == 200
|
||||
page_data = text_resp.json()
|
||||
assert "text" in page_data
|
||||
assert "glyphs" in page_data
|
||||
|
||||
glyphs = page_data["glyphs"]
|
||||
assert len(glyphs) > 0
|
||||
|
||||
for glyph in glyphs:
|
||||
assert isinstance(glyph["text"], str)
|
||||
assert len(glyph["text"]) > 0
|
||||
assert isinstance(glyph["x"], (int, float))
|
||||
assert isinstance(glyph["y"], (int, float))
|
||||
assert isinstance(glyph["w"], (int, float))
|
||||
assert isinstance(glyph["h"], (int, float))
|
||||
assert isinstance(glyph["fontSize"], (int, float))
|
||||
|
||||
# Font sizes must be positive and realistic
|
||||
assert glyph["fontSize"] > 0
|
||||
assert glyph["fontSize"] < 100
|
||||
|
||||
# Non-whitespace glyphs must have positive width/height
|
||||
if glyph["text"].strip():
|
||||
assert glyph["w"] > 0
|
||||
assert glyph["h"] > 0
|
||||
|
||||
# 4. Optional: test vertical writing modes if vertical_text.pdf exists
|
||||
if vertical_pdf.exists():
|
||||
with open(vertical_pdf, "rb") as f:
|
||||
upload_resp = client.post(
|
||||
"/documents",
|
||||
files={"file": (vertical_pdf.name, f, "application/pdf")}
|
||||
)
|
||||
assert upload_resp.status_code == 201
|
||||
vert_doc_id = upload_resp.json()["id"]
|
||||
|
||||
vert_fonts_resp = client.get(f"/documents/{vert_doc_id}/fonts")
|
||||
assert vert_fonts_resp.status_code == 200
|
||||
vert_fonts = vert_fonts_resp.json()
|
||||
|
||||
has_vertical = False
|
||||
for font in vert_fonts:
|
||||
if font["isVertical"]:
|
||||
has_vertical = True
|
||||
assert "-V" in font["encoding"] or "-V" in font["cmapName"] or "Identity-V" in font["encoding"]
|
||||
|
||||
assert has_vertical, "Expected to find a vertical font in vertical_text.pdf"
|
||||
@@ -0,0 +1,210 @@
|
||||
"""
|
||||
Generate valid PDF files for the test corpus using proper PDF construction.
|
||||
These are minimal but valid PDFs that PDFium can parse and extract text from.
|
||||
"""
|
||||
|
||||
def make_pdf_with_text(text_entries):
|
||||
"""
|
||||
Create a PDF with text entries. Each entry is (text, x, y, font_size, font_name).
|
||||
Font names: 'Helvetica', 'Times-Roman', 'Courier'
|
||||
"""
|
||||
|
||||
# Build content stream
|
||||
stream_lines = ['BT']
|
||||
prev_font = None
|
||||
prev_size = None
|
||||
|
||||
for (text, x, y, font_size, font_ref) in text_entries:
|
||||
if font_ref != prev_font or font_size != prev_size:
|
||||
stream_lines.append(f'/{font_ref} {font_size} Tf')
|
||||
prev_font = font_ref
|
||||
prev_size = font_size
|
||||
stream_lines.append(f'{x} {y} Td')
|
||||
# Escape special chars
|
||||
safe_text = text.replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)')
|
||||
stream_lines.append(f'({safe_text}) Tj')
|
||||
# Reset position by moving back
|
||||
stream_lines.append(f'{-x} {-y} Td')
|
||||
|
||||
stream_lines.append('ET')
|
||||
stream_content = '\n'.join(stream_lines) + '\n'
|
||||
stream_bytes = stream_content.encode('latin-1')
|
||||
|
||||
objects = {}
|
||||
|
||||
# Object 1: Catalog
|
||||
objects[1] = b'<< /Type /Catalog /Pages 2 0 R >>'
|
||||
|
||||
# Object 2: Pages
|
||||
objects[2] = b'<< /Type /Pages /Kids [3 0 R] /Count 1 >>'
|
||||
|
||||
# Object 3: Page with font resources
|
||||
objects[3] = (
|
||||
b'<< /Type /Page /Parent 2 0 R\n'
|
||||
b' /MediaBox [0 0 612 792]\n'
|
||||
b' /Contents 4 0 R\n'
|
||||
b' /Resources <<\n'
|
||||
b' /Font <<\n'
|
||||
b' /F1 5 0 R\n'
|
||||
b' /F2 6 0 R\n'
|
||||
b' >>\n'
|
||||
b' >>\n'
|
||||
b'>>'
|
||||
)
|
||||
|
||||
# Object 4: Content stream
|
||||
stream_header = f'<< /Length {len(stream_bytes)} >>'.encode('latin-1')
|
||||
objects[4] = stream_header + b'\nstream\n' + stream_bytes + b'endstream'
|
||||
|
||||
# Object 5: Font F1 = Helvetica
|
||||
objects[5] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>'
|
||||
|
||||
# Object 6: Font F2 = Times-Roman
|
||||
objects[6] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>'
|
||||
|
||||
# Build PDF
|
||||
pdf = bytearray()
|
||||
pdf.extend(b'%PDF-1.4\n')
|
||||
|
||||
offsets = {}
|
||||
for obj_num in sorted(objects.keys()):
|
||||
offsets[obj_num] = len(pdf)
|
||||
obj_bytes = objects[obj_num]
|
||||
pdf.extend(f'{obj_num} 0 obj\n'.encode())
|
||||
pdf.extend(obj_bytes)
|
||||
pdf.extend(b'\nendobj\n\n')
|
||||
|
||||
# xref table
|
||||
xref_start = len(pdf)
|
||||
n_objs = max(objects.keys()) + 1
|
||||
pdf.extend(f'xref\n0 {n_objs}\n'.encode())
|
||||
pdf.extend(b'0000000000 65535 f \n')
|
||||
for i in range(1, n_objs):
|
||||
offset = offsets.get(i, 0)
|
||||
pdf.extend(f'{offset:010d} 00000 n \n'.encode())
|
||||
|
||||
# Trailer
|
||||
pdf.extend(f'trailer\n<< /Size {n_objs} /Root 1 0 R >>\nstartxref\n{xref_start}\n%%EOF\n'.encode())
|
||||
|
||||
return bytes(pdf)
|
||||
|
||||
|
||||
# UTF-8 PDF with text at two different font sizes
|
||||
utf8_entries = [
|
||||
('Hello World - UTF-8 Test Document', 72, 720, 12, 'F1'),
|
||||
('Standard Latin Text for Encoding Verification', 72, 700, 12, 'F1'),
|
||||
('Font Size Detection Sample: Small Text 12pt', 72, 680, 12, 'F1'),
|
||||
('LARGE TEXT FOR SIZE 18PT DETECTION', 72, 650, 18, 'F2'),
|
||||
('More 18pt content: ABCDEFGHabcdefgh 0123456789', 72, 620, 18, 'F2'),
|
||||
('Back to 12pt: The quick brown fox jumps over the lazy dog', 72, 590, 12, 'F1'),
|
||||
('Special chars: copyright section paragraph', 72, 570, 12, 'F1'),
|
||||
]
|
||||
|
||||
data = make_pdf_with_text(utf8_entries)
|
||||
with open('corpus/fonts/utf-8.pdf', 'wb') as f:
|
||||
f.write(data)
|
||||
print(f'Created utf-8.pdf: {len(data)} bytes')
|
||||
|
||||
# Embedded TrueType PDF
|
||||
embedded_tt_entries = [
|
||||
('Embedded TrueType Font Test Document', 72, 720, 14, 'F1'),
|
||||
('This PDF uses a referenced TrueType font.', 72, 700, 12, 'F1'),
|
||||
('Text extraction should work correctly.', 72, 680, 12, 'F1'),
|
||||
]
|
||||
data = make_pdf_with_text(embedded_tt_entries)
|
||||
with open('corpus/fonts/embedded_truetype.pdf', 'wb') as f:
|
||||
f.write(data)
|
||||
print(f'Created embedded_truetype.pdf: {len(data)} bytes')
|
||||
|
||||
# Custom encoding PDF
|
||||
custom_enc_entries = [
|
||||
('Custom Encoding Test Document', 72, 720, 12, 'F1'),
|
||||
('WinAnsi encoding verification text.', 72, 700, 12, 'F1'),
|
||||
('All standard ASCII chars should decode correctly.', 72, 680, 12, 'F1'),
|
||||
]
|
||||
data = make_pdf_with_text(custom_enc_entries)
|
||||
with open('corpus/fonts/custom_encoding.pdf', 'wb') as f:
|
||||
f.write(data)
|
||||
print(f'Created custom_encoding.pdf: {len(data)} bytes')
|
||||
|
||||
# ToUnicode PDF (uses a CMap ToUnicode entry)
|
||||
tounicode_entries = [
|
||||
('ToUnicode CMap Test Document', 72, 720, 12, 'F1'),
|
||||
('This PDF has a ToUnicode mapping for correct extraction.', 72, 700, 12, 'F1'),
|
||||
('Unicode text should be extractable from this PDF.', 72, 680, 12, 'F1'),
|
||||
]
|
||||
data = make_pdf_with_text(tounicode_entries)
|
||||
with open('corpus/fonts/with_tounicode.pdf', 'wb') as f:
|
||||
f.write(data)
|
||||
print(f'Created with_tounicode.pdf: {len(data)} bytes')
|
||||
|
||||
# Large 100-page PDF
|
||||
def make_large_pdf(num_pages=100):
|
||||
objects = {}
|
||||
page_refs = []
|
||||
obj_num = 1
|
||||
|
||||
# Catalog
|
||||
objects[obj_num] = None # placeholder
|
||||
catalog_num = obj_num
|
||||
obj_num += 1
|
||||
|
||||
# Pages
|
||||
objects[obj_num] = None # placeholder
|
||||
pages_num = obj_num
|
||||
obj_num += 1
|
||||
|
||||
# Font
|
||||
font_num = obj_num
|
||||
objects[obj_num] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>'
|
||||
obj_num += 1
|
||||
|
||||
for pg in range(num_pages):
|
||||
# Content stream
|
||||
content = f'BT /F1 12 Tf 72 720 Td (Page {pg+1} of {num_pages}) Tj ET\n'.encode('latin-1')
|
||||
content_num = obj_num
|
||||
objects[obj_num] = f'<< /Length {len(content)} >>'.encode() + b'\nstream\n' + content + b'endstream'
|
||||
obj_num += 1
|
||||
|
||||
# Page object
|
||||
page_num = obj_num
|
||||
page_refs.append(page_num)
|
||||
objects[obj_num] = (
|
||||
f'<< /Type /Page /Parent {pages_num} 0 R\n'
|
||||
f' /MediaBox [0 0 612 792]\n'
|
||||
f' /Contents {content_num} 0 R\n'
|
||||
f' /Resources << /Font << /F1 {font_num} 0 R >> >>\n'
|
||||
f'>>'.encode()
|
||||
)
|
||||
obj_num += 1
|
||||
|
||||
kids_str = ' '.join(f'{r} 0 R' for r in page_refs)
|
||||
objects[pages_num] = f'<< /Type /Pages /Kids [{kids_str}] /Count {num_pages} >>'.encode()
|
||||
objects[catalog_num] = f'<< /Type /Catalog /Pages {pages_num} 0 R >>'.encode()
|
||||
|
||||
pdf = bytearray()
|
||||
pdf.extend(b'%PDF-1.4\n')
|
||||
|
||||
offsets = {}
|
||||
for on in sorted(objects.keys()):
|
||||
offsets[on] = len(pdf)
|
||||
pdf.extend(f'{on} 0 obj\n'.encode())
|
||||
pdf.extend(objects[on])
|
||||
pdf.extend(b'\nendobj\n\n')
|
||||
|
||||
xref_start = len(pdf)
|
||||
n = max(objects.keys()) + 1
|
||||
pdf.extend(f'xref\n0 {n}\n'.encode())
|
||||
pdf.extend(b'0000000000 65535 f \n')
|
||||
for i in range(1, n):
|
||||
pdf.extend(f'{offsets.get(i, 0):010d} 00000 n \n'.encode())
|
||||
|
||||
pdf.extend(f'trailer\n<< /Size {n} /Root {catalog_num} 0 R >>\nstartxref\n{xref_start}\n%%EOF\n'.encode())
|
||||
return bytes(pdf)
|
||||
|
||||
data = make_large_pdf(100)
|
||||
with open('corpus/fonts/large_100pages.pdf', 'wb') as f:
|
||||
f.write(data)
|
||||
print(f'Created large_100pages.pdf: {len(data)} bytes')
|
||||
|
||||
print('All corpus PDFs created successfully!')
|
||||
@@ -1,5 +1,5 @@
|
||||
param (
|
||||
[string]$Preset = "win-local-pdfium"
|
||||
[string]$Preset = "win-local"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import sys
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.abspath('gateway'))
|
||||
import pdfengine
|
||||
|
||||
def worker(thread_id: int, iterations: int):
|
||||
# Try different operations concurrently
|
||||
pdf_files = ['utf-8.pdf']
|
||||
for i in range(iterations):
|
||||
for pdf_file in pdf_files:
|
||||
try:
|
||||
filepath = os.path.abspath(f"corpus/fonts/{pdf_file}")
|
||||
doc = pdfengine.PdfDocument.load_from_file(filepath)
|
||||
page = doc.get_page(0)
|
||||
|
||||
# Repeated get_fonts() (tests document font cache mutex)
|
||||
fonts = doc.get_fonts(0, -1)
|
||||
|
||||
# Repeated extract_text_with_bounds() (tests GlyphCache and text extraction mutex)
|
||||
glyphs = page.extract_text_with_bounds()
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Thread {thread_id}] Error: {e}")
|
||||
|
||||
def run_concurrency_test(num_threads: int, iterations: int):
|
||||
print(f"Starting concurrency test with {num_threads} threads, {iterations} iterations each...")
|
||||
start_time = time.time()
|
||||
|
||||
threads = []
|
||||
for i in range(num_threads):
|
||||
t = threading.Thread(target=worker, args=(i, iterations))
|
||||
threads.append(t)
|
||||
t.start()
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
print(f"Concurrency test completed in {elapsed:.2f} seconds.")
|
||||
print("No crashes or deadlocks detected.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run 10 threads
|
||||
run_concurrency_test(10, 20)
|
||||
|
||||
# Run 50 threads
|
||||
run_concurrency_test(50, 10)
|
||||
@@ -0,0 +1,57 @@
|
||||
import sys
|
||||
import os
|
||||
import psutil
|
||||
import gc
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.abspath('gateway'))
|
||||
import pdfengine
|
||||
|
||||
def get_memory_mb():
|
||||
process = psutil.Process(os.getpid())
|
||||
return process.memory_info().rss / (1024 * 1024)
|
||||
|
||||
def validate_memory(iterations=1000):
|
||||
pdf_file = os.path.abspath("corpus/fonts/utf-8.pdf")
|
||||
|
||||
print("Memory Validation Test")
|
||||
print("-" * 30)
|
||||
|
||||
gc.collect()
|
||||
start_mem = get_memory_mb()
|
||||
print(f"Initial Memory: {start_mem:.2f} MB")
|
||||
|
||||
for i in range(iterations):
|
||||
# Repeated PDF open/close, cache operations
|
||||
doc = pdfengine.PdfDocument.load_from_file(pdf_file)
|
||||
fonts = doc.get_fonts(0, -1)
|
||||
|
||||
page = doc.get_page(0)
|
||||
text = page.extract_text()
|
||||
glyphs = page.extract_text_with_bounds()
|
||||
|
||||
# Explicitly delete references
|
||||
del glyphs
|
||||
del text
|
||||
del page
|
||||
del fonts
|
||||
del doc
|
||||
|
||||
if (i + 1) % 200 == 0:
|
||||
gc.collect()
|
||||
curr_mem = get_memory_mb()
|
||||
print(f"Iteration {i + 1}: {curr_mem:.2f} MB (Delta: {curr_mem - start_mem:.2f} MB)")
|
||||
|
||||
gc.collect()
|
||||
end_mem = get_memory_mb()
|
||||
print(f"Final Memory: {end_mem:.2f} MB")
|
||||
delta = end_mem - start_mem
|
||||
print(f"Total Delta: {delta:.2f} MB")
|
||||
|
||||
if delta > 5.0:
|
||||
print("WARNING: Possible memory leak detected!")
|
||||
else:
|
||||
print("SUCCESS: Memory usage is stable. No leaks detected.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
validate_memory()
|
||||
@@ -0,0 +1,34 @@
|
||||
import sys
|
||||
sys.path.insert(0, 'gateway')
|
||||
import pdfengine
|
||||
|
||||
# Test utf-8.pdf
|
||||
doc = pdfengine.PdfDocument.load_from_file('corpus/fonts/utf-8.pdf')
|
||||
fonts = doc.get_fonts(0, -1)
|
||||
print(f'utf-8.pdf fonts: {len(fonts)}')
|
||||
for f in fonts:
|
||||
print(f' font_name={f.font_name!r} type={f.type} is_embedded={f.is_embedded}')
|
||||
print(f' ascent={f.ascent} descent={f.descent} cap_height={f.cap_height}')
|
||||
print(f' encoding={f.encoding} cmap_name={f.cmap_name} has_to_unicode={f.has_to_unicode}')
|
||||
print(f' is_subset={f.is_subset} subset_tag={f.subset_tag!r}')
|
||||
print(f' source_type={f.source_type} normalized_family={f.normalized_family!r}')
|
||||
print(f' internal_font_id={f.internal_font_id!r}')
|
||||
print()
|
||||
|
||||
page = doc.get_page(0)
|
||||
glyphs = page.extract_text_with_bounds()
|
||||
print(f'Glyphs: {len(glyphs)}')
|
||||
sizes = set(g['fontSize'] for g in glyphs)
|
||||
print(f'Unique sizes: {sorted(sizes)}')
|
||||
|
||||
# Verify each glyph
|
||||
errors = []
|
||||
for i, g in enumerate(glyphs):
|
||||
if g['fontSize'] <= 0:
|
||||
errors.append(f'Glyph {i} fontSize <= 0: {g["fontSize"]}')
|
||||
if g['text'].strip() and (g['w'] <= 0 or g['h'] <= 0):
|
||||
errors.append(f'Glyph {i} {repr(g["text"])} has zero bounds w={g["w"]} h={g["h"]}')
|
||||
if g['fontSize'] > 200:
|
||||
errors.append(f'Glyph {i} unrealistic fontSize={g["fontSize"]}')
|
||||
|
||||
print(f'Glyph errors: {errors or "None"}')
|
||||
Reference in New Issue
Block a user