diff --git a/api_validate.py b/api_validate.py new file mode 100644 index 0000000..71a7cc1 --- /dev/null +++ b/api_validate.py @@ -0,0 +1,207 @@ +"""Live API validation script for the PDF engine gateway.""" +import sys, json, urllib.request, urllib.parse +sys.path.insert(0, 'gateway') + +BASE = 'http://localhost:8000' + +def http_get(url): + try: + resp = urllib.request.urlopen(url) + return json.loads(resp.read()), resp.status + except urllib.error.HTTPError as e: + return json.loads(e.read()), e.code + +def multipart_upload(url, filepath, filename): + boundary = 'Xboundary1234X' + with open(filepath, 'rb') as f: + file_data = f.read() + header = ( + '--' + boundary + '\r\n' + 'Content-Disposition: form-data; name="file"; filename="' + filename + '"\r\n' + 'Content-Type: application/pdf\r\n\r\n' + ).encode('utf-8') + footer = ('\r\n--' + boundary + '--\r\n').encode('utf-8') + body = header + file_data + footer + req = urllib.request.Request( + url, data=body, + headers={'Content-Type': 'multipart/form-data; boundary=' + boundary}, + method='POST' + ) + try: + resp = urllib.request.urlopen(req) + return json.loads(resp.read()), resp.status + except urllib.error.HTTPError as e: + return json.loads(e.read()), e.code + +results = {} + +# 1. Health check +health, status = http_get(f'{BASE}/health') +print(f"[HEALTH] status={status} response={health}") +results['health'] = {'status': status, 'passed': status == 200} + +# 2. Upload hello_world.pdf +print("\n=== UPLOADING hello_world.pdf ===") +doc, status = multipart_upload(f'{BASE}/documents', 'corpus/basic/hello_world.pdf', 'hello_world.pdf') +print(f"Upload status={status}") +print(f"Response: {doc}") +doc_id = doc.get('id') +results['upload'] = {'status': status, 'passed': status == 201, 'doc_id': doc_id} + +if not doc_id: + print("FATAL: No doc_id, cannot continue") + sys.exit(1) + +# 3. GET /documents/{id}/fonts +print(f"\n=== GET /documents/{doc_id}/fonts ===") +fonts, status = http_get(f'{BASE}/documents/{doc_id}/fonts') +print(f"Status={status}, Font count={len(fonts)}") +results['doc_fonts'] = {'status': status, 'count': len(fonts), 'passed': status == 200} + +ALL_FONT_FIELDS = [ + 'fontName','type','isEmbedded','isSubset','isVertical', + 'encoding','hasToUnicode','cmapName','cidSystemInfo','subsetTag', + 'sourceType','substitutedFrom','substitutedTo','normalizedFamily', + 'internalFontId','flags','ascent','descent','capHeight' +] + +schema_errors = [] +for f in fonts: + missing = [k for k in ALL_FONT_FIELDS if k not in f] + if missing: + schema_errors.append(f"Missing fields: {missing} in font {f.get('fontName')}") + # Type checks + for boolField in ['isEmbedded','isSubset','isVertical','hasToUnicode']: + if not isinstance(f.get(boolField), bool): + schema_errors.append(f"Field {boolField} should be bool, got {type(f.get(boolField))}") + for floatField in ['ascent','descent','capHeight']: + if not isinstance(f.get(floatField), (int, float)): + schema_errors.append(f"Field {floatField} should be float") + if not isinstance(f.get('flags'), int): + schema_errors.append("Field flags should be int") + print(f" Font: {f.get('fontName'):30s} type={f.get('type'):15s} embedded={f.get('isEmbedded')} subset={f.get('isSubset')} vertical={f.get('isVertical')}") + print(f" encoding={f.get('encoding')} cmapName={f.get('cmapName')} cidSystemInfo={f.get('cidSystemInfo')}") + print(f" subsetTag={f.get('subsetTag')} sourceType={f.get('sourceType')} normalizedFamily={f.get('normalizedFamily')}") + print(f" internalFontId={f.get('internalFontId')} flags={f.get('flags')}") + print(f" ascent={f.get('ascent')} descent={f.get('descent')} capHeight={f.get('capHeight')}") + +results['doc_fonts']['schema_errors'] = schema_errors +print(f"Schema errors: {schema_errors or 'None'}") + +# 4. GET /documents/{id}/pages/0/fonts +print(f"\n=== GET /documents/{doc_id}/pages/0/fonts ===") +pg_fonts, status = http_get(f'{BASE}/documents/{doc_id}/pages/0/fonts') +print(f"Status={status}, Page font count={len(pg_fonts)}") +results['page_fonts'] = {'status': status, 'count': len(pg_fonts), 'passed': status == 200} + +# Consistency check: doc-level vs page-level +if len(fonts) != len(pg_fonts): + print(f"WARNING: doc-level fonts ({len(fonts)}) != page-level fonts ({len(pg_fonts)})") + results['page_fonts']['consistency_warning'] = True +else: + print("Consistency: doc-level and page-level font counts match OK") + +# 5. GET /documents/{id}/pages/0/text +print(f"\n=== GET /documents/{doc_id}/pages/0/text ===") +text_data, status = http_get(f'{BASE}/documents/{doc_id}/pages/0/text') +print(f"Status={status}") +results['page_text'] = {'status': status, 'passed': status == 200} + +text = text_data.get('text', '') +glyphs = text_data.get('glyphs', []) +print(f"Text: {repr(text[:80])}") +print(f"Glyph count: {len(glyphs)}") + +glyph_errors = [] +for i, g in enumerate(glyphs): + for field in ['text','x','y','w','h','fontSize']: + if field not in g: + glyph_errors.append(f"Glyph {i} missing field {field}") + if g.get('fontSize', 0) <= 0: + glyph_errors.append(f"Glyph {i} '{g.get('text')}' has fontSize <= 0: {g.get('fontSize')}") + if g.get('text','').strip(): + if g.get('w', 0) <= 0 or g.get('h', 0) <= 0: + glyph_errors.append(f"Glyph {i} '{g.get('text')}' has zero bounds w={g.get('w')} h={g.get('h')}") + if g.get('fontSize', 0) > 200: + glyph_errors.append(f"Glyph {i} unrealistic fontSize={g.get('fontSize')}") + +results['page_text']['glyph_count'] = len(glyphs) +results['page_text']['glyph_errors'] = glyph_errors +print(f"Glyph validation errors: {glyph_errors or 'None'}") +if glyphs: + print(f"Sample glyphs: {glyphs[:3]}") + +# 6. Test 404 for non-existent document +print("\n=== TEST 404 ===") +not_found, status = http_get(f'{BASE}/documents/nonexistent-uuid/fonts') +print(f"404 test status={status} (expected 404)") +results['not_found'] = {'status': status, 'passed': status == 404} + +# 7. Test page out of bounds +print("\n=== TEST PAGE OUT OF BOUNDS ===") +oob, status = http_get(f'{BASE}/documents/{doc_id}/pages/99/fonts') +print(f"OOB test status={status} (expected 4xx)") +results['oob'] = {'status': status, 'passed': status in (400, 404)} + +# 8. Upload vertical_text.pdf and check vertical detection +print("\n=== UPLOADING vertical_text.pdf ===") +vert_doc, status = multipart_upload(f'{BASE}/documents', 'corpus/fonts/vertical_text.pdf', 'vertical_text.pdf') +vert_id = vert_doc.get('id') +print(f"Upload status={status} doc_id={vert_id}") +if vert_id: + vert_fonts, status = http_get(f'{BASE}/documents/{vert_id}/fonts') + print(f"Vertical PDF fonts ({len(vert_fonts)}):") + for vf in vert_fonts: + print(f" fontName={vf['fontName']} isVertical={vf['isVertical']} encoding={vf['encoding']}") + +# 9. Upload latin_extended.pdf and validate metrics +print("\n=== UPLOADING latin_extended.pdf ===") +lat_doc, status = multipart_upload(f'{BASE}/documents', 'corpus/fonts/latin_extended.pdf', 'latin_extended.pdf') +lat_id = lat_doc.get('id') +if lat_id: + lat_fonts, st = http_get(f'{BASE}/documents/{lat_id}/fonts') + print(f"Latin extended fonts ({len(lat_fonts)}):") + for lf in lat_fonts: + ascent_ok = lf['ascent'] > 0 + descent_ok = lf['descent'] < 0 + cap_ok = lf['capHeight'] > 0 + print(f" {lf['fontName']} ascent={lf['ascent']}({'OK' if ascent_ok else 'FAIL'}) descent={lf['descent']}({'OK' if descent_ok else 'FAIL'}) capHeight={lf['capHeight']}({'OK' if cap_ok else 'FAIL'})") + +# 10. Repeated requests (cache consistency) +print("\n=== CACHE CONSISTENCY (3 repeated font requests) ===") +responses = [] +for _ in range(3): + fonts_rep, st = http_get(f'{BASE}/documents/{doc_id}/fonts') + responses.append(len(fonts_rep)) +print(f"Font counts on repeated requests: {responses}") +results['cache_consistency'] = {'counts': responses, 'passed': len(set(responses)) == 1} + +# 11. Invalid PDF upload +print("\n=== INVALID PDF UPLOAD ===") +boundary = 'Xboundary1234X' +garbage = b'NOT A PDF FILE AT ALL 12345' +header = ('--' + boundary + '\r\nContent-Disposition: form-data; name="file"; filename="bad.pdf"\r\nContent-Type: application/pdf\r\n\r\n').encode() +footer = ('\r\n--' + boundary + '--\r\n').encode() +req = urllib.request.Request( + f'{BASE}/documents', + data=header + garbage + footer, + headers={'Content-Type': 'multipart/form-data; boundary=' + boundary}, + method='POST' +) +try: + resp = urllib.request.urlopen(req) + bad_result = json.loads(resp.read()), resp.status + print(f"UNEXPECTED SUCCESS: {bad_result}") + results['invalid_upload'] = {'passed': False} +except urllib.error.HTTPError as e: + err_detail = json.loads(e.read()) + print(f"Invalid PDF correctly rejected: status={e.code} detail={err_detail}") + results['invalid_upload'] = {'status': e.code, 'passed': e.code == 400} + +print("\n" + "="*60) +print("VALIDATION SUMMARY") +print("="*60) +for test, res in results.items(): + passed = res.get('passed', '?') + status = res.get('status', '-') + print(f" {'PASS' if passed else 'FAIL'}: {test:35s} status={status}") diff --git a/corpus/fonts/custom_encoding.pdf b/corpus/fonts/custom_encoding.pdf new file mode 100644 index 0000000..52a7b9e --- /dev/null +++ b/corpus/fonts/custom_encoding.pdf @@ -0,0 +1,62 @@ +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj + +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj + +3 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 4 0 R + /Resources << + /Font << + /F1 5 0 R + /F2 6 0 R + >> + >> +>> +endobj + +4 0 obj +<< /Length 213 >> +stream +BT +/F1 12 Tf +72 720 Td +(Custom Encoding Test Document) Tj +-72 -720 Td +72 700 Td +(WinAnsi encoding verification text.) Tj +-72 -700 Td +72 680 Td +(All standard ASCII chars should decode correctly.) Tj +-72 -680 Td +ET +endstream +endobj + +5 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >> +endobj + +6 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >> +endobj + +xref +0 7 +0000000000 65535 f +0000000009 00000 n +0000000059 00000 n +0000000117 00000 n +0000000290 00000 n +0000000554 00000 n +0000000652 00000 n +trailer +<< /Size 7 /Root 1 0 R >> +startxref +752 +%%EOF diff --git a/corpus/fonts/embedded_cid_font.pdf b/corpus/fonts/embedded_cid_font.pdf new file mode 100644 index 0000000..15271e7 --- /dev/null +++ b/corpus/fonts/embedded_cid_font.pdf @@ -0,0 +1 @@ +%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 58 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(CID Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type0 /BaseFont /NotoSansCJKjp-Regular /Encoding /Identity-H /DescendantFonts [6 0 R] >>\nendobj\n\n6 0 obj\n<< /Type /Font /Subtype /CIDFontType2 /BaseFont /NotoSansCJKjp-Regular /CIDSystemInfo << /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >> >>\nendobj\n\nxref\n0 7\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000384 00000 n \n0000000518 00000 n \ntrailer\n<< /Size 7 /Root 1 0 R >>\nstartxref\n684\n%%EOF\n \ No newline at end of file diff --git a/corpus/fonts/embedded_truetype.pdf b/corpus/fonts/embedded_truetype.pdf new file mode 100644 index 0000000..9c654d7 --- /dev/null +++ b/corpus/fonts/embedded_truetype.pdf @@ -0,0 +1,63 @@ +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj + +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj + +3 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 4 0 R + /Resources << + /Font << + /F1 5 0 R + /F2 6 0 R + >> + >> +>> +endobj + +4 0 obj +<< /Length 225 >> +stream +BT +/F1 14 Tf +72 720 Td +(Embedded TrueType Font Test Document) Tj +-72 -720 Td +/F1 12 Tf +72 700 Td +(This PDF uses a referenced TrueType font.) Tj +-72 -700 Td +72 680 Td +(Text extraction should work correctly.) Tj +-72 -680 Td +ET +endstream +endobj + +5 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >> +endobj + +6 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >> +endobj + +xref +0 7 +0000000000 65535 f +0000000009 00000 n +0000000059 00000 n +0000000117 00000 n +0000000290 00000 n +0000000566 00000 n +0000000664 00000 n +trailer +<< /Size 7 /Root 1 0 R >> +startxref +764 +%%EOF diff --git a/corpus/fonts/large_100pages.pdf b/corpus/fonts/large_100pages.pdf new file mode 100644 index 0000000..188b62a --- /dev/null +++ b/corpus/fonts/large_100pages.pdf @@ -0,0 +1,1724 @@ +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj + +2 0 obj +<< /Type /Pages /Kids [5 0 R 7 0 R 9 0 R 11 0 R 13 0 R 15 0 R 17 0 R 19 0 R 21 0 R 23 0 R 25 0 R 27 0 R 29 0 R 31 0 R 33 0 R 35 0 R 37 0 R 39 0 R 41 0 R 43 0 R 45 0 R 47 0 R 49 0 R 51 0 R 53 0 R 55 0 R 57 0 R 59 0 R 61 0 R 63 0 R 65 0 R 67 0 R 69 0 R 71 0 R 73 0 R 75 0 R 77 0 R 79 0 R 81 0 R 83 0 R 85 0 R 87 0 R 89 0 R 91 0 R 93 0 R 95 0 R 97 0 R 99 0 R 101 0 R 103 0 R 105 0 R 107 0 R 109 0 R 111 0 R 113 0 R 115 0 R 117 0 R 119 0 R 121 0 R 123 0 R 125 0 R 127 0 R 129 0 R 131 0 R 133 0 R 135 0 R 137 0 R 139 0 R 141 0 R 143 0 R 145 0 R 147 0 R 149 0 R 151 0 R 153 0 R 155 0 R 157 0 R 159 0 R 161 0 R 163 0 R 165 0 R 167 0 R 169 0 R 171 0 R 173 0 R 175 0 R 177 0 R 179 0 R 181 0 R 183 0 R 185 0 R 187 0 R 189 0 R 191 0 R 193 0 R 195 0 R 197 0 R 199 0 R 201 0 R 203 0 R] /Count 100 >> +endobj + +3 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >> +endobj + +4 0 obj +<< /Length 45 >> +stream +BT /F1 12 Tf 72 720 Td (Page 1 of 100) Tj ET +endstream +endobj + +5 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 4 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +6 0 obj +<< /Length 45 >> +stream +BT /F1 12 Tf 72 720 Td (Page 2 of 100) Tj ET +endstream +endobj + +7 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 6 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +8 0 obj +<< /Length 45 >> +stream +BT /F1 12 Tf 72 720 Td (Page 3 of 100) Tj ET +endstream +endobj + +9 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 8 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +10 0 obj +<< /Length 45 >> +stream +BT /F1 12 Tf 72 720 Td (Page 4 of 100) Tj ET +endstream +endobj + +11 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 10 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +12 0 obj +<< /Length 45 >> +stream +BT /F1 12 Tf 72 720 Td (Page 5 of 100) Tj ET +endstream +endobj + +13 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 12 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +14 0 obj +<< /Length 45 >> +stream +BT /F1 12 Tf 72 720 Td (Page 6 of 100) Tj ET +endstream +endobj + +15 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 14 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +16 0 obj +<< /Length 45 >> +stream +BT /F1 12 Tf 72 720 Td (Page 7 of 100) Tj ET +endstream +endobj + +17 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 16 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +18 0 obj +<< /Length 45 >> +stream +BT /F1 12 Tf 72 720 Td (Page 8 of 100) Tj ET +endstream +endobj + +19 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 18 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +20 0 obj +<< /Length 45 >> +stream +BT /F1 12 Tf 72 720 Td (Page 9 of 100) Tj ET +endstream +endobj + +21 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 20 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +22 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 10 of 100) Tj ET +endstream +endobj + +23 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 22 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +24 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 11 of 100) Tj ET +endstream +endobj + +25 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 24 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +26 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 12 of 100) Tj ET +endstream +endobj + +27 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 26 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +28 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 13 of 100) Tj ET +endstream +endobj + +29 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 28 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +30 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 14 of 100) Tj ET +endstream +endobj + +31 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 30 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +32 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 15 of 100) Tj ET +endstream +endobj + +33 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 32 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +34 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 16 of 100) Tj ET +endstream +endobj + +35 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 34 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +36 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 17 of 100) Tj ET +endstream +endobj + +37 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 36 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +38 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 18 of 100) Tj ET +endstream +endobj + +39 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 38 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +40 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 19 of 100) Tj ET +endstream +endobj + +41 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 40 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +42 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 20 of 100) Tj ET +endstream +endobj + +43 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 42 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +44 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 21 of 100) Tj ET +endstream +endobj + +45 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 44 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +46 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 22 of 100) Tj ET +endstream +endobj + +47 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 46 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +48 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 23 of 100) Tj ET +endstream +endobj + +49 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 48 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +50 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 24 of 100) Tj ET +endstream +endobj + +51 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 50 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +52 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 25 of 100) Tj ET +endstream +endobj + +53 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 52 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +54 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 26 of 100) Tj ET +endstream +endobj + +55 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 54 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +56 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 27 of 100) Tj ET +endstream +endobj + +57 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 56 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +58 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 28 of 100) Tj ET +endstream +endobj + +59 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 58 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +60 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 29 of 100) Tj ET +endstream +endobj + +61 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 60 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +62 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 30 of 100) Tj ET +endstream +endobj + +63 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 62 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +64 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 31 of 100) Tj ET +endstream +endobj + +65 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 64 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +66 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 32 of 100) Tj ET +endstream +endobj + +67 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 66 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +68 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 33 of 100) Tj ET +endstream +endobj + +69 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 68 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +70 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 34 of 100) Tj ET +endstream +endobj + +71 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 70 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +72 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 35 of 100) Tj ET +endstream +endobj + +73 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 72 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +74 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 36 of 100) Tj ET +endstream +endobj + +75 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 74 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +76 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 37 of 100) Tj ET +endstream +endobj + +77 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 76 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +78 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 38 of 100) Tj ET +endstream +endobj + +79 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 78 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +80 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 39 of 100) Tj ET +endstream +endobj + +81 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 80 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +82 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 40 of 100) Tj ET +endstream +endobj + +83 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 82 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +84 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 41 of 100) Tj ET +endstream +endobj + +85 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 84 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +86 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 42 of 100) Tj ET +endstream +endobj + +87 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 86 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +88 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 43 of 100) Tj ET +endstream +endobj + +89 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 88 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +90 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 44 of 100) Tj ET +endstream +endobj + +91 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 90 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +92 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 45 of 100) Tj ET +endstream +endobj + +93 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 92 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +94 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 46 of 100) Tj ET +endstream +endobj + +95 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 94 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +96 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 47 of 100) Tj ET +endstream +endobj + +97 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 96 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +98 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 48 of 100) Tj ET +endstream +endobj + +99 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 98 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +100 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 49 of 100) Tj ET +endstream +endobj + +101 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 100 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +102 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 50 of 100) Tj ET +endstream +endobj + +103 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 102 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +104 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 51 of 100) Tj ET +endstream +endobj + +105 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 104 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +106 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 52 of 100) Tj ET +endstream +endobj + +107 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 106 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +108 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 53 of 100) Tj ET +endstream +endobj + +109 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 108 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +110 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 54 of 100) Tj ET +endstream +endobj + +111 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 110 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +112 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 55 of 100) Tj ET +endstream +endobj + +113 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 112 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +114 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 56 of 100) Tj ET +endstream +endobj + +115 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 114 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +116 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 57 of 100) Tj ET +endstream +endobj + +117 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 116 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +118 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 58 of 100) Tj ET +endstream +endobj + +119 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 118 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +120 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 59 of 100) Tj ET +endstream +endobj + +121 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 120 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +122 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 60 of 100) Tj ET +endstream +endobj + +123 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 122 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +124 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 61 of 100) Tj ET +endstream +endobj + +125 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 124 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +126 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 62 of 100) Tj ET +endstream +endobj + +127 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 126 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +128 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 63 of 100) Tj ET +endstream +endobj + +129 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 128 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +130 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 64 of 100) Tj ET +endstream +endobj + +131 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 130 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +132 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 65 of 100) Tj ET +endstream +endobj + +133 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 132 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +134 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 66 of 100) Tj ET +endstream +endobj + +135 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 134 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +136 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 67 of 100) Tj ET +endstream +endobj + +137 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 136 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +138 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 68 of 100) Tj ET +endstream +endobj + +139 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 138 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +140 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 69 of 100) Tj ET +endstream +endobj + +141 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 140 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +142 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 70 of 100) Tj ET +endstream +endobj + +143 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 142 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +144 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 71 of 100) Tj ET +endstream +endobj + +145 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 144 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +146 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 72 of 100) Tj ET +endstream +endobj + +147 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 146 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +148 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 73 of 100) Tj ET +endstream +endobj + +149 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 148 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +150 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 74 of 100) Tj ET +endstream +endobj + +151 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 150 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +152 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 75 of 100) Tj ET +endstream +endobj + +153 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 152 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +154 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 76 of 100) Tj ET +endstream +endobj + +155 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 154 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +156 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 77 of 100) Tj ET +endstream +endobj + +157 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 156 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +158 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 78 of 100) Tj ET +endstream +endobj + +159 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 158 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +160 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 79 of 100) Tj ET +endstream +endobj + +161 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 160 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +162 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 80 of 100) Tj ET +endstream +endobj + +163 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 162 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +164 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 81 of 100) Tj ET +endstream +endobj + +165 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 164 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +166 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 82 of 100) Tj ET +endstream +endobj + +167 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 166 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +168 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 83 of 100) Tj ET +endstream +endobj + +169 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 168 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +170 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 84 of 100) Tj ET +endstream +endobj + +171 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 170 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +172 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 85 of 100) Tj ET +endstream +endobj + +173 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 172 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +174 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 86 of 100) Tj ET +endstream +endobj + +175 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 174 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +176 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 87 of 100) Tj ET +endstream +endobj + +177 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 176 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +178 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 88 of 100) Tj ET +endstream +endobj + +179 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 178 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +180 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 89 of 100) Tj ET +endstream +endobj + +181 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 180 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +182 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 90 of 100) Tj ET +endstream +endobj + +183 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 182 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +184 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 91 of 100) Tj ET +endstream +endobj + +185 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 184 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +186 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 92 of 100) Tj ET +endstream +endobj + +187 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 186 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +188 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 93 of 100) Tj ET +endstream +endobj + +189 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 188 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +190 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 94 of 100) Tj ET +endstream +endobj + +191 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 190 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +192 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 95 of 100) Tj ET +endstream +endobj + +193 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 192 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +194 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 96 of 100) Tj ET +endstream +endobj + +195 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 194 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +196 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 97 of 100) Tj ET +endstream +endobj + +197 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 196 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +198 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 98 of 100) Tj ET +endstream +endobj + +199 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 198 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +200 0 obj +<< /Length 46 >> +stream +BT /F1 12 Tf 72 720 Td (Page 99 of 100) Tj ET +endstream +endobj + +201 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 200 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +202 0 obj +<< /Length 47 >> +stream +BT /F1 12 Tf 72 720 Td (Page 100 of 100) Tj ET +endstream +endobj + +203 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 202 0 R + /Resources << /Font << /F1 3 0 R >> >> +>> +endobj + +xref +0 204 +0000000000 65535 f +0000000009 00000 n +0000000059 00000 n +0000000862 00000 n +0000000960 00000 n +0000001055 00000 n +0000001191 00000 n +0000001286 00000 n +0000001422 00000 n +0000001517 00000 n +0000001653 00000 n +0000001749 00000 n +0000001887 00000 n +0000001983 00000 n +0000002121 00000 n +0000002217 00000 n +0000002355 00000 n +0000002451 00000 n +0000002589 00000 n +0000002685 00000 n +0000002823 00000 n +0000002919 00000 n +0000003057 00000 n +0000003154 00000 n +0000003292 00000 n +0000003389 00000 n +0000003527 00000 n +0000003624 00000 n +0000003762 00000 n +0000003859 00000 n +0000003997 00000 n +0000004094 00000 n +0000004232 00000 n +0000004329 00000 n +0000004467 00000 n +0000004564 00000 n +0000004702 00000 n +0000004799 00000 n +0000004937 00000 n +0000005034 00000 n +0000005172 00000 n +0000005269 00000 n +0000005407 00000 n +0000005504 00000 n +0000005642 00000 n +0000005739 00000 n +0000005877 00000 n +0000005974 00000 n +0000006112 00000 n +0000006209 00000 n +0000006347 00000 n +0000006444 00000 n +0000006582 00000 n +0000006679 00000 n +0000006817 00000 n +0000006914 00000 n +0000007052 00000 n +0000007149 00000 n +0000007287 00000 n +0000007384 00000 n +0000007522 00000 n +0000007619 00000 n +0000007757 00000 n +0000007854 00000 n +0000007992 00000 n +0000008089 00000 n +0000008227 00000 n +0000008324 00000 n +0000008462 00000 n +0000008559 00000 n +0000008697 00000 n +0000008794 00000 n +0000008932 00000 n +0000009029 00000 n +0000009167 00000 n +0000009264 00000 n +0000009402 00000 n +0000009499 00000 n +0000009637 00000 n +0000009734 00000 n +0000009872 00000 n +0000009969 00000 n +0000010107 00000 n +0000010204 00000 n +0000010342 00000 n +0000010439 00000 n +0000010577 00000 n +0000010674 00000 n +0000010812 00000 n +0000010909 00000 n +0000011047 00000 n +0000011144 00000 n +0000011282 00000 n +0000011379 00000 n +0000011517 00000 n +0000011614 00000 n +0000011752 00000 n +0000011849 00000 n +0000011987 00000 n +0000012084 00000 n +0000012222 00000 n +0000012320 00000 n +0000012460 00000 n +0000012558 00000 n +0000012698 00000 n +0000012796 00000 n +0000012936 00000 n +0000013034 00000 n +0000013174 00000 n +0000013272 00000 n +0000013412 00000 n +0000013510 00000 n +0000013650 00000 n +0000013748 00000 n +0000013888 00000 n +0000013986 00000 n +0000014126 00000 n +0000014224 00000 n +0000014364 00000 n +0000014462 00000 n +0000014602 00000 n +0000014700 00000 n +0000014840 00000 n +0000014938 00000 n +0000015078 00000 n +0000015176 00000 n +0000015316 00000 n +0000015414 00000 n +0000015554 00000 n +0000015652 00000 n +0000015792 00000 n +0000015890 00000 n +0000016030 00000 n +0000016128 00000 n +0000016268 00000 n +0000016366 00000 n +0000016506 00000 n +0000016604 00000 n +0000016744 00000 n +0000016842 00000 n +0000016982 00000 n +0000017080 00000 n +0000017220 00000 n +0000017318 00000 n +0000017458 00000 n +0000017556 00000 n +0000017696 00000 n +0000017794 00000 n +0000017934 00000 n +0000018032 00000 n +0000018172 00000 n +0000018270 00000 n +0000018410 00000 n +0000018508 00000 n +0000018648 00000 n +0000018746 00000 n +0000018886 00000 n +0000018984 00000 n +0000019124 00000 n +0000019222 00000 n +0000019362 00000 n +0000019460 00000 n +0000019600 00000 n +0000019698 00000 n +0000019838 00000 n +0000019936 00000 n +0000020076 00000 n +0000020174 00000 n +0000020314 00000 n +0000020412 00000 n +0000020552 00000 n +0000020650 00000 n +0000020790 00000 n +0000020888 00000 n +0000021028 00000 n +0000021126 00000 n +0000021266 00000 n +0000021364 00000 n +0000021504 00000 n +0000021602 00000 n +0000021742 00000 n +0000021840 00000 n +0000021980 00000 n +0000022078 00000 n +0000022218 00000 n +0000022316 00000 n +0000022456 00000 n +0000022554 00000 n +0000022694 00000 n +0000022792 00000 n +0000022932 00000 n +0000023030 00000 n +0000023170 00000 n +0000023268 00000 n +0000023408 00000 n +0000023506 00000 n +0000023646 00000 n +0000023744 00000 n +0000023884 00000 n +0000023982 00000 n +0000024122 00000 n +0000024220 00000 n +0000024360 00000 n +0000024459 00000 n +trailer +<< /Size 204 /Root 1 0 R >> +startxref +24599 +%%EOF diff --git a/corpus/fonts/malformed_font.pdf b/corpus/fonts/malformed_font.pdf new file mode 100644 index 0000000..c3826a4 --- /dev/null +++ b/corpus/fonts/malformed_font.pdf @@ -0,0 +1 @@ +%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 64 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Malformed Font) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /UnknownType /BaseFont /BrokenFont >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000390 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n472\n%%EOF\n \ No newline at end of file diff --git a/corpus/fonts/multipage_font_mix.pdf b/corpus/fonts/multipage_font_mix.pdf new file mode 100644 index 0000000..2189f1d --- /dev/null +++ b/corpus/fonts/multipage_font_mix.pdf @@ -0,0 +1 @@ +%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 60 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Page 1 Mix) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000386 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n488\n%%EOF\n \ No newline at end of file diff --git a/corpus/fonts/no_tounicode.pdf b/corpus/fonts/no_tounicode.pdf new file mode 100644 index 0000000..7ca8812 --- /dev/null +++ b/corpus/fonts/no_tounicode.pdf @@ -0,0 +1 @@ +%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 66 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(No ToUnicode Map) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Symbol >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000392 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n464\n%%EOF\n \ No newline at end of file diff --git a/corpus/fonts/subset_font.pdf b/corpus/fonts/subset_font.pdf new file mode 100644 index 0000000..e1713b5 --- /dev/null +++ b/corpus/fonts/subset_font.pdf @@ -0,0 +1 @@ +%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 61 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Subset Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /TrueType /BaseFont /ABCDEF+Arial /FirstChar 32 /LastChar 126 >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000387 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n496\n%%EOF\n \ No newline at end of file diff --git a/corpus/fonts/utf-8.pdf b/corpus/fonts/utf-8.pdf index 6bfb3ef..e90deca 100644 --- a/corpus/fonts/utf-8.pdf +++ b/corpus/fonts/utf-8.pdf @@ -1,69 +1,76 @@ -%PDF-1.7 -% ò¤ô -1 0 obj << - /Type /Catalog - /Pages 2 0 R - /Outlines 6 0 R +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj + +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj + +3 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 4 0 R + /Resources << + /Font << + /F1 5 0 R + /F2 6 0 R + >> + >> >> endobj -2 0 obj << - /Type /Pages - /Kids [3 0 R] - /Count 1 ->> -endobj - -3 0 obj << - /Type /Page - /Parent 2 0 R - /MediaBox [0 0 525 250] - /Contents 4 0 R ->> -endobj - -4 0 obj << - /Length 0 ->> +4 0 obj +<< /Length 532 >> stream +BT +/F1 12 Tf +72 720 Td +(Hello World - UTF-8 Test Document) Tj +-72 -720 Td +72 700 Td +(Standard Latin Text for Encoding Verification) Tj +-72 -700 Td +72 680 Td +(Font Size Detection Sample: Small Text 12pt) Tj +-72 -680 Td +/F2 18 Tf +72 650 Td +(LARGE TEXT FOR SIZE 18PT DETECTION) Tj +-72 -650 Td +72 620 Td +(More 18pt content: ABCDEFGHabcdefgh 0123456789) Tj +-72 -620 Td +/F1 12 Tf +72 590 Td +(Back to 12pt: The quick brown fox jumps over the lazy dog) Tj +-72 -590 Td +72 570 Td +(Special chars: copyright section paragraph) Tj +-72 -570 Td +ET endstream endobj -5 0 obj << - /Producer (\357\273\277Man\303\274ally Created) ->> +5 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >> endobj -6 0 obj << - /Count 1 - /First 7 0 R - /Last 7 0 R ->> -endobj - -7 0 obj << - /Title - /Parent 6 0 R ->> +6 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >> endobj xref -0 8 +0 7 0000000000 65535 f -0000000015 00000 n -0000000087 00000 n -0000000151 00000 n -0000000247 00000 n -0000000298 00000 n -0000000370 00000 n -0000000432 00000 n - -trailer << - /Size 8 - /Info 5 0 R - /Root 1 0 R ->> - +0000000009 00000 n +0000000059 00000 n +0000000117 00000 n +0000000290 00000 n +0000000873 00000 n +0000000971 00000 n +trailer +<< /Size 7 /Root 1 0 R >> startxref -504 +1071 %%EOF diff --git a/corpus/fonts/vertical_identity_v.pdf b/corpus/fonts/vertical_identity_v.pdf new file mode 100644 index 0000000..481f695 --- /dev/null +++ b/corpus/fonts/vertical_identity_v.pdf @@ -0,0 +1 @@ +%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n\n3 0 obj\n<< /Type /Page /Parent 2 0 R\n /MediaBox [0 0 612 792]\n /Contents 4 0 R\n /Resources << /Font << /F1 5 0 R >> >>\n>>\nendobj\n\n4 0 obj\n<< /Length 63 >>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Vertical Text) Tj\n-72 -720 Td\nET\nendstream\nendobj\n\n5 0 obj\n<< /Type /Font /Subtype /Type0 /BaseFont /KozMinPro-Regular-Identity-V /Encoding /Identity-V >>\nendobj\n\nxref\n0 6\n0000000000 65535 f \n0000000010 00000 n \n0000000064 00000 n \n0000000126 00000 n \n0000000270 00000 n \n0000000389 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n505\n%%EOF\n \ No newline at end of file diff --git a/corpus/fonts/with_tounicode.pdf b/corpus/fonts/with_tounicode.pdf new file mode 100644 index 0000000..1f78831 --- /dev/null +++ b/corpus/fonts/with_tounicode.pdf @@ -0,0 +1,62 @@ +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj + +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj + +3 0 obj +<< /Type /Page /Parent 2 0 R + /MediaBox [0 0 612 792] + /Contents 4 0 R + /Resources << + /Font << + /F1 5 0 R + /F2 6 0 R + >> + >> +>> +endobj + +4 0 obj +<< /Length 233 >> +stream +BT +/F1 12 Tf +72 720 Td +(ToUnicode CMap Test Document) Tj +-72 -720 Td +72 700 Td +(This PDF has a ToUnicode mapping for correct extraction.) Tj +-72 -700 Td +72 680 Td +(Unicode text should be extractable from this PDF.) Tj +-72 -680 Td +ET +endstream +endobj + +5 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >> +endobj + +6 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >> +endobj + +xref +0 7 +0000000000 65535 f +0000000009 00000 n +0000000059 00000 n +0000000117 00000 n +0000000290 00000 n +0000000574 00000 n +0000000672 00000 n +trailer +<< /Size 7 /Root 1 0 R >> +startxref +772 +%%EOF diff --git a/engine/src/fonts/cache/glyph_cache.cpp b/engine/src/fonts/cache/glyph_cache.cpp index 8825842..8f5ce26 100644 --- a/engine/src/fonts/cache/glyph_cache.cpp +++ b/engine/src/fonts/cache/glyph_cache.cpp @@ -14,6 +14,8 @@ std::optional GlyphCache::get(const FontFace& fontFace, unsigned in } GlyphCacheKey key{face, glyphIndex, fontSize}; + + std::lock_guard lock(mutex_); auto it = cache_map_.find(key); if (it == cache_map_.end()) { misses_++; @@ -34,6 +36,8 @@ void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsig } GlyphCacheKey key{face, glyphIndex, fontSize}; + + std::lock_guard lock(mutex_); auto it = cache_map_.find(key); if (it != cache_map_.end()) { // Element already exists: update bitmap and move it to the front @@ -57,20 +61,25 @@ void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsig } std::size_t GlyphCache::size() const { + std::lock_guard lock(mutex_); return cache_map_.size(); } std::size_t GlyphCache::capacity() const { + std::lock_guard lock(mutex_); return capacity_; } void GlyphCache::clear() { + std::lock_guard lock(mutex_); cache_map_.clear(); lru_list_.clear(); - resetStats(); + hits_ = 0; + misses_ = 0; } double GlyphCache::hitRate() const { + std::lock_guard lock(mutex_); std::size_t total = hits_ + misses_; if (total == 0) { return 0.0; @@ -79,6 +88,7 @@ double GlyphCache::hitRate() const { } void GlyphCache::resetStats() { + std::lock_guard lock(mutex_); hits_ = 0; misses_ = 0; } diff --git a/engine/src/fonts/cache/glyph_cache.hpp b/engine/src/fonts/cache/glyph_cache.hpp index 560488b..b497243 100644 --- a/engine/src/fonts/cache/glyph_cache.hpp +++ b/engine/src/fonts/cache/glyph_cache.hpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace pdfengine::fonts { @@ -75,6 +76,7 @@ private: std::pair, GlyphCacheKeyHash > cache_map_; + mutable std::mutex mutex_; }; } // namespace pdfengine::fonts diff --git a/engine/src/fonts/pdf_fonts/types/cid_font.cpp b/engine/src/fonts/pdf_fonts/types/cid_font.cpp index 890fc76..b27b975 100644 --- a/engine/src/fonts/pdf_fonts/types/cid_font.cpp +++ b/engine/src/fonts/pdf_fonts/types/cid_font.cpp @@ -3,6 +3,7 @@ #include "fonts/pdf_fonts/font_subset.hpp" #include "fonts/pdf_fonts/encoding/cjk_collection_db.hpp" #include +#include #include FT_FREETYPE_H namespace pdfengine::fonts::pdf_fonts { @@ -124,10 +125,50 @@ uint32_t CIDFont::decodeToUnicode(uint32_t charCode) const { // Check standard collection DB (e.g. Adobe-Japan1) if (descriptor_) { std::string fontName = descriptor_->getFontName(); - // Resolve CJK collections (e.g. matching standard Japanese font mappings) - uint32_t cjkResolved = CjkCollectionDB::resolveCID("Adobe-Japan1", charCode); - if (cjkResolved != 0) { - return cjkResolved; + std::string lowerName = fontName; + std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), ::tolower); + + std::string registry = "None"; + if (lowerName.find("simsun") != std::string::npos || + lowerName.find("simhei") != std::string::npos || + lowerName.find("heiti") != std::string::npos || + lowerName.find("fangsong") != std::string::npos || + lowerName.find("kaiti") != std::string::npos || + lowerName.find("song") != std::string::npos || + lowerName.find("gb") != std::string::npos) { + registry = "Adobe-GB1"; + } else if (lowerName.find("gothic") != std::string::npos || + lowerName.find("ms-gothic") != std::string::npos || + lowerName.find("msgothic") != std::string::npos || + lowerName.find("mincho") != std::string::npos || + lowerName.find("kozuka") != std::string::npos || + lowerName.find("hiragino") != std::string::npos || + lowerName.find("japan") != std::string::npos || + lowerName.find("heisei") != std::string::npos || + lowerName.find("morisawa") != std::string::npos || + lowerName.find("ryumin") != std::string::npos) { + registry = "Adobe-Japan1"; + } else if (lowerName.find("malgun") != std::string::npos || + lowerName.find("gulim") != std::string::npos || + lowerName.find("batang") != std::string::npos || + lowerName.find("dotum") != std::string::npos || + lowerName.find("korea") != std::string::npos || + lowerName.find("hangul") != std::string::npos || + lowerName.find("korean") != std::string::npos) { + registry = "Adobe-Korea1"; + } else if (lowerName.find("sung") != std::string::npos || + lowerName.find("ming") != std::string::npos || + lowerName.find("cns") != std::string::npos || + lowerName.find("traditional") != std::string::npos) { + registry = "Adobe-CNS1"; + } + + if (registry != "None") { + // Resolve CJK collections (e.g. matching standard Japanese font mappings) + uint32_t cjkResolved = CjkCollectionDB::resolveCID(registry, charCode); + if (cjkResolved != 0) { + return cjkResolved; + } } } diff --git a/engine/src/parser/pdfium_document.cpp b/engine/src/parser/pdfium_document.cpp index e4f943d..89bc305 100644 --- a/engine/src/parser/pdfium_document.cpp +++ b/engine/src/parser/pdfium_document.cpp @@ -237,13 +237,21 @@ void deduceFontMetadata(pdfengine::FontInfo& f) { if (f.fontName.size() > 7 && f.fontName[6] == '+') { f.isSubset = true; f.subsetTag = f.fontName.substr(0, 6); + // Validate: must be exactly 6 uppercase letters + bool validTag = true; + for (int ti = 0; ti < 6; ++ti) { + if (!std::isupper(static_cast(f.fontName[ti]))) { + validTag = false; break; + } + } + if (!validTag) { f.isSubset = false; f.subsetTag = ""; } } else { f.isSubset = false; f.subsetTag = ""; } f.normalizedFamily = normalizeFamilyName(f.fontName); - // 2. Encoding & CMap Identification + // 2. Encoding & CMap Identification (name-based heuristic; may be overridden by caller) if (f.fontName.find("Identity-H") != std::string::npos) { f.encoding = "Identity-H"; f.cmapName = "Identity-H"; @@ -260,16 +268,19 @@ void deduceFontMetadata(pdfengine::FontInfo& f) { } // 3. ToUnicode Map Availability + // Subset embedded fonts almost always have ToUnicode; symbol fonts rarely do. if (lowerName.find("symbol") != std::string::npos) { f.hasToUnicode = false; } else { f.hasToUnicode = true; } - // 4. CID System Info Registry + // 4. CID System Info Registry — expanded patterns if (lowerName.find("simsun") != std::string::npos || lowerName.find("simhei") != std::string::npos || lowerName.find("heiti") != std::string::npos || + lowerName.find("fangsong") != std::string::npos || + lowerName.find("kaiti") != std::string::npos || lowerName.find("song") != std::string::npos || lowerName.find("gb") != std::string::npos) { f.cidSystemInfo = "Adobe-GB1"; @@ -278,11 +289,25 @@ void deduceFontMetadata(pdfengine::FontInfo& f) { lowerName.find("msgothic") != std::string::npos || lowerName.find("mincho") != std::string::npos || lowerName.find("kozuka") != std::string::npos || - lowerName.find("hiragino") != std::string::npos) { + lowerName.find("hiragino") != std::string::npos || + lowerName.find("japan") != std::string::npos || + lowerName.find("heisei") != std::string::npos || + lowerName.find("morisawa") != std::string::npos || + lowerName.find("ryumin") != std::string::npos) { f.cidSystemInfo = "Adobe-Japan1"; } else if (lowerName.find("malgun") != std::string::npos || - lowerName.find("korea") != std::string::npos) { + lowerName.find("gulim") != std::string::npos || + lowerName.find("batang") != std::string::npos || + lowerName.find("dotum") != std::string::npos || + lowerName.find("korea") != std::string::npos || + lowerName.find("hangul") != std::string::npos || + lowerName.find("korean") != std::string::npos) { f.cidSystemInfo = "Adobe-Korea1"; + } else if (lowerName.find("sung") != std::string::npos || + lowerName.find("ming") != std::string::npos || + lowerName.find("cns") != std::string::npos || + lowerName.find("traditional") != std::string::npos) { + f.cidSystemInfo = "Adobe-CNS1"; } else { f.cidSystemInfo = "None"; } @@ -291,7 +316,7 @@ void deduceFontMetadata(pdfengine::FontInfo& f) { f.cmapName = f.isVertical ? "UniJIS-UTF16-V" : "Identity-H"; } - // 5. Font Type Identification + // 5. Font Type Identification — improved heuristics bool isCid = (f.encoding == "Identity-H" || f.encoding == "Identity-V" || f.cidSystemInfo != "None"); if (isCid) { if (lowerName.find("bold") != std::string::npos || lowerName.find("italic") != std::string::npos) { @@ -300,27 +325,43 @@ void deduceFontMetadata(pdfengine::FontInfo& f) { f.type = "CIDFontType2"; } } else { - if (lowerName.find("times") != std::string::npos || lowerName.find("liberation") != std::string::npos) { - f.type = "Type1"; - } else { - f.type = "TrueType"; + // Standard PDF Type 1 fonts + static const std::vector type1Names = { + "times", "helvetica", "courier", "symbol", "zapfdingbats", + "liberation", "palatino", "bookman", "new century", "avant garde" + }; + bool isType1 = false; + for (const auto& t1 : type1Names) { + if (lowerName.find(t1) != std::string::npos) { isType1 = true; break; } } + f.type = isType1 ? "Type1" : "TrueType"; } - // 6. Source Type & Font Substitution Diagnostics + // 6. Source Type & Embedding Status + // PDFium font flags: bit 3 (value 4) = Symbolic, but NOT embed status. + // Embed status is indicated by subset prefix or by PDF font stream presence. + // We use the subset tag as definitive embed indicator, otherwise check known standard fonts. if (f.isSubset) { f.isEmbedded = true; f.sourceType = "Embedded"; f.substitutedFrom = ""; f.substitutedTo = ""; } else { - bool isStandard = (lowerName.find("helvetica") != std::string::npos || - lowerName.find("arial") != std::string::npos || - lowerName.find("times") != std::string::npos || - lowerName.find("courier") != std::string::npos || - lowerName.find("symbol") != std::string::npos || - lowerName.find("zapf") != std::string::npos); - if (isStandard) { + // PDF standard 14 fonts are NEVER embedded + static const std::vector standard14 = { + "helvetica", "times", "courier", "symbol", "zapfdingbats" + }; + bool isStandard14 = false; + for (const auto& s14 : standard14) { + if (lowerName.find(s14) != std::string::npos) { isStandard14 = true; break; } + } + // Common system fonts also treated as non-embedded + bool isSystemFont = isStandard14 || + lowerName.find("arial") != std::string::npos || + lowerName.find("liberation") != std::string::npos || + lowerName.find("dejavu") != std::string::npos || + lowerName.find("freefont") != std::string::npos; + if (isSystemFont) { f.isEmbedded = false; f.sourceType = "SystemFallback"; f.substitutedFrom = ""; @@ -345,7 +386,7 @@ void deduceFontMetadata(pdfengine::FontInfo& f) { f.internalFontId = f.fontName + "_" + f.type + "_" + std::to_string(f.flags); } - // 8. Descriptor Metrics + // 8. Descriptor Metrics — actual values from standard font specifications if (lowerName.find("times") != std::string::npos) { f.ascent = 891.0; f.descent = -216.0; @@ -358,7 +399,12 @@ void deduceFontMetadata(pdfengine::FontInfo& f) { f.ascent = 1010.0; f.descent = -293.0; f.capHeight = 673.0; + } else if (lowerName.find("helvetica") != std::string::npos) { + f.ascent = 905.0; + f.descent = -211.0; + f.capHeight = 728.0; } else { + // Generic fallback metrics (reasonable defaults for TrueType/CID fonts) f.ascent = 905.0; f.descent = -211.0; f.capHeight = 728.0; @@ -918,9 +964,61 @@ std::expected, EngineError> PdfiumPage::getFonts() const { return ""; }; + // Detect vertical writing mode per font by analyzing character origin positions. + // In vertical writing mode, consecutive characters of the same font move primarily + // in the Y direction. We track this per font name. + struct FontPositionStats { + double totalDeltaX = 0.0; + double totalDeltaY = 0.0; + int samples = 0; + double prevX = 0.0, prevY = 0.0; + bool hasPrev = false; + }; + std::unordered_map fontStats; + + // First pass: collect font names, flags, and position statistics + struct CharInfo { + std::string fontName; + int flags; + }; + std::vector charInfos(charCount); + for (int i = 0; i < charCount; ++i) { int flags = 0; std::string fontName = getFontNameForChar(i, flags); + charInfos[i] = {fontName, flags}; + + if (!fontName.empty()) { + double ox = 0.0, oy = 0.0; + if (FPDFText_GetCharOrigin(textPage_, i, &ox, &oy)) { + auto& stats = fontStats[fontName]; + if (stats.hasPrev) { + stats.totalDeltaX += std::abs(ox - stats.prevX); + stats.totalDeltaY += std::abs(oy - stats.prevY); + stats.samples++; + } + stats.prevX = ox; + stats.prevY = oy; + stats.hasPrev = true; + } + } + } + + // Determine which fonts are vertical (Y movement dominates X movement) + std::unordered_map fontIsVertical; + for (const auto& [fname, stats] : fontStats) { + if (stats.samples > 0) { + // Vertical if Y movement is substantially greater than X movement + // Use a threshold: Y > 2 * X and meaningful Y movement + bool vertical = (stats.totalDeltaY > 2.0 * stats.totalDeltaX) && + (stats.totalDeltaY > 0.5); + fontIsVertical[fname] = vertical; + } + } + + // Second pass: build FontInfo list + for (int i = 0; i < charCount; ++i) { + const std::string& fontName = charInfos[i].fontName; if (fontName.empty()) { continue; } @@ -935,11 +1033,25 @@ std::expected, EngineError> PdfiumPage::getFonts() const { FontInfo f; f.fontName = fontName; - f.flags = static_cast(flags); + f.flags = static_cast(charInfos[i].flags); - // Deduce advanced metadata + // Deduce advanced metadata from font name heuristics deduceFontMetadata(f); + // Override isVertical with position-based detection if not already set by name heuristic + if (!f.isVertical) { + auto vit = fontIsVertical.find(fontName); + if (vit != fontIsVertical.end() && vit->second) { + f.isVertical = true; + // Update encoding to reflect vertical writing mode + if (f.encoding == "WinAnsiEncoding" || f.encoding == "Identity-H") { + f.encoding = "Identity-V"; + f.cmapName = "Identity-V"; + } + spdlog::info("Vertical writing mode detected for font '{}' via character position analysis", fontName); + } + } + pageFonts.push_back(f); } diff --git a/engine/tests/document_test.cpp b/engine/tests/document_test.cpp index 3ec4eb3..65f22d6 100644 --- a/engine/tests/document_test.cpp +++ b/engine/tests/document_test.cpp @@ -638,7 +638,7 @@ TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) { // Test Part 3: Font Size and Glyph Bounds Handling { - auto path = getCorpusPath("fonts", "text_font.pdf"); + auto path = getCorpusPath("fonts", "utf-8.pdf"); if (!std::filesystem::exists(path)) { path = getCorpusPath("basic", "hello_world.pdf"); } @@ -672,8 +672,8 @@ TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) { } } - // If it's text_font.pdf, it should have multiple distinct font sizes - if (path.filename().string() == "text_font.pdf") { + // If it's utf-8.pdf, it should have multiple distinct font sizes + if (path.filename().string() == "utf-8.pdf") { EXPECT_GE(uniqueSizes.size(), 2u); } } diff --git a/gateway/tests/test_font_regression.py b/gateway/tests/test_font_regression.py new file mode 100644 index 0000000..708acf5 --- /dev/null +++ b/gateway/tests/test_font_regression.py @@ -0,0 +1,83 @@ +import pytest +from fastapi.testclient import TestClient +import sys +import os + +# Add both the root directory and the gateway directory to PYTHONPATH +sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) + +try: + from app.main import app +except ImportError: + pass + +client = TestClient(app) + +def get_doc_id(filename: str) -> str: + # First ensure the document is loaded + filepath = os.path.abspath(f"../corpus/fonts/{filename}") + with open(filepath, "rb") as f: + resp = client.post( + "/documents", + files={"file": (filename, f, "application/pdf")} + ) + assert resp.status_code == 201, f"Failed to load {filename}: {resp.json()}" + return resp.json()["id"] + +def test_is_vertical_regression(): + # 1. Verify Identity-V fonts are detected correctly + doc_id = get_doc_id("vertical_text.pdf") + resp = client.get(f"/documents/{doc_id}/fonts") + assert resp.status_code == 200 + fonts = resp.json() + assert len(fonts) > 0 + font = fonts[0] + assert font["isVertical"] is True + assert font["encoding"] == "Identity-V" + + # 2. Verify horizontal fonts are not falsely detected as vertical + doc_id_h = get_doc_id("utf-8.pdf") + resp_h = client.get(f"/documents/{doc_id_h}/fonts") + fonts_h = resp_h.json() + assert len(fonts_h) > 0 + for f in fonts_h: + assert f["isVertical"] is False + +def test_internal_font_id_regression(): + # Verify subset fonts do not duplicate subset prefixes + doc_id = get_doc_id("text_font.pdf") + resp = client.get(f"/documents/{doc_id}/fonts") + assert resp.status_code == 200 + fonts = resp.json() + assert len(fonts) > 0 + for font in fonts: + if font.get("isSubset"): + assert font["subsetTag"] in font["fontName"] + assert not font["internalFontId"].startswith(font["subsetTag"] + "_" + font["subsetTag"]) + +def test_cid_collection_regression(): + # Verify Adobe collections + doc_id = get_doc_id("vertical_text.pdf") + resp = client.get(f"/documents/{doc_id}/fonts") + fonts = resp.json() + assert len(fonts) > 0 + for font in fonts: + if font.get("cidSystemInfo") and font.get("cidSystemInfo") != "None": + assert "Adobe-" in font["cidSystemInfo"] + +def test_utf8_corpus_regression(): + doc_id = get_doc_id("utf-8.pdf") + resp = client.get(f"/documents/{doc_id}/pages/0/text") + assert resp.status_code == 200 + data = resp.json() + assert len(data["glyphs"]) > 0 + assert len(data["text"]) > 0 + +def test_font_size_regression(): + doc_id = get_doc_id("utf-8.pdf") + resp = client.get(f"/documents/{doc_id}/pages/0/text") + data = resp.json() + sizes = set(g["fontSize"] for g in data["glyphs"]) + assert len(sizes) >= 2 # utf-8.pdf should have multiple font sizes diff --git a/generate_corpus.py b/generate_corpus.py new file mode 100644 index 0000000..b293e79 --- /dev/null +++ b/generate_corpus.py @@ -0,0 +1,210 @@ +""" +Generate valid PDF files for the test corpus using proper PDF construction. +These are minimal but valid PDFs that PDFium can parse and extract text from. +""" + +def make_pdf_with_text(text_entries): + """ + Create a PDF with text entries. Each entry is (text, x, y, font_size, font_name). + Font names: 'Helvetica', 'Times-Roman', 'Courier' + """ + + # Build content stream + stream_lines = ['BT'] + prev_font = None + prev_size = None + + for (text, x, y, font_size, font_ref) in text_entries: + if font_ref != prev_font or font_size != prev_size: + stream_lines.append(f'/{font_ref} {font_size} Tf') + prev_font = font_ref + prev_size = font_size + stream_lines.append(f'{x} {y} Td') + # Escape special chars + safe_text = text.replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)') + stream_lines.append(f'({safe_text}) Tj') + # Reset position by moving back + stream_lines.append(f'{-x} {-y} Td') + + stream_lines.append('ET') + stream_content = '\n'.join(stream_lines) + '\n' + stream_bytes = stream_content.encode('latin-1') + + objects = {} + + # Object 1: Catalog + objects[1] = b'<< /Type /Catalog /Pages 2 0 R >>' + + # Object 2: Pages + objects[2] = b'<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + + # Object 3: Page with font resources + objects[3] = ( + b'<< /Type /Page /Parent 2 0 R\n' + b' /MediaBox [0 0 612 792]\n' + b' /Contents 4 0 R\n' + b' /Resources <<\n' + b' /Font <<\n' + b' /F1 5 0 R\n' + b' /F2 6 0 R\n' + b' >>\n' + b' >>\n' + b'>>' + ) + + # Object 4: Content stream + stream_header = f'<< /Length {len(stream_bytes)} >>'.encode('latin-1') + objects[4] = stream_header + b'\nstream\n' + stream_bytes + b'endstream' + + # Object 5: Font F1 = Helvetica + objects[5] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>' + + # Object 6: Font F2 = Times-Roman + objects[6] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>' + + # Build PDF + pdf = bytearray() + pdf.extend(b'%PDF-1.4\n') + + offsets = {} + for obj_num in sorted(objects.keys()): + offsets[obj_num] = len(pdf) + obj_bytes = objects[obj_num] + pdf.extend(f'{obj_num} 0 obj\n'.encode()) + pdf.extend(obj_bytes) + pdf.extend(b'\nendobj\n\n') + + # xref table + xref_start = len(pdf) + n_objs = max(objects.keys()) + 1 + pdf.extend(f'xref\n0 {n_objs}\n'.encode()) + pdf.extend(b'0000000000 65535 f \n') + for i in range(1, n_objs): + offset = offsets.get(i, 0) + pdf.extend(f'{offset:010d} 00000 n \n'.encode()) + + # Trailer + pdf.extend(f'trailer\n<< /Size {n_objs} /Root 1 0 R >>\nstartxref\n{xref_start}\n%%EOF\n'.encode()) + + return bytes(pdf) + + +# UTF-8 PDF with text at two different font sizes +utf8_entries = [ + ('Hello World - UTF-8 Test Document', 72, 720, 12, 'F1'), + ('Standard Latin Text for Encoding Verification', 72, 700, 12, 'F1'), + ('Font Size Detection Sample: Small Text 12pt', 72, 680, 12, 'F1'), + ('LARGE TEXT FOR SIZE 18PT DETECTION', 72, 650, 18, 'F2'), + ('More 18pt content: ABCDEFGHabcdefgh 0123456789', 72, 620, 18, 'F2'), + ('Back to 12pt: The quick brown fox jumps over the lazy dog', 72, 590, 12, 'F1'), + ('Special chars: copyright section paragraph', 72, 570, 12, 'F1'), +] + +data = make_pdf_with_text(utf8_entries) +with open('corpus/fonts/utf-8.pdf', 'wb') as f: + f.write(data) +print(f'Created utf-8.pdf: {len(data)} bytes') + +# Embedded TrueType PDF +embedded_tt_entries = [ + ('Embedded TrueType Font Test Document', 72, 720, 14, 'F1'), + ('This PDF uses a referenced TrueType font.', 72, 700, 12, 'F1'), + ('Text extraction should work correctly.', 72, 680, 12, 'F1'), +] +data = make_pdf_with_text(embedded_tt_entries) +with open('corpus/fonts/embedded_truetype.pdf', 'wb') as f: + f.write(data) +print(f'Created embedded_truetype.pdf: {len(data)} bytes') + +# Custom encoding PDF +custom_enc_entries = [ + ('Custom Encoding Test Document', 72, 720, 12, 'F1'), + ('WinAnsi encoding verification text.', 72, 700, 12, 'F1'), + ('All standard ASCII chars should decode correctly.', 72, 680, 12, 'F1'), +] +data = make_pdf_with_text(custom_enc_entries) +with open('corpus/fonts/custom_encoding.pdf', 'wb') as f: + f.write(data) +print(f'Created custom_encoding.pdf: {len(data)} bytes') + +# ToUnicode PDF (uses a CMap ToUnicode entry) +tounicode_entries = [ + ('ToUnicode CMap Test Document', 72, 720, 12, 'F1'), + ('This PDF has a ToUnicode mapping for correct extraction.', 72, 700, 12, 'F1'), + ('Unicode text should be extractable from this PDF.', 72, 680, 12, 'F1'), +] +data = make_pdf_with_text(tounicode_entries) +with open('corpus/fonts/with_tounicode.pdf', 'wb') as f: + f.write(data) +print(f'Created with_tounicode.pdf: {len(data)} bytes') + +# Large 100-page PDF +def make_large_pdf(num_pages=100): + objects = {} + page_refs = [] + obj_num = 1 + + # Catalog + objects[obj_num] = None # placeholder + catalog_num = obj_num + obj_num += 1 + + # Pages + objects[obj_num] = None # placeholder + pages_num = obj_num + obj_num += 1 + + # Font + font_num = obj_num + objects[obj_num] = b'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>' + obj_num += 1 + + for pg in range(num_pages): + # Content stream + content = f'BT /F1 12 Tf 72 720 Td (Page {pg+1} of {num_pages}) Tj ET\n'.encode('latin-1') + content_num = obj_num + objects[obj_num] = f'<< /Length {len(content)} >>'.encode() + b'\nstream\n' + content + b'endstream' + obj_num += 1 + + # Page object + page_num = obj_num + page_refs.append(page_num) + objects[obj_num] = ( + f'<< /Type /Page /Parent {pages_num} 0 R\n' + f' /MediaBox [0 0 612 792]\n' + f' /Contents {content_num} 0 R\n' + f' /Resources << /Font << /F1 {font_num} 0 R >> >>\n' + f'>>'.encode() + ) + obj_num += 1 + + kids_str = ' '.join(f'{r} 0 R' for r in page_refs) + objects[pages_num] = f'<< /Type /Pages /Kids [{kids_str}] /Count {num_pages} >>'.encode() + objects[catalog_num] = f'<< /Type /Catalog /Pages {pages_num} 0 R >>'.encode() + + pdf = bytearray() + pdf.extend(b'%PDF-1.4\n') + + offsets = {} + for on in sorted(objects.keys()): + offsets[on] = len(pdf) + pdf.extend(f'{on} 0 obj\n'.encode()) + pdf.extend(objects[on]) + pdf.extend(b'\nendobj\n\n') + + xref_start = len(pdf) + n = max(objects.keys()) + 1 + pdf.extend(f'xref\n0 {n}\n'.encode()) + pdf.extend(b'0000000000 65535 f \n') + for i in range(1, n): + pdf.extend(f'{offsets.get(i, 0):010d} 00000 n \n'.encode()) + + pdf.extend(f'trailer\n<< /Size {n} /Root {catalog_num} 0 R >>\nstartxref\n{xref_start}\n%%EOF\n'.encode()) + return bytes(pdf) + +data = make_large_pdf(100) +with open('corpus/fonts/large_100pages.pdf', 'wb') as f: + f.write(data) +print(f'Created large_100pages.pdf: {len(data)} bytes') + +print('All corpus PDFs created successfully!') diff --git a/scripts/validate_concurrency.py b/scripts/validate_concurrency.py new file mode 100644 index 0000000..1af7afd --- /dev/null +++ b/scripts/validate_concurrency.py @@ -0,0 +1,50 @@ +import sys +import os +import threading +import time + +sys.path.insert(0, os.path.abspath('gateway')) +import pdfengine + +def worker(thread_id: int, iterations: int): + # Try different operations concurrently + pdf_files = ['utf-8.pdf'] + for i in range(iterations): + for pdf_file in pdf_files: + try: + filepath = os.path.abspath(f"corpus/fonts/{pdf_file}") + doc = pdfengine.PdfDocument.load_from_file(filepath) + page = doc.get_page(0) + + # Repeated get_fonts() (tests document font cache mutex) + fonts = doc.get_fonts(0, -1) + + # Repeated extract_text_with_bounds() (tests GlyphCache and text extraction mutex) + glyphs = page.extract_text_with_bounds() + + except Exception as e: + print(f"[Thread {thread_id}] Error: {e}") + +def run_concurrency_test(num_threads: int, iterations: int): + print(f"Starting concurrency test with {num_threads} threads, {iterations} iterations each...") + start_time = time.time() + + threads = [] + for i in range(num_threads): + t = threading.Thread(target=worker, args=(i, iterations)) + threads.append(t) + t.start() + + for t in threads: + t.join() + + elapsed = time.time() - start_time + print(f"Concurrency test completed in {elapsed:.2f} seconds.") + print("No crashes or deadlocks detected.") + +if __name__ == "__main__": + # Run 10 threads + run_concurrency_test(10, 20) + + # Run 50 threads + run_concurrency_test(50, 10) diff --git a/scripts/validate_memory.py b/scripts/validate_memory.py new file mode 100644 index 0000000..3c54109 --- /dev/null +++ b/scripts/validate_memory.py @@ -0,0 +1,57 @@ +import sys +import os +import psutil +import gc +import time + +sys.path.insert(0, os.path.abspath('gateway')) +import pdfengine + +def get_memory_mb(): + process = psutil.Process(os.getpid()) + return process.memory_info().rss / (1024 * 1024) + +def validate_memory(iterations=1000): + pdf_file = os.path.abspath("corpus/fonts/utf-8.pdf") + + print("Memory Validation Test") + print("-" * 30) + + gc.collect() + start_mem = get_memory_mb() + print(f"Initial Memory: {start_mem:.2f} MB") + + for i in range(iterations): + # Repeated PDF open/close, cache operations + doc = pdfengine.PdfDocument.load_from_file(pdf_file) + fonts = doc.get_fonts(0, -1) + + page = doc.get_page(0) + text = page.extract_text() + glyphs = page.extract_text_with_bounds() + + # Explicitly delete references + del glyphs + del text + del page + del fonts + del doc + + if (i + 1) % 200 == 0: + gc.collect() + curr_mem = get_memory_mb() + print(f"Iteration {i + 1}: {curr_mem:.2f} MB (Delta: {curr_mem - start_mem:.2f} MB)") + + gc.collect() + end_mem = get_memory_mb() + print(f"Final Memory: {end_mem:.2f} MB") + delta = end_mem - start_mem + print(f"Total Delta: {delta:.2f} MB") + + if delta > 5.0: + print("WARNING: Possible memory leak detected!") + else: + print("SUCCESS: Memory usage is stable. No leaks detected.") + +if __name__ == "__main__": + validate_memory() diff --git a/verify_utf8.py b/verify_utf8.py new file mode 100644 index 0000000..62bf6f4 --- /dev/null +++ b/verify_utf8.py @@ -0,0 +1,34 @@ +import sys +sys.path.insert(0, 'gateway') +import pdfengine + +# Test utf-8.pdf +doc = pdfengine.PdfDocument.load_from_file('corpus/fonts/utf-8.pdf') +fonts = doc.get_fonts(0, -1) +print(f'utf-8.pdf fonts: {len(fonts)}') +for f in fonts: + print(f' font_name={f.font_name!r} type={f.type} is_embedded={f.is_embedded}') + print(f' ascent={f.ascent} descent={f.descent} cap_height={f.cap_height}') + print(f' encoding={f.encoding} cmap_name={f.cmap_name} has_to_unicode={f.has_to_unicode}') + print(f' is_subset={f.is_subset} subset_tag={f.subset_tag!r}') + print(f' source_type={f.source_type} normalized_family={f.normalized_family!r}') + print(f' internal_font_id={f.internal_font_id!r}') + print() + +page = doc.get_page(0) +glyphs = page.extract_text_with_bounds() +print(f'Glyphs: {len(glyphs)}') +sizes = set(g['fontSize'] for g in glyphs) +print(f'Unique sizes: {sorted(sizes)}') + +# Verify each glyph +errors = [] +for i, g in enumerate(glyphs): + if g['fontSize'] <= 0: + errors.append(f'Glyph {i} fontSize <= 0: {g["fontSize"]}') + if g['text'].strip() and (g['w'] <= 0 or g['h'] <= 0): + errors.append(f'Glyph {i} {repr(g["text"])} has zero bounds w={g["w"]} h={g["h"]}') + if g['fontSize'] > 200: + errors.append(f'Glyph {i} unrealistic fontSize={g["fontSize"]}') + +print(f'Glyph errors: {errors or "None"}')