From 0862afa1ce5f4cc1748330cb985f42d4102165ac Mon Sep 17 00:00:00 2001 From: saqib mir Date: Wed, 3 Jun 2026 10:51:33 +0530 Subject: [PATCH 1/7] update some minor fixes --- bindings/python/pdfengine_py.cpp | 3 + corpus/fonts/cns1_test.pdf | 90 ++++ engine/include/pdfengine/pdf_document.hpp | 2 + engine/src/parser/pdfium_document.cpp | 67 ++- engine/src/parser/pdfium_document.hpp | 6 + engine/tests/document_test.cpp | 2 +- gateway/app/routers/render.py | 23 + gateway/tests/font_api_validation.py | 388 +++++++++++++++ gateway/tests/test_font_regression.py | 179 ++++++- gateway/tests/test_glyph_metrics.py | 93 ++++ gateway/tests/test_routes.py | 6 +- gateway/tests/validation_output.txt | 554 ++++++++++++++++++++++ wasm/bindings/pdf_engine_facade.cpp | 5 + wasm/bindings/pdf_engine_facade.hpp | 1 + 14 files changed, 1405 insertions(+), 14 deletions(-) create mode 100644 corpus/fonts/cns1_test.pdf create mode 100644 gateway/tests/font_api_validation.py create mode 100644 gateway/tests/test_glyph_metrics.py create mode 100644 gateway/tests/validation_output.txt diff --git a/bindings/python/pdfengine_py.cpp b/bindings/python/pdfengine_py.cpp index f88c437..bc95e31 100644 --- a/bindings/python/pdfengine_py.cpp +++ b/bindings/python/pdfengine_py.cpp @@ -141,6 +141,9 @@ PYBIND11_MODULE(pdfengine, m) { .def("get_fonts", [](const pdfengine::PdfPage& self) { return get_or_throw(self.getFonts()); }) + .def("get_glyph_width", [](const pdfengine::PdfPage& self, const std::string& fontName, uint32_t charcode, double fontSize) { + return get_or_throw(self.getGlyphWidth(fontName, charcode, fontSize)); + }, py::arg("font_name"), py::arg("charcode"), py::arg("font_size")) .def("page_to_device", &pdfengine::PdfPage::pageToDevice, py::arg("page_point"), py::arg("device_width"), py::arg("device_height"), py::arg("rotate") = 0) .def("device_to_page", &pdfengine::PdfPage::deviceToPage, diff --git a/corpus/fonts/cns1_test.pdf b/corpus/fonts/cns1_test.pdf new file mode 100644 index 0000000..9dcad82 --- /dev/null +++ b/corpus/fonts/cns1_test.pdf @@ -0,0 +1,90 @@ +%PDF-1.3 +%“Œ‹ž ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /MSung-Light /DescendantFonts [ << +/BaseFont /MSung-Light /CIDSystemInfo << +/Ordering (CNS1) /Registry (Adobe) /Supplement 1 +>> /DW 1000 /FontDescriptor << +/Ascent 752 /CapHeight 737 /Descent -271 /Flags 6 /FontBBox [ -160 -249 1015 888 ] /FontName /MSung-Light + /ItalicAngle 0 /Leading 148 /MaxWidth 1000 /MissingWidth 500 /StemH 45 /StemV 58 + /Type /FontDescriptor /XHeight 553 +>> /Subtype /CIDFontType0 /Type /Font + /W [ 1 2 250 3 [ 408 668 490 875 698 250 240 ] 10 [ 240 417 667 250 313 250 520 500 ] 18 26 500 + 27 28 250 29 31 667 32 [ 396 921 677 615 719 760 625 552 771 802 + 354 ] 43 [ 354 781 604 927 750 823 563 823 729 542 + 698 771 729 948 771 677 635 344 520 344 + 469 500 250 469 521 427 521 438 271 469 + 531 250 ] + 75 [ 250 458 240 802 531 500 521 ] 82 [ 521 365 333 292 521 458 677 479 458 427 + 480 496 480 667 ] ] +>> ] /Encoding /UniGB-UCS2-H /Name /F2 /Subtype /Type0 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (anonymous) /CreationDate (D:20260602153802+05'00') /Creator (anonymous) /Keywords () /ModDate (D:20260602153802+05'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (unspecified) /Title (untitled) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 107 +>> +stream +GapQh0E=F,0U\H3T\pNYT^QKk?tc>IP,;W#U1^23ihPEM_?C]6_CBF/28[_U!/s9cYpe/lM_Qn>nC.&g0fCf=endstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000001155 00000 n +0000001358 00000 n +0000001426 00000 n +0000001687 00000 n +0000001746 00000 n +trailer +<< +/ID +[<01c8dab3d2c3e771bf716fccf2b52ce4><01c8dab3d2c3e771bf716fccf2b52ce4>] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1943 +%%EOF diff --git a/engine/include/pdfengine/pdf_document.hpp b/engine/include/pdfengine/pdf_document.hpp index 9601173..48953fe 100644 --- a/engine/include/pdfengine/pdf_document.hpp +++ b/engine/include/pdfengine/pdf_document.hpp @@ -93,6 +93,8 @@ public: [[nodiscard]] virtual std::expected, EngineError> getFonts() const = 0; [[nodiscard]] virtual std::expected, EngineError> extractAnnotationsText() const = 0; + [[nodiscard]] virtual std::expected getGlyphWidth(const std::string& fontName, uint32_t charcode, double fontSize) const = 0; + [[nodiscard]] virtual DevicePoint pageToDevice(const Point2D& pagePoint, int deviceWidth, int deviceHeight, int rotate = 0) const noexcept = 0; [[nodiscard]] virtual Point2D deviceToPage(const DevicePoint& devicePoint, int deviceWidth, int deviceHeight, int rotate = 0) const noexcept = 0; }; diff --git a/engine/src/parser/pdfium_document.cpp b/engine/src/parser/pdfium_document.cpp index 57892ef..e2c9213 100644 --- a/engine/src/parser/pdfium_document.cpp +++ b/engine/src/parser/pdfium_document.cpp @@ -399,8 +399,13 @@ void deduceFontMetadata(pdfengine::FontInfo& f) { } // 7. Stable Internal Font Identifier + // For subset fonts, fontName already contains the subset prefix (e.g. "ABCDEF+Arial"). + // Use the full fontName directly — it already encodes both the subset tag and + // the base font name, separated by '+'. Concatenating subsetTag + "_" + fontName + // would duplicate the prefix ("ABCDEF_ABCDEF+Arial"). if (f.isSubset && !f.subsetTag.empty()) { - f.internalFontId = f.subsetTag + "_" + f.fontName; + // fontName is "ABCDEF+Arial"; use it as-is for the stable ID. + f.internalFontId = f.fontName; } else { f.internalFontId = f.fontName + "_" + f.type + "_" + std::to_string(f.flags); } @@ -777,7 +782,7 @@ std::expected, EngineError> PdfiumPage::extractTextWith } std::string utf8_char = code_point_to_utf8(cp); - if (utf8_char.empty()) { + if (utf8_char.empty() || cp == '\r' || cp == '\n') { continue; } @@ -859,6 +864,60 @@ Point2D PdfiumPage::deviceToPage(const DevicePoint& devicePoint, int deviceWidth #endif } +std::expected PdfiumPage::getGlyphWidth(const std::string& fontName, uint32_t charcode, double fontSize) const { +#ifdef PDFENGINE_WITH_PDFIUM + if (!page_) { + return std::unexpected(EngineError::Unknown); + } + + FPDF_FONT font = nullptr; + { + std::lock_guard lock(textMutex_); + + auto cached = fontHandleCache_.find(fontName); + if (cached != fontHandleCache_.end()) { + font = cached->second; + } else { + int objectCount = FPDFPage_CountObjects(page_); + for (int i = 0; i < objectCount; ++i) { + FPDF_PAGEOBJECT obj = FPDFPage_GetObject(page_, i); + if (!obj || FPDFPageObj_GetType(obj) != FPDF_PAGEOBJ_TEXT) continue; + + FPDF_FONT pageFont = FPDFTextObj_GetFont(obj); + if (!pageFont) continue; + + size_t nameLen = FPDFFont_GetBaseFontName(pageFont, nullptr, 0); + if (nameLen > 0) { + std::vector nameBuf(nameLen); + if (FPDFFont_GetBaseFontName(pageFont, nameBuf.data(), nameLen) > 0) { + std::string currentName(nameBuf.data()); + if (currentName == fontName) { + font = pageFont; + fontHandleCache_[fontName] = font; + break; + } + } + } + } + } + } + + if (!font) { + return std::unexpected(EngineError::Unknown); + } + + float width = 0.0f; + if (!FPDFFont_GetGlyphWidth(font, charcode, static_cast(fontSize), &width)) { + return std::unexpected(EngineError::Unknown); + } + + return static_cast(width); +#else + (void)fontName; (void)charcode; (void)fontSize; + return std::unexpected(EngineError::Unknown); +#endif +} + void PdfiumPage::ensureTextPageLoaded() const { #ifdef PDFENGINE_WITH_PDFIUM std::lock_guard lock(textMutex_); @@ -1314,8 +1373,10 @@ std::expected, EngineError> PdfiumPage::getFonts() const { // deduceFontMetadata() (SystemFallback or Substituted). // --- Recalculate stable identifier with corrected data --- + // fontName already includes the subset prefix (e.g. "ABCDEF+Arial"); + // using it directly avoids the duplicate-prefix bug. if (f.isSubset && !f.subsetTag.empty()) { - f.internalFontId = f.subsetTag + "_" + f.fontName; + f.internalFontId = f.fontName; } else { f.internalFontId = f.fontName + "_" + f.type + "_" + std::to_string(f.flags); } diff --git a/engine/src/parser/pdfium_document.hpp b/engine/src/parser/pdfium_document.hpp index 7526a05..741de3a 100644 --- a/engine/src/parser/pdfium_document.hpp +++ b/engine/src/parser/pdfium_document.hpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace pdfengine::parser { @@ -43,6 +44,8 @@ public: std::expected, EngineError> getFonts() const override; std::expected, EngineError> extractAnnotationsText() const override; + std::expected getGlyphWidth(const std::string& fontName, uint32_t charcode, double fontSize) const override; + DevicePoint pageToDevice(const Point2D& pagePoint, int deviceWidth, int deviceHeight, int rotate = 0) const noexcept override; Point2D deviceToPage(const DevicePoint& devicePoint, int deviceWidth, int deviceHeight, int rotate = 0) const noexcept override; @@ -51,6 +54,9 @@ private: mutable NativeTextHandle textPage_ = nullptr; int pageIndex_ = 0; mutable std::mutex textMutex_; +#ifdef PDFENGINE_WITH_PDFIUM + mutable std::unordered_map fontHandleCache_; +#endif void ensureTextPageLoaded() const; }; diff --git a/engine/tests/document_test.cpp b/engine/tests/document_test.cpp index c8db0fd..d21fa5a 100644 --- a/engine/tests/document_test.cpp +++ b/engine/tests/document_test.cpp @@ -593,7 +593,7 @@ TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) { } EXPECT_EQ(f.sourceType, "Embedded"); EXPECT_TRUE(f.isEmbedded); - EXPECT_EQ(f.internalFontId, f.subsetTag + "_" + f.fontName); + EXPECT_EQ(f.internalFontId, f.fontName); } else { EXPECT_TRUE(f.subsetTag.empty()); EXPECT_EQ(f.internalFontId, f.fontName + "_" + f.type + "_" + std::to_string(f.flags)); diff --git a/gateway/app/routers/render.py b/gateway/app/routers/render.py index 8ffd228..6472800 100644 --- a/gateway/app/routers/render.py +++ b/gateway/app/routers/render.py @@ -166,3 +166,26 @@ def get_page_fonts(document_id: str, page_index: int) -> List[FontInfoResponse]: except Exception as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + +@router.get("/{page_index}/fonts/glyph-width") +def get_page_glyph_width(document_id: str, page_index: int, font_name: str, charcode: int, font_size: float = 12.0): + if not engine.is_available(): + raise HTTPException( + status_code=status.HTTP_501_NOT_IMPLEMENTED, + detail="Engine bridge (bindings/python) not yet available." + ) + + doc_info = document_store.get_document(document_id) + if not doc_info: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found") + + try: + doc = doc_info["doc_instance"] + page = doc.get_page(page_index) + width = page.get_glyph_width(font_name, charcode, font_size) + return {"width": width} + except IndexError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Page index out of range") + except Exception as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) + diff --git a/gateway/tests/font_api_validation.py b/gateway/tests/font_api_validation.py new file mode 100644 index 0000000..d8dd085 --- /dev/null +++ b/gateway/tests/font_api_validation.py @@ -0,0 +1,388 @@ +""" +Font Extraction API Validation Script +====================================== +Tests font extraction APIs against real PDFs from corpus/fonts/. +Validates: upload, document fonts, page fonts, text extraction, metadata accuracy. +""" + +import json +import sys +import os +import time +import httpx + +BASE_URL = "http://localhost:8000" +CORPUS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "corpus", "fonts")) + +# PDFs to test (in priority order) +TARGET_PDFS = [ + "utf-8.pdf", + "vertical_text.pdf", + "subset_font.pdf", +] + +REQUIRED_FONT_FIELDS = [ + "fontName", "type", "isEmbedded", "isSubset", "isVertical", + "encoding", "cmapName", "cidSystemInfo", "subsetTag", + "sourceType", "substitutedFrom", "substitutedTo", + "normalizedFamily", "internalFontId", "flags", + "ascent", "descent", "capHeight", "hasToUnicode", +] + + +def separator(title: str): + print(f"\n{'='*80}") + print(f" {title}") + print(f"{'='*80}\n") + + +def sub_separator(title: str): + print(f"\n--- {title} ---\n") + + +def validate_font_fields(font: dict, pdf_name: str, font_index: int) -> list: + """Validate that all required fields are present and non-null.""" + issues = [] + for field in REQUIRED_FONT_FIELDS: + if field not in font: + issues.append(f" [MISSING] Font #{font_index} ({font.get('fontName', '?')}): field '{field}' is missing") + return issues + + +def check_duplicates(fonts: list) -> list: + """Check for duplicate font entries.""" + seen = set() + dupes = [] + for f in fonts: + key = f.get("fontName", "") + "|" + f.get("type", "") + "|" + f.get("internalFontId", "") + if key in seen: + dupes.append(f.get("fontName", "?")) + seen.add(key) + return dupes + + +def check_empty_names(fonts: list) -> list: + """Check for empty font names.""" + return [i for i, f in enumerate(fonts) if not f.get("fontName", "").strip()] + + +def validate_pdf(client: httpx.Client, pdf_filename: str) -> dict: + """Run all validation steps for a single PDF.""" + result = { + "filename": pdf_filename, + "doc_id": None, + "upload_status": None, + "upload_response": None, + "doc_fonts": None, + "doc_font_count": 0, + "page_fonts": None, + "page_font_count": 0, + "text_result": None, + "glyph_count": 0, + "font_sizes": set(), + "issues": [], + "errors": [], + } + + pdf_path = os.path.join(CORPUS_DIR, pdf_filename) + if not os.path.exists(pdf_path): + result["errors"].append(f"PDF file not found: {pdf_path}") + return result + + # ========== STEP 1: Upload ========== + sub_separator(f"Step 1: Upload {pdf_filename}") + try: + with open(pdf_path, "rb") as f: + resp = client.post( + "/documents", + files={"file": (os.path.basename(pdf_path), f, "application/pdf")}, + ) + result["upload_status"] = resp.status_code + result["upload_response"] = resp.json() + print(f" Status: {resp.status_code}") + print(f" Response: {json.dumps(resp.json(), indent=2)}") + + if resp.status_code != 201: + result["errors"].append(f"Upload failed with status {resp.status_code}: {resp.text}") + return result + + result["doc_id"] = resp.json()["id"] + print(f" Document ID: {result['doc_id']}") + except Exception as e: + result["errors"].append(f"Upload exception: {e}") + return result + + doc_id = result["doc_id"] + + # ========== STEP 2: Document Font Extraction ========== + sub_separator("Step 2: Document Font Extraction") + try: + resp = client.get(f"/documents/{doc_id}/fonts") + print(f" Status: {resp.status_code}") + + if resp.status_code != 200: + result["errors"].append(f"Document fonts failed: {resp.status_code} - {resp.text}") + else: + fonts = resp.json() + result["doc_fonts"] = fonts + result["doc_font_count"] = len(fonts) + print(f" Font count: {len(fonts)}") + + # Print each font in full + for i, f in enumerate(fonts): + print(f"\n Font #{i}:") + print(f" {json.dumps(f, indent=4)}") + + # Validate font count > 0 + if len(fonts) == 0: + result["issues"].append("Document fonts: count is 0") + + # Validate no empty names + empty = check_empty_names(fonts) + if empty: + result["issues"].append(f"Document fonts: empty font names at indices {empty}") + + # Validate no duplicates + dupes = check_duplicates(fonts) + if dupes: + result["issues"].append(f"Document fonts: duplicate fonts: {dupes}") + + # Validate all fields present + for i, f in enumerate(fonts): + field_issues = validate_font_fields(f, pdf_filename, i) + result["issues"].extend(field_issues) + + except Exception as e: + result["errors"].append(f"Document fonts exception: {e}") + + # ========== STEP 3: Page Font Extraction ========== + sub_separator("Step 3: Page Font Extraction (page 0)") + try: + resp = client.get(f"/documents/{doc_id}/pages/0/fonts") + print(f" Status: {resp.status_code}") + + if resp.status_code != 200: + result["errors"].append(f"Page fonts failed: {resp.status_code} - {resp.text}") + else: + page_fonts = resp.json() + result["page_fonts"] = page_fonts + result["page_font_count"] = len(page_fonts) + print(f" Page font count: {len(page_fonts)}") + + for i, f in enumerate(page_fonts): + print(f"\n Page Font #{i}:") + print(f" {json.dumps(f, indent=4)}") + + if len(page_fonts) == 0: + result["issues"].append("Page fonts: count is 0") + + # Compare with document fonts + if result["doc_fonts"] is not None: + doc_font_names = {f["fontName"] for f in result["doc_fonts"]} + page_font_names = {f["fontName"] for f in page_fonts} + + print(f"\n Document font names: {sorted(doc_font_names)}") + print(f" Page font names: {sorted(page_font_names)}") + + # Page fonts should be a subset of document fonts + extra_in_page = page_font_names - doc_font_names + if extra_in_page: + result["issues"].append(f"Page fonts not in document fonts: {extra_in_page}") + print(f" [ISSUE] Page has fonts not in document-level: {extra_in_page}") + else: + print(f" [OK] Page fonts are a subset of document fonts") + + except Exception as e: + result["errors"].append(f"Page fonts exception: {e}") + + # ========== STEP 4: Text Extraction ========== + sub_separator("Step 4: Text Extraction (page 0)") + try: + resp = client.get(f"/documents/{doc_id}/pages/0/text") + print(f" Status: {resp.status_code}") + + if resp.status_code != 200: + result["errors"].append(f"Text extraction failed: {resp.status_code} - {resp.text}") + else: + text_data = resp.json() + result["text_result"] = text_data + text_content = text_data.get("text", "") + glyphs = text_data.get("glyphs", []) + result["glyph_count"] = len(glyphs) + + print(f" Extracted text: {repr(text_content[:300])}") + print(f" Glyph count: {len(glyphs)}") + + if not text_content.strip(): + result["issues"].append("Text extraction: empty text") + + if len(glyphs) == 0: + result["issues"].append("Text extraction: no glyphs returned") + else: + # Show first 5 glyphs as samples + print(f"\n First 5 glyphs (sample):") + for i, g in enumerate(glyphs[:5]): + print(f" Glyph #{i}: {json.dumps(g, indent=6)}") + + # Check glyph structure + for i, g in enumerate(glyphs): + font_size = g.get("fontSize", 0) + if font_size > 0: + result["font_sizes"].add(font_size) + elif font_size == 0 and i < 5: + result["issues"].append(f"Glyph #{i}: fontSize is 0") + + # Validate coordinate fields exist + for coord in ["x", "y", "right", "bottom"]: + if coord not in g and i < 3: + result["issues"].append(f"Glyph #{i}: missing coordinate '{coord}'") + + print(f"\n Font sizes detected: {sorted(result['font_sizes'])}") + + except Exception as e: + result["errors"].append(f"Text extraction exception: {e}") + + # ========== STEP 5: Content-Specific Validation ========== + sub_separator("Step 5: Content-Specific Validation") + + if "vertical" in pdf_filename.lower() and result["doc_fonts"]: + vertical_fonts = [f for f in result["doc_fonts"] if f.get("isVertical")] + print(f" Vertical text PDF - fonts with isVertical=true: {len(vertical_fonts)}") + if len(vertical_fonts) == 0: + result["issues"].append("vertical_text.pdf: No fonts have isVertical=true") + print(f" [ISSUE] No vertical fonts detected!") + else: + for vf in vertical_fonts: + print(f" - {vf['fontName']} (isVertical=true)") + print(f" [OK] Vertical fonts detected correctly") + + if "subset" in pdf_filename.lower() and result["doc_fonts"]: + subset_fonts = [f for f in result["doc_fonts"] if f.get("isSubset")] + print(f" Subset font PDF - fonts with isSubset=true: {len(subset_fonts)}") + if len(subset_fonts) == 0: + result["issues"].append("subset_font.pdf: No fonts have isSubset=true") + print(f" [ISSUE] No subset fonts detected!") + else: + for sf in subset_fonts: + print(f" - {sf['fontName']} (isSubset=true, subsetTag='{sf.get('subsetTag', '')}')") + print(f" [OK] Subset fonts detected correctly") + + if "utf" in pdf_filename.lower() and result["text_result"]: + text = result["text_result"].get("text", "") + print(f" UTF-8 PDF - extracted text: {repr(text[:300])}") + # Check for non-ASCII characters + non_ascii = [c for c in text if ord(c) > 127] + if non_ascii: + print(f" Non-ASCII characters found: {len(non_ascii)} chars") + print(f" Sample non-ASCII: {repr(''.join(non_ascii[:30]))}") + print(f" [OK] UTF-8 text extracts with non-ASCII content") + else: + print(f" [INFO] No non-ASCII characters detected - content may be ASCII-only") + + return result + + +def main(): + print("=" * 80) + print(" FONT EXTRACTION API VALIDATION") + print(f" Server: {BASE_URL}") + print(f" Corpus: {CORPUS_DIR}") + print(f" Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S')}") + print("=" * 80) + + client = httpx.Client(base_url=BASE_URL, timeout=30.0) + + # Verify server is up + try: + r = client.get("/") + print(f"\n Server status: OK ({r.status_code})") + except Exception as e: + print(f"\n [FATAL] Cannot connect to server: {e}") + sys.exit(1) + + # Verify corpus directory + if not os.path.isdir(CORPUS_DIR): + print(f"\n [FATAL] Corpus directory not found: {CORPUS_DIR}") + sys.exit(1) + + available = [f for f in TARGET_PDFS if os.path.exists(os.path.join(CORPUS_DIR, f))] + print(f" PDFs to validate: {available}") + + results = [] + for pdf in available: + separator(f"VALIDATING: {pdf}") + result = validate_pdf(client, pdf) + results.append(result) + + # ========== FINAL REPORT ========== + separator("FINAL REPORT") + + for r in results: + print(f"\n{'~'*60}") + print(f" PDF: {r['filename']}") + print(f" Document ID: {r['doc_id']}") + print(f" Upload Status: {r['upload_status']}") + print(f" Document Font Count: {r['doc_font_count']}") + print(f" Page Font Count: {r['page_font_count']}") + print(f" Glyph Count: {r['glyph_count']}") + print(f" Font Sizes: {sorted(r['font_sizes']) if r['font_sizes'] else 'N/A'}") + print(f" Issues: {len(r['issues'])}") + for issue in r['issues']: + print(f" >> {issue}") + print(f" Errors: {len(r['errors'])}") + for err in r['errors']: + print(f" XX {err}") + + # Summary Answers + separator("SUMMARY ANSWERS") + + total_issues = sum(len(r["issues"]) for r in results) + total_errors = sum(len(r["errors"]) for r in results) + + all_fonts_extracted = all(r["doc_font_count"] > 0 for r in results if not r["errors"]) + print(f" 1. Are fonts being extracted correctly?") + print(f" {'YES' if all_fonts_extracted else 'NO'} - {sum(r['doc_font_count'] for r in results)} total fonts across {len(results)} PDFs") + + page_doc_consistent = all( + not any("not in document" in i for i in r["issues"]) + for r in results + ) + print(f"\n 2. Are page fonts and document fonts consistent?") + print(f" {'YES' if page_doc_consistent else 'NO'}") + + font_sizes_ok = all(r["glyph_count"] > 0 for r in results if not r["errors"]) + print(f"\n 3. Are font sizes being extracted correctly?") + print(f" {'YES' if font_sizes_ok else 'NO'}") + + # Check vertical/subset + vertical_ok = True + subset_ok = True + for r in results: + if "vertical" in r["filename"] and any("isVertical" in i for i in r["issues"]): + vertical_ok = False + if "subset" in r["filename"] and any("isSubset" in i for i in r["issues"]): + subset_ok = False + + print(f"\n 4. Are vertical/subset fonts detected correctly?") + print(f" Vertical: {'YES' if vertical_ok else 'NO'}") + print(f" Subset: {'YES' if subset_ok else 'NO'}") + + print(f"\n 5. Are there any metadata inaccuracies?") + if total_issues == 0 and total_errors == 0: + print(f" NO - All {len(results)} PDFs passed validation cleanly") + else: + print(f" YES - {total_issues} issues and {total_errors} errors found") + for r in results: + for issue in r["issues"]: + print(f" - [{r['filename']}] {issue}") + + print(f"\n{'='*80}") + print(f" VALIDATION COMPLETE: {total_issues} issues, {total_errors} errors") + print(f"{'='*80}") + + client.close() + return 0 if total_errors == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/gateway/tests/test_font_regression.py b/gateway/tests/test_font_regression.py index 708acf5..7ee710f 100644 --- a/gateway/tests/test_font_regression.py +++ b/gateway/tests/test_font_regression.py @@ -1,3 +1,4 @@ +import re import pytest from fastapi.testclient import TestClient import sys @@ -15,8 +16,12 @@ except ImportError: client = TestClient(app) +# Regex: six uppercase ASCII letters followed by '+' +SUBSET_PREFIX_RE = re.compile(r'^[A-Z]{6}\+') + + def get_doc_id(filename: str) -> str: - # First ensure the document is loaded + """Upload a PDF from corpus/fonts and return its document ID.""" filepath = os.path.abspath(f"../corpus/fonts/{filename}") with open(filepath, "rb") as f: resp = client.post( @@ -26,8 +31,13 @@ def get_doc_id(filename: str) -> str: assert resp.status_code == 201, f"Failed to load {filename}: {resp.json()}" return resp.json()["id"] + +# ========================================================================= +# 1. Vertical font regression +# ========================================================================= + def test_is_vertical_regression(): - # 1. Verify Identity-V fonts are detected correctly + """Identity-V fonts must be flagged isVertical; horizontal fonts must not.""" doc_id = get_doc_id("vertical_text.pdf") resp = client.get(f"/documents/{doc_id}/fonts") assert resp.status_code == 200 @@ -36,8 +46,7 @@ def test_is_vertical_regression(): font = fonts[0] assert font["isVertical"] is True assert font["encoding"] == "Identity-V" - - # 2. Verify horizontal fonts are not falsely detected as vertical + doc_id_h = get_doc_id("utf-8.pdf") resp_h = client.get(f"/documents/{doc_id_h}/fonts") fonts_h = resp_h.json() @@ -45,20 +54,123 @@ def test_is_vertical_regression(): for f in fonts_h: assert f["isVertical"] is False -def test_internal_font_id_regression(): - # Verify subset fonts do not duplicate subset prefixes + +# ========================================================================= +# 2. Internal Font ID: no duplicate subset prefix (core regression) +# ========================================================================= + +def test_internal_font_id_no_duplicate_prefix(): + """ + Regression: subset fonts must NOT produce "ABCDEF_ABCDEF+Arial". + The internalFontId for a subset font should be the fontName itself + (e.g. "ABCDEF+Arial"), which already encodes the subset tag. + """ + doc_id = get_doc_id("subset_font.pdf") + resp = client.get(f"/documents/{doc_id}/fonts") + assert resp.status_code == 200 + fonts = resp.json() + assert len(fonts) > 0 + + for font in fonts: + fid = font["internalFontId"] + tag = font.get("subsetTag", "") + + if font.get("isSubset") and tag: + # Must NOT start with "TAG_TAG" + assert not fid.startswith(tag + "_" + tag), ( + f"Duplicate subset prefix detected: internalFontId='{fid}'" + ) + # Must equal fontName directly (e.g. "ABCDEF+Arial") + assert fid == font["fontName"], ( + f"Expected internalFontId==fontName for subset font, " + f"got '{fid}' vs '{font['fontName']}'" + ) + + +def test_internal_font_id_subset_format(): + """ + For any subset font the internalFontId must match the pattern + ABCDEF+BaseName — exactly the fontName reported by PDFium. + """ doc_id = get_doc_id("text_font.pdf") resp = client.get(f"/documents/{doc_id}/fonts") assert resp.status_code == 200 fonts = resp.json() assert len(fonts) > 0 + for font in fonts: if font.get("isSubset"): assert font["subsetTag"] in font["fontName"] - assert not font["internalFontId"].startswith(font["subsetTag"] + "_" + font["subsetTag"]) + # internalFontId == fontName (e.g. "ABCDEF+Arial") + assert font["internalFontId"] == font["fontName"] + # The ID must contain exactly one '+' from the subset tag + assert font["internalFontId"].count("+") == 1 + + +# ========================================================================= +# 3. Non-subset font ID format +# ========================================================================= + +def test_internal_font_id_non_subset_format(): + """ + Non-subset fonts must have internalFontId = fontName_type_flags. + Examples: "Helvetica_Type1_32", "Times-Roman_Type1_32". + """ + doc_id = get_doc_id("utf-8.pdf") + resp = client.get(f"/documents/{doc_id}/fonts") + assert resp.status_code == 200 + fonts = resp.json() + assert len(fonts) > 0 + + for font in fonts: + if not font.get("isSubset"): + expected = f"{font['fontName']}_{font['type']}_{font['flags']}" + assert font["internalFontId"] == expected, ( + f"Non-subset internalFontId mismatch: " + f"got '{font['internalFontId']}', expected '{expected}'" + ) + # Must NOT contain a '+' (no subset prefix) + assert "+" not in font["internalFontId"] + + +# ========================================================================= +# 4. ID stability across document-level and page-level APIs +# ========================================================================= + +def test_font_id_stable_across_apis(): + """ + The internalFontId for the same font must be identical whether queried + from the document-level /fonts endpoint or the page-level /pages/0/fonts. + """ + for pdf in ("subset_font.pdf", "utf-8.pdf", "vertical_text.pdf"): + filepath = os.path.abspath(f"../corpus/fonts/{pdf}") + if not os.path.exists(filepath): + continue + + doc_id = get_doc_id(pdf) + + doc_resp = client.get(f"/documents/{doc_id}/fonts") + page_resp = client.get(f"/documents/{doc_id}/pages/0/fonts") + assert doc_resp.status_code == 200 + assert page_resp.status_code == 200 + + doc_ids = {f["fontName"]: f["internalFontId"] for f in doc_resp.json()} + page_ids = {f["fontName"]: f["internalFontId"] for f in page_resp.json()} + + for name in page_ids: + assert name in doc_ids, f"Page font '{name}' not in doc fonts for {pdf}" + assert page_ids[name] == doc_ids[name], ( + f"ID mismatch for '{name}' in {pdf}: " + f"doc='{doc_ids[name]}' vs page='{page_ids[name]}'" + ) + + +# ========================================================================= +# 5. CID collection regression +# ========================================================================= def test_cid_collection_regression(): - # Verify Adobe collections + """Adobe CID collections must use the 'Adobe-' prefix.""" doc_id = get_doc_id("vertical_text.pdf") resp = client.get(f"/documents/{doc_id}/fonts") fonts = resp.json() @@ -67,6 +179,29 @@ def test_cid_collection_regression(): if font.get("cidSystemInfo") and font.get("cidSystemInfo") != "None": assert "Adobe-" in font["cidSystemInfo"] + +def test_cns1_regression(): + """Verify Adobe-CNS1 (Traditional Chinese) CID fonts and text extraction.""" + doc_id = get_doc_id("cns1_test.pdf") + + # 1. Verify font extraction + resp = client.get(f"/documents/{doc_id}/fonts") + assert resp.status_code == 200 + fonts = resp.json() + assert len(fonts) > 0 + cns1_fonts = [f for f in fonts if f.get("cidSystemInfo") == "Adobe-CNS1"] + assert len(cns1_fonts) > 0, "No Adobe-CNS1 fonts detected" + + # 2. Verify text extraction + resp = client.get(f"/documents/{doc_id}/pages/0/text") + assert resp.status_code == 200 + text = resp.json()["text"] + assert "\u4e00\u4e2d\u4ed7" in text, "Failed to extract Traditional Chinese text" + +# ========================================================================= +# 6. UTF-8 corpus regression +# ========================================================================= + def test_utf8_corpus_regression(): doc_id = get_doc_id("utf-8.pdf") resp = client.get(f"/documents/{doc_id}/pages/0/text") @@ -75,9 +210,35 @@ def test_utf8_corpus_regression(): assert len(data["glyphs"]) > 0 assert len(data["text"]) > 0 + +# ========================================================================= +# 7. Font size regression +# ========================================================================= + def test_font_size_regression(): doc_id = get_doc_id("utf-8.pdf") resp = client.get(f"/documents/{doc_id}/pages/0/text") data = resp.json() sizes = set(g["fontSize"] for g in data["glyphs"]) - assert len(sizes) >= 2 # utf-8.pdf should have multiple font sizes + assert len(sizes) >= 2 # utf-8.pdf should have multiple font sizes + + +# ========================================================================= +# 8. ID uniqueness +# ========================================================================= + +def test_font_ids_unique_within_document(): + """Each font within a document must have a distinct internalFontId.""" + for pdf in ("subset_font.pdf", "utf-8.pdf", "vertical_text.pdf"): + filepath = os.path.abspath(f"../corpus/fonts/{pdf}") + if not os.path.exists(filepath): + continue + + doc_id = get_doc_id(pdf) + resp = client.get(f"/documents/{doc_id}/fonts") + fonts = resp.json() + + ids = [f["internalFontId"] for f in fonts] + assert len(ids) == len(set(ids)), ( + f"Duplicate internalFontId values in {pdf}: {ids}" + ) diff --git a/gateway/tests/test_glyph_metrics.py b/gateway/tests/test_glyph_metrics.py new file mode 100644 index 0000000..1954adb --- /dev/null +++ b/gateway/tests/test_glyph_metrics.py @@ -0,0 +1,93 @@ +import pytest +import os +from fastapi.testclient import TestClient +from app.main import app + +client = TestClient(app) + + +def get_doc_id(filename: str) -> str: + """Upload a PDF from corpus/fonts and return its document ID.""" + filepath = os.path.abspath(f"gateway/../corpus/fonts/{filename}") + if not os.path.exists(filepath): + filepath = os.path.abspath(f"../corpus/fonts/{filename}") + with open(filepath, "rb") as f: + resp = client.post( + "/documents", + files={"file": (filename, f, "application/pdf")} + ) + assert resp.status_code == 201, f"Failed to load {filename}: {resp.json()}" + return resp.json()["id"] + + +def test_glyph_width_proportionality(): + """Verify that wide characters return larger glyph widths than narrow characters.""" + doc_id = get_doc_id("utf-8.pdf") + + # 1. Fetch fonts for the page + resp = client.get(f"/documents/{doc_id}/pages/0/fonts") + assert resp.status_code == 200 + fonts = resp.json() + assert len(fonts) > 0, "No fonts extracted from utf-8.pdf" + + # Find an embedded or substituted font name + font_name = fonts[0]["fontName"] + + # 2. Get width of wide character 'W' (charcode 87) + w_resp = client.get( + f"/documents/{doc_id}/pages/0/fonts/glyph-width", + params={"font_name": font_name, "charcode": 87, "font_size": 12.0} + ) + assert w_resp.status_code == 200 + w_width = w_resp.json()["width"] + assert w_width > 0.0 + + # 3. Get width of narrow character 'i' (charcode 105) + i_resp = client.get( + f"/documents/{doc_id}/pages/0/fonts/glyph-width", + params={"font_name": font_name, "charcode": 105, "font_size": 12.0} + ) + assert i_resp.status_code == 200 + i_width = i_resp.json()["width"] + assert i_width > 0.0 + + # 'W' must be strictly wider than 'i' in proportional typefaces + assert w_width > i_width, f"Expected width('W') > width('i'), got {w_width} vs {i_width}" + + +def test_glyph_width_font_size_scaling(): + """Verify that glyph width scales proportionally with font size.""" + doc_id = get_doc_id("utf-8.pdf") + resp = client.get(f"/documents/{doc_id}/pages/0/fonts") + fonts = resp.json() + font_name = fonts[0]["fontName"] + + # Width at 12pt + resp_12 = client.get( + f"/documents/{doc_id}/pages/0/fonts/glyph-width", + params={"font_name": font_name, "charcode": 65, "font_size": 12.0} + ) + assert resp_12.status_code == 200 + width_12 = resp_12.json()["width"] + + # Width at 24pt + resp_24 = client.get( + f"/documents/{doc_id}/pages/0/fonts/glyph-width", + params={"font_name": font_name, "charcode": 65, "font_size": 24.0} + ) + assert resp_24.status_code == 200 + width_24 = resp_24.json()["width"] + + # Scaling must be linear: width(24pt) = 2.0 * width(12pt) + assert pytest.approx(width_24) == 2.0 * width_12 + + +def test_glyph_width_invalid_font(): + """Verify that querying an invalid font returns a bad request error.""" + doc_id = get_doc_id("utf-8.pdf") + resp = client.get( + f"/documents/{doc_id}/pages/0/fonts/glyph-width", + params={"font_name": "NonExistentFontName123", "charcode": 65, "font_size": 12.0} + ) + assert resp.status_code == 400 + assert "detail" in resp.json() diff --git a/gateway/tests/test_routes.py b/gateway/tests/test_routes.py index 2849845..bf5d6a7 100644 --- a/gateway/tests/test_routes.py +++ b/gateway/tests/test_routes.py @@ -137,6 +137,10 @@ def test_extract_page_text(client: TestClient): first_glyph = glyphs[0] for key in ["text", "x", "y", "w", "h", "fontSize"]: assert key in first_glyph + + for g in glyphs: + assert g["fontSize"] != 1.0, f"Fake fontSize 1.0 detected for glyph: {g}" + assert g["text"] not in ["\r", "\n"], f"Control character detected in glyph bounds: {g}" def test_apply_edits_and_incremental_save(client: TestClient): with open(HELLO_WORLD_PDF, "rb") as f: @@ -266,7 +270,7 @@ def test_font_size_and_diagnostics_advanced(client: TestClient): assert font["sourceType"] == "Embedded" assert len(font["subsetTag"]) == 6 assert font["subsetTag"].isupper() - assert font["internalFontId"] == f"{font['subsetTag']}_{font['fontName']}" + assert font["internalFontId"] == font["fontName"] else: assert len(font["subsetTag"]) == 0 assert font["internalFontId"] == f"{font['fontName']}_{font['type']}_{font['flags']}" diff --git a/gateway/tests/validation_output.txt b/gateway/tests/validation_output.txt new file mode 100644 index 0000000..a4a8bb8 --- /dev/null +++ b/gateway/tests/validation_output.txt @@ -0,0 +1,554 @@ +================================================================================ + FONT EXTRACTION API VALIDATION + Server: http://localhost:8000 + Corpus: C:\Users\Maskan\Desktop\pdf_editor\pdf\corpus\fonts + Timestamp: 2026-06-02 12:36:47 +================================================================================ + + Server status: OK (200) + PDFs to validate: ['utf-8.pdf', 'vertical_text.pdf', 'subset_font.pdf'] + +================================================================================ + VALIDATING: utf-8.pdf +================================================================================ + + +--- Step 1: Upload utf-8.pdf --- + + Status: 201 + Response: { + "id": "6764e35e-3594-4b50-a378-3ea122a54d9d", + "filename": "utf-8.pdf", + "sizeBytes": 1275, + "totalPages": 1, + "uploadedAt": "2026-06-02T07:06:50.429369Z", + "status": "ready" +} + Document ID: 6764e35e-3594-4b50-a378-3ea122a54d9d + +--- Step 2: Document Font Extraction --- + + Status: 200 + Font count: 2 + + Font #0: + { + "fontName": "Helvetica", + "type": "Type1", + "isEmbedded": false, + "isSubset": false, + "isVertical": false, + "encoding": "WinAnsiEncoding", + "hasToUnicode": true, + "cmapName": "None", + "cidSystemInfo": "None", + "subsetTag": "", + "sourceType": "SystemFallback", + "substitutedFrom": "", + "substitutedTo": "", + "normalizedFamily": "Helvetica", + "internalFontId": "Helvetica_Type1_32", + "flags": 32, + "ascent": 905.0, + "descent": -211.0, + "capHeight": 728.0 +} + + Font #1: + { + "fontName": "Times-Roman", + "type": "Type1", + "isEmbedded": false, + "isSubset": false, + "isVertical": false, + "encoding": "WinAnsiEncoding", + "hasToUnicode": true, + "cmapName": "None", + "cidSystemInfo": "None", + "subsetTag": "", + "sourceType": "SystemFallback", + "substitutedFrom": "", + "substitutedTo": "", + "normalizedFamily": "Times", + "internalFontId": "Times-Roman_Type1_32", + "flags": 32, + "ascent": 891.0, + "descent": -216.0, + "capHeight": 662.0 +} + +--- Step 3: Page Font Extraction (page 0) --- + + Status: 200 + Page font count: 2 + + Page Font #0: + { + "fontName": "Helvetica", + "type": "Type1", + "isEmbedded": false, + "isSubset": false, + "isVertical": false, + "encoding": "WinAnsiEncoding", + "hasToUnicode": true, + "cmapName": "None", + "cidSystemInfo": "None", + "subsetTag": "", + "sourceType": "SystemFallback", + "substitutedFrom": "", + "substitutedTo": "", + "normalizedFamily": "Helvetica", + "internalFontId": "Helvetica_Type1_32", + "flags": 32, + "ascent": 905.0, + "descent": -211.0, + "capHeight": 728.0 +} + + Page Font #1: + { + "fontName": "Times-Roman", + "type": "Type1", + "isEmbedded": false, + "isSubset": false, + "isVertical": false, + "encoding": "WinAnsiEncoding", + "hasToUnicode": true, + "cmapName": "None", + "cidSystemInfo": "None", + "subsetTag": "", + "sourceType": "SystemFallback", + "substitutedFrom": "", + "substitutedTo": "", + "normalizedFamily": "Times", + "internalFontId": "Times-Roman_Type1_32", + "flags": 32, + "ascent": 891.0, + "descent": -216.0, + "capHeight": 662.0 +} + + Document font names: ['Helvetica', 'Times-Roman'] + Page font names: ['Helvetica', 'Times-Roman'] + [OK] Page fonts are a subset of document fonts + +--- Step 4: Text Extraction (page 0) --- + + Status: 200 + Extracted text: 'Hello World - UTF-8 Test Document\r\nStandard Latin Text for Encoding Verification\r\nFont Size Detection Sample: Small Text 12pt\r\nLARGE TEXT FOR SIZE 18PT DETECTION\r\nMore 18pt content: ABCDEFGHabcdefgh 0123456789\r\nBack to 12pt: The quick brown fox jumps over the lazy dog\r\nSpecial chars: copyright secti' + Glyph count: 312 + + First 5 glyphs (sample): + Glyph #0: { + "text": "H", + "x": 72.95999908447266, + "y": 720.0, + "w": 6.7440032958984375, + "h": 8.59197998046875, + "fontSize": 12.0 +} + Glyph #1: { + "text": "e", + "x": 81.10800170898438, + "y": 719.8679809570312, + "w": 5.736000061035156, + "h": 6.49200439453125, + "fontSize": 12.0 +} + Glyph #2: { + "text": "l", + "x": 88.10400390625, + "y": 720.0, + "w": 1.055999755859375, + "h": 8.59197998046875, + "fontSize": 12.0 +} + Glyph #3: { + "text": "l", + "x": 90.76799774169922, + "y": 720.0, + "w": 1.055999755859375, + "h": 8.59197998046875, + "fontSize": 12.0 +} + Glyph #4: { + "text": "o", + "x": 93.05999755859375, + "y": 719.8679809570312, + "w": 5.832000732421875, + "h": 6.49200439453125, + "fontSize": 12.0 +} + + Font sizes detected: [1.0, 12.0, 18.0] + +--- Step 5: Content-Specific Validation --- + + UTF-8 PDF - extracted text: 'Hello World - UTF-8 Test Document\r\nStandard Latin Text for Encoding Verification\r\nFont Size Detection Sample: Small Text 12pt\r\nLARGE TEXT FOR SIZE 18PT DETECTION\r\nMore 18pt content: ABCDEFGHabcdefgh 0123456789\r\nBack to 12pt: The quick brown fox jumps over the lazy dog\r\nSpecial chars: copyright secti' + [INFO] No non-ASCII characters detected - content may be ASCII-only + +================================================================================ + VALIDATING: vertical_text.pdf +================================================================================ + + +--- Step 1: Upload vertical_text.pdf --- + + Status: 201 + Response: { + "id": "d28fd4e4-a059-4a9b-bf15-b6cf25caf280", + "filename": "vertical_text.pdf", + "sizeBytes": 3518, + "totalPages": 1, + "uploadedAt": "2026-06-02T07:06:50.465835Z", + "status": "ready" +} + Document ID: d28fd4e4-a059-4a9b-bf15-b6cf25caf280 + +--- Step 2: Document Font Extraction --- + + Status: 200 + Font count: 1 + + Font #0: + { + "fontName": "Test", + "type": "TrueType", + "isEmbedded": false, + "isSubset": false, + "isVertical": true, + "encoding": "Identity-V", + "hasToUnicode": true, + "cmapName": "Identity-V", + "cidSystemInfo": "None", + "subsetTag": "", + "sourceType": "Substituted", + "substitutedFrom": "Test", + "substitutedTo": "Arial", + "normalizedFamily": "Test", + "internalFontId": "Test_TrueType_524320", + "flags": 524320, + "ascent": 905.0, + "descent": -211.0, + "capHeight": 728.0 +} + +--- Step 3: Page Font Extraction (page 0) --- + + Status: 200 + Page font count: 1 + + Page Font #0: + { + "fontName": "Test", + "type": "TrueType", + "isEmbedded": false, + "isSubset": false, + "isVertical": true, + "encoding": "Identity-V", + "hasToUnicode": true, + "cmapName": "Identity-V", + "cidSystemInfo": "None", + "subsetTag": "", + "sourceType": "Substituted", + "substitutedFrom": "Test", + "substitutedTo": "Arial", + "normalizedFamily": "Test", + "internalFontId": "Test_TrueType_524320", + "flags": 524320, + "ascent": 905.0, + "descent": -211.0, + "capHeight": 728.0 +} + + Document font names: ['Test'] + Page font names: ['Test'] + [OK] Page fonts are a subset of document fonts + +--- Step 4: Text Extraction (page 0) --- + + Status: 200 + Extracted text: 'Hello World!\r\nHello' + Glyph count: 19 + + First 5 glyphs (sample): + Glyph #0: { + "text": "H", + "x": 6.832000255584717, + "y": 180.1840057373047, + "w": 6.552000522613525, + "h": 8.699996948242188, + "fontSize": 12.0 +} + Glyph #1: { + "text": "e", + "x": 7.324000358581543, + "y": 171.39999389648438, + "w": 5.495999336242676, + "h": 6.756011962890625, + "fontSize": 12.0 +} + Glyph #2: { + "text": "l", + "x": 9.687999725341797, + "y": 160.49200439453125, + "w": 1.055999755859375, + "h": 9.251998901367188, + "fontSize": 12.0 +} + Glyph #3: { + "text": "l", + "x": 9.687999725341797, + "y": 149.4759979248047, + "w": 1.055999755859375, + "h": 9.251998901367188, + "fontSize": 12.0 +} + Glyph #4: { + "text": "o", + "x": 7.324000358581543, + "y": 140.69200134277344, + "w": 5.951999664306641, + "h": 6.7559967041015625, + "fontSize": 12.0 +} + + Font sizes detected: [1.0, 12.0] + +--- Step 5: Content-Specific Validation --- + + Vertical text PDF - fonts with isVertical=true: 1 + - Test (isVertical=true) + [OK] Vertical fonts detected correctly + +================================================================================ + VALIDATING: subset_font.pdf +================================================================================ + + +--- Step 1: Upload subset_font.pdf --- + + Status: 201 + Response: { + "id": "cf3f2218-6ee7-4e34-973d-6e1b61c13cf8", + "filename": "subset_font.pdf", + "sizeBytes": 646, + "totalPages": 1, + "uploadedAt": "2026-06-02T07:06:50.497336Z", + "status": "ready" +} + Document ID: cf3f2218-6ee7-4e34-973d-6e1b61c13cf8 + +--- Step 2: Document Font Extraction --- + + Status: 200 + Font count: 1 + + Font #0: + { + "fontName": "ABCDEF+Arial", + "type": "TrueType", + "isEmbedded": true, + "isSubset": true, + "isVertical": false, + "encoding": "WinAnsiEncoding", + "hasToUnicode": true, + "cmapName": "None", + "cidSystemInfo": "None", + "subsetTag": "ABCDEF", + "sourceType": "Embedded", + "substitutedFrom": "", + "substitutedTo": "", + "normalizedFamily": "Arial", + "internalFontId": "ABCDEF_ABCDEF+Arial", + "flags": 0, + "ascent": 905.0, + "descent": -211.0, + "capHeight": 728.0 +} + +--- Step 3: Page Font Extraction (page 0) --- + + Status: 200 + Page font count: 1 + + Page Font #0: + { + "fontName": "ABCDEF+Arial", + "type": "TrueType", + "isEmbedded": true, + "isSubset": true, + "isVertical": false, + "encoding": "WinAnsiEncoding", + "hasToUnicode": true, + "cmapName": "None", + "cidSystemInfo": "None", + "subsetTag": "ABCDEF", + "sourceType": "Embedded", + "substitutedFrom": "", + "substitutedTo": "", + "normalizedFamily": "Arial", + "internalFontId": "ABCDEF_ABCDEF+Arial", + "flags": 0, + "ascent": 905.0, + "descent": -211.0, + "capHeight": 728.0 +} + + Document font names: ['ABCDEF+Arial'] + Page font names: ['ABCDEF+Arial'] + [OK] Page fonts are a subset of document fonts + +--- Step 4: Text Extraction (page 0) --- + + Status: 200 + Extracted text: 'Subset Text' + Glyph count: 11 + + First 5 glyphs (sample): + Glyph #0: { + "text": "S", + "x": 72.54000091552734, + "y": 719.8679809570312, + "w": 6.839996337890625, + "h": 8.8680419921875, + "fontSize": 12.0 +} + Glyph #1: { + "text": "u", + "x": 80.77200317382812, + "y": 719.8679809570312, + "w": 5.0399932861328125, + "h": 6.36004638671875, + "fontSize": 12.0 +} + Glyph #2: { + "text": "b", + "x": 87.45600128173828, + "y": 719.8679809570312, + "w": 5.400001525878906, + "h": 8.7239990234375, + "fontSize": 12.0 +} + Glyph #3: { + "text": "s", + "x": 93.72000122070312, + "y": 719.8679809570312, + "w": 5.159996032714844, + "h": 6.49200439453125, + "fontSize": 12.0 +} + Glyph #4: { + "text": "e", + "x": 99.79199981689453, + "y": 719.8679809570312, + "w": 5.736000061035156, + "h": 6.49200439453125, + "fontSize": 12.0 +} + + Font sizes detected: [12.0] + +--- Step 5: Content-Specific Validation --- + + Subset font PDF - fonts with isSubset=true: 1 + - ABCDEF+Arial (isSubset=true, subsetTag='ABCDEF') + [OK] Subset fonts detected correctly + +================================================================================ + FINAL REPORT +================================================================================ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + PDF: utf-8.pdf + Document ID: 6764e35e-3594-4b50-a378-3ea122a54d9d + Upload Status: 201 + Document Font Count: 2 + Page Font Count: 2 + Glyph Count: 312 + Font Sizes: [1.0, 12.0, 18.0] + Issues: 6 + >> Glyph #0: missing coordinate 'right' + >> Glyph #0: missing coordinate 'bottom' + >> Glyph #1: missing coordinate 'right' + >> Glyph #1: missing coordinate 'bottom' + >> Glyph #2: missing coordinate 'right' + >> Glyph #2: missing coordinate 'bottom' + Errors: 0 + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + PDF: vertical_text.pdf + Document ID: d28fd4e4-a059-4a9b-bf15-b6cf25caf280 + Upload Status: 201 + Document Font Count: 1 + Page Font Count: 1 + Glyph Count: 19 + Font Sizes: [1.0, 12.0] + Issues: 6 + >> Glyph #0: missing coordinate 'right' + >> Glyph #0: missing coordinate 'bottom' + >> Glyph #1: missing coordinate 'right' + >> Glyph #1: missing coordinate 'bottom' + >> Glyph #2: missing coordinate 'right' + >> Glyph #2: missing coordinate 'bottom' + Errors: 0 + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + PDF: subset_font.pdf + Document ID: cf3f2218-6ee7-4e34-973d-6e1b61c13cf8 + Upload Status: 201 + Document Font Count: 1 + Page Font Count: 1 + Glyph Count: 11 + Font Sizes: [12.0] + Issues: 6 + >> Glyph #0: missing coordinate 'right' + >> Glyph #0: missing coordinate 'bottom' + >> Glyph #1: missing coordinate 'right' + >> Glyph #1: missing coordinate 'bottom' + >> Glyph #2: missing coordinate 'right' + >> Glyph #2: missing coordinate 'bottom' + Errors: 0 + +================================================================================ + SUMMARY ANSWERS +================================================================================ + + 1. Are fonts being extracted correctly? + YES - 4 total fonts across 3 PDFs + + 2. Are page fonts and document fonts consistent? + YES + + 3. Are font sizes being extracted correctly? + YES + + 4. Are vertical/subset fonts detected correctly? + Vertical: YES + Subset: YES + + 5. Are there any metadata inaccuracies? + YES - 18 issues and 0 errors found + - [utf-8.pdf] Glyph #0: missing coordinate 'right' + - [utf-8.pdf] Glyph #0: missing coordinate 'bottom' + - [utf-8.pdf] Glyph #1: missing coordinate 'right' + - [utf-8.pdf] Glyph #1: missing coordinate 'bottom' + - [utf-8.pdf] Glyph #2: missing coordinate 'right' + - [utf-8.pdf] Glyph #2: missing coordinate 'bottom' + - [vertical_text.pdf] Glyph #0: missing coordinate 'right' + - [vertical_text.pdf] Glyph #0: missing coordinate 'bottom' + - [vertical_text.pdf] Glyph #1: missing coordinate 'right' + - [vertical_text.pdf] Glyph #1: missing coordinate 'bottom' + - [vertical_text.pdf] Glyph #2: missing coordinate 'right' + - [vertical_text.pdf] Glyph #2: missing coordinate 'bottom' + - [subset_font.pdf] Glyph #0: missing coordinate 'right' + - [subset_font.pdf] Glyph #0: missing coordinate 'bottom' + - [subset_font.pdf] Glyph #1: missing coordinate 'right' + - [subset_font.pdf] Glyph #1: missing coordinate 'bottom' + - [subset_font.pdf] Glyph #2: missing coordinate 'right' + - [subset_font.pdf] Glyph #2: missing coordinate 'bottom' + +================================================================================ + VALIDATION COMPLETE: 18 issues, 0 errors +================================================================================ diff --git a/wasm/bindings/pdf_engine_facade.cpp b/wasm/bindings/pdf_engine_facade.cpp index e0613d3..02cec89 100644 --- a/wasm/bindings/pdf_engine_facade.cpp +++ b/wasm/bindings/pdf_engine_facade.cpp @@ -90,6 +90,11 @@ std::expected, pdfengine::EngineError> WasmMockPage::ex return std::vector(); } +std::expected WasmMockPage::getGlyphWidth(const std::string& fontName, uint32_t charcode, double fontSize) const { + (void)fontName; (void)charcode; + return 6.0 * (fontSize / 12.0); +} + std::expected, pdfengine::EngineError> WasmMockPage::getFonts() const { // Simulate a non-embedded Helvetica font being substituted with Liberation Sans. // This is the canonical Phase 1 font substitution scenario. diff --git a/wasm/bindings/pdf_engine_facade.hpp b/wasm/bindings/pdf_engine_facade.hpp index 1c9a080..2798996 100644 --- a/wasm/bindings/pdf_engine_facade.hpp +++ b/wasm/bindings/pdf_engine_facade.hpp @@ -21,6 +21,7 @@ public: [[nodiscard]] std::expected, pdfengine::EngineError> extractTextWithBounds() const override; [[nodiscard]] std::expected, pdfengine::EngineError> getFonts() const override; [[nodiscard]] std::expected, pdfengine::EngineError> extractAnnotationsText() const override; + [[nodiscard]] std::expected getGlyphWidth(const std::string& fontName, uint32_t charcode, double fontSize) const override; [[nodiscard]] pdfengine::DevicePoint pageToDevice(const pdfengine::Point2D& pagePoint, int deviceWidth, int deviceHeight, int rotate = 0) const noexcept override; [[nodiscard]] pdfengine::Point2D deviceToPage(const pdfengine::DevicePoint& devicePoint, int deviceWidth, int deviceHeight, int rotate = 0) const noexcept override; From ce5ad5bb166b13a76eac9ba94e1eb4b35a5eb496 Mon Sep 17 00:00:00 2001 From: saqib mir Date: Fri, 5 Jun 2026 10:27:39 +0530 Subject: [PATCH 2/7] fix --- engine/src/parser/pdfium_document.cpp | 50 +++++++++++++++++---------- engine/src/parser/pdfium_document.hpp | 5 ++- gateway/app/routers/documents.py | 6 ++-- gateway/app/routers/edits.py | 20 +++++------ gateway/app/routers/render.py | 20 +++++------ 5 files changed, 58 insertions(+), 43 deletions(-) diff --git a/engine/src/parser/pdfium_document.cpp b/engine/src/parser/pdfium_document.cpp index e2c9213..350694a 100644 --- a/engine/src/parser/pdfium_document.cpp +++ b/engine/src/parser/pdfium_document.cpp @@ -1,5 +1,4 @@ #include "parser/pdfium_document.hpp" - #ifdef PDFENGINE_WITH_PDFIUM #include #include @@ -996,11 +995,26 @@ std::expected, EngineError> PdfiumDocument::getPage(int if (pageIndex < 0 || pageIndex >= pageCount()) { return std::unexpected(EngineError::PageOutOfBounds); } + + { + std::lock_guard lock(pageCacheMutex_); + auto it = pageCache_.find(pageIndex); + if (it != pageCache_.end()) { + return it->second; + } + } + FPDF_PAGE page = FPDF_LoadPage(doc_, pageIndex); if (!page) { return std::unexpected(EngineError::Unknown); } - return std::make_shared(page, pageIndex); + + auto pageObj = std::make_shared(page, pageIndex); + { + std::lock_guard lock(pageCacheMutex_); + pageCache_[pageIndex] = pageObj; + } + return pageObj; #else (void)pageIndex; return std::unexpected(EngineError::Unknown); @@ -1167,7 +1181,7 @@ std::expected PdfiumDocument::applyEdits(const std::string& e return std::unexpected(EngineError::Unknown); } - invalidateFontCache(); + invalidateCaches(); return {}; #else (void)editsJson; @@ -1440,12 +1454,6 @@ std::expected, EngineError> PdfiumDocument::getFonts(int s return std::vector(); } - // Security Safeguard: cap maximum scan range to 1000 pages to prevent memory/CPU exhaustion - int scanCount = endPage - startPage + 1; - if (scanCount > 1000) { - spdlog::warn("Requested scan range ({} pages) exceeds limit. Capping scan to 1000 pages.", scanCount); - endPage = startPage + 999; - } // Return full document-level cache if available and full range is requested if (startPage == 0 && endPage == total - 1 && hasCachedFonts_) { @@ -1454,15 +1462,13 @@ std::expected, EngineError> PdfiumDocument::getFonts(int s std::vector aggregated; for (int i = startPage; i <= endPage; ++i) { - FPDF_PAGE page = FPDF_LoadPage(doc_, i); - if (!page) { + auto pageRes = const_cast(this)->getPage(i); + if (!pageRes) { spdlog::error("Failed to load page index {} for font diagnostics", i); continue; } - // Stack-allocated wrapper ensures FPDF handles are closed properly upon destruction - PdfiumPage tempPage(page, i); - auto pageFontsRes = tempPage.getFonts(); + auto pageFontsRes = pageRes.value()->getFonts(); if (pageFontsRes) { for (const auto& f : *pageFontsRes) { auto it = std::find_if(aggregated.begin(), aggregated.end(), [&](const FontInfo& existing) { @@ -1503,11 +1509,17 @@ std::expected, EngineError> PdfiumDocument::getFonts(int s #endif } -void PdfiumDocument::invalidateFontCache() { - std::lock_guard lock(fontsMutex_); - cachedFonts_.clear(); - hasCachedFonts_ = false; - spdlog::info("Document font cache has been invalidated."); +void PdfiumDocument::invalidateCaches() { + { + std::lock_guard lock(fontsMutex_); + cachedFonts_.clear(); + hasCachedFonts_ = false; + } + { + std::lock_guard lock(pageCacheMutex_); + pageCache_.clear(); + } + spdlog::info("Document caches have been invalidated."); } } diff --git a/engine/src/parser/pdfium_document.hpp b/engine/src/parser/pdfium_document.hpp index 741de3a..abe0d53 100644 --- a/engine/src/parser/pdfium_document.hpp +++ b/engine/src/parser/pdfium_document.hpp @@ -77,7 +77,7 @@ public: std::expected, EngineError> getPage(int pageIndex) override; std::expected, EngineError> getFonts(int startPage = 0, int endPage = -1) const override; - void invalidateFontCache(); + void invalidateCaches(); std::expected applyEdits(const std::string& editsJson) override; std::expected, EngineError> saveIncremental() const override; @@ -89,6 +89,9 @@ private: mutable std::vector cachedFonts_; mutable bool hasCachedFonts_ = false; mutable std::mutex fontsMutex_; + + mutable std::unordered_map> pageCache_; + mutable std::mutex pageCacheMutex_; }; // Exposed for testing diff --git a/gateway/app/routers/documents.py b/gateway/app/routers/documents.py index 7019b9e..c319c8e 100644 --- a/gateway/app/routers/documents.py +++ b/gateway/app/routers/documents.py @@ -1,5 +1,5 @@ -from typing import List -from fastapi import APIRouter, HTTPException, status, File, UploadFile +from typing import List, Annotated +from fastapi import APIRouter, HTTPException, status, File, UploadFile, Query from pydantic import BaseModel from app.services import engine @@ -159,7 +159,7 @@ class FontInfoResponse(BaseModel): capHeight: float @router.get("/{document_id}/fonts", response_model=List[FontInfoResponse]) -def get_document_fonts(document_id: str, start_page: int = 0, end_page: int = -1) -> List[FontInfoResponse]: +def get_document_fonts(document_id: str, start_page: Annotated[int, Query(ge=0)] = 0, end_page: Annotated[int, Query(ge=-1)] = -1) -> List[FontInfoResponse]: if not engine.is_available(): raise HTTPException( status_code=status.HTTP_501_NOT_IMPLEMENTED, diff --git a/gateway/app/routers/edits.py b/gateway/app/routers/edits.py index 5bfb916..5bd5322 100644 --- a/gateway/app/routers/edits.py +++ b/gateway/app/routers/edits.py @@ -18,7 +18,7 @@ class TextOverlayData(BaseModel): y: float width: float height: float - fontSize: float + fontSize: float = Field(..., gt=0) fontFamily: str color: str @@ -59,7 +59,7 @@ class FreeTextData(BaseModel): width: float height: float text: str - fontSize: float = 12.0 + fontSize: float = Field(12.0, gt=0) color: str = "#000000" class StickyNoteData(BaseModel): @@ -83,49 +83,49 @@ class PageRotationData(BaseModel): class TextOverlayOperation(BaseModel): id: str type: Literal["text_overlay"] - pageIndex: int + pageIndex: int = Field(..., ge=0) data: TextOverlayData class RedactionOperation(BaseModel): id: str type: Literal["redaction"] - pageIndex: int + pageIndex: int = Field(..., ge=0) data: RedactionData class ImageOverlayOperation(BaseModel): id: str type: Literal["image_overlay"] - pageIndex: int + pageIndex: int = Field(..., ge=0) data: ImageOverlayData class HighlightOperation(BaseModel): id: str type: Literal["highlight"] - pageIndex: int + pageIndex: int = Field(..., ge=0) data: HighlightData class FreeTextOperation(BaseModel): id: str type: Literal["free_text"] - pageIndex: int + pageIndex: int = Field(..., ge=0) data: FreeTextData class CommentOperation(BaseModel): id: str type: Literal["comment"] - pageIndex: int + pageIndex: int = Field(..., ge=0) data: StickyNoteData class FreehandOperation(BaseModel): id: str type: Literal["freehand"] - pageIndex: int + pageIndex: int = Field(..., ge=0) data: FreehandData class PageRotationOperation(BaseModel): id: str type: Literal["page_rotation"] - pageIndex: int + pageIndex: int = Field(..., ge=0) data: PageRotationData EditOperation = Annotated[ diff --git a/gateway/app/routers/render.py b/gateway/app/routers/render.py index 6472800..d6a0495 100644 --- a/gateway/app/routers/render.py +++ b/gateway/app/routers/render.py @@ -1,5 +1,5 @@ -from typing import List -from fastapi import APIRouter, HTTPException, status, Response +from typing import List, Annotated +from fastapi import APIRouter, HTTPException, status, Response, Path, Query from app.services import engine from app.services.store import document_store @@ -9,7 +9,7 @@ router = APIRouter(prefix="/documents/{document_id}/pages", tags=["render"]) compat_router = APIRouter(tags=["render"]) @router.get("/{page_index}/render") -def render_page(document_id: str, page_index: int, dpi: int = 96) -> Response: +def render_page(document_id: str, page_index: Annotated[int, Path(ge=0)], dpi: int = 96) -> Response: if not engine.is_available(): raise HTTPException( status_code=status.HTTP_501_NOT_IMPLEMENTED, @@ -31,7 +31,7 @@ def render_page(document_id: str, page_index: int, dpi: int = 96) -> Response: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) @router.get("/{page_index}/text") -def extract_page_text(document_id: str, page_index: int): +def extract_page_text(document_id: str, page_index: Annotated[int, Path(ge=0)]): if not engine.is_available(): raise HTTPException( status_code=status.HTTP_501_NOT_IMPLEMENTED, @@ -57,12 +57,12 @@ def extract_page_text(document_id: str, page_index: int): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) @compat_router.get("/render/{document_id}") -def render_page_compat(document_id: str, page: int = 0, zoom: float = 1.0, rotation: int = 0) -> Response: +def render_page_compat(document_id: str, page: Annotated[int, Query(ge=0)] = 0, zoom: float = 1.0, rotation: int = 0) -> Response: dpi = int(96 * zoom) return render_page(document_id, page, dpi) @router.get("/{page_index}") -def get_page_info(document_id: str, page_index: int): +def get_page_info(document_id: str, page_index: Annotated[int, Path(ge=0)]): if not engine.is_available(): raise HTTPException( status_code=status.HTTP_501_NOT_IMPLEMENTED, @@ -83,7 +83,7 @@ def get_page_info(document_id: str, page_index: int): raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) @router.get("/{page_index}/transform/page-to-device") -def transform_page_to_device(document_id: str, page_index: int, x: float, y: float, device_width: int, device_height: int, rotate: int = 0): +def transform_page_to_device(document_id: str, page_index: Annotated[int, Path(ge=0)], x: float, y: float, device_width: int, device_height: int, rotate: int = 0): if not engine.is_available(): raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail="Engine unavailable") @@ -103,7 +103,7 @@ def transform_page_to_device(document_id: str, page_index: int, x: float, y: flo raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) @router.get("/{page_index}/transform/device-to-page") -def transform_device_to_page(document_id: str, page_index: int, x: int, y: int, device_width: int, device_height: int, rotate: int = 0): +def transform_device_to_page(document_id: str, page_index: Annotated[int, Path(ge=0)], x: int, y: int, device_width: int, device_height: int, rotate: int = 0): if not engine.is_available(): raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail="Engine unavailable") @@ -122,7 +122,7 @@ def transform_device_to_page(document_id: str, page_index: int, x: int, y: int, except Exception as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) @router.get("/{page_index}/fonts", response_model=List[FontInfoResponse]) -def get_page_fonts(document_id: str, page_index: int) -> List[FontInfoResponse]: +def get_page_fonts(document_id: str, page_index: Annotated[int, Path(ge=0)]) -> List[FontInfoResponse]: if not engine.is_available(): raise HTTPException( status_code=status.HTTP_501_NOT_IMPLEMENTED, @@ -168,7 +168,7 @@ def get_page_fonts(document_id: str, page_index: int) -> List[FontInfoResponse]: @router.get("/{page_index}/fonts/glyph-width") -def get_page_glyph_width(document_id: str, page_index: int, font_name: str, charcode: int, font_size: float = 12.0): +def get_page_glyph_width(document_id: str, page_index: Annotated[int, Path(ge=0)], font_name: str, charcode: int, font_size: Annotated[float, Query(gt=0)] = 12.0): if not engine.is_available(): raise HTTPException( status_code=status.HTTP_501_NOT_IMPLEMENTED, From bfa8d0eddf3a4b58429f5c6a37dd2f1cea1202f4 Mon Sep 17 00:00:00 2001 From: saqib mir Date: Sat, 6 Jun 2026 11:21:31 +0530 Subject: [PATCH 3/7] fix the issue --- engine/CMakeLists.txt | 3 + engine/include/pdfengine/pdf_document.hpp | 3 + engine/src/fonts/cache/glyph_cache.cpp | 12 ++-- engine/src/fonts/cache/glyph_cache.hpp | 6 +- engine/src/fonts/face/font_face.cpp | 42 ++++++------ engine/src/fonts/face/font_face.hpp | 7 +- engine/src/fonts/face/free_type_manager.cpp | 28 ++++++++ engine/src/fonts/face/free_type_manager.hpp | 27 ++++++++ engine/src/fonts/loader/font_resolver.cpp | 64 +++++++++++++++++++ engine/src/fonts/loader/font_resolver.hpp | 24 +++++++ engine/src/fonts/pdf_fonts/types/cid_font.cpp | 3 +- engine/src/fonts/shaping/hb_shaper.cpp | 21 +++++- engine/src/parser/pdfium_document.cpp | 48 ++++++++++++++ engine/src/parser/pdfium_document.hpp | 1 + 14 files changed, 258 insertions(+), 31 deletions(-) create mode 100644 engine/src/fonts/face/free_type_manager.cpp create mode 100644 engine/src/fonts/face/free_type_manager.hpp create mode 100644 engine/src/fonts/loader/font_resolver.cpp create mode 100644 engine/src/fonts/loader/font_resolver.hpp diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 2ab2ea6..a80f8cf 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -17,6 +17,9 @@ add_library(pdfengine STATIC src/parser/pdfium_loader.cpp src/parser/pdfium_document.cpp src/fonts/face/font_face.cpp + src/fonts/face/free_type_manager.cpp + src/fonts/loader/font_resolver.cpp + src/fonts/pdf_fonts/font_loader.cpp src/fonts/shaping/hb_shaper.cpp src/fonts/cache/glyph_bitmap.cpp src/fonts/cache/glyph_cache.cpp diff --git a/engine/include/pdfengine/pdf_document.hpp b/engine/include/pdfengine/pdf_document.hpp index 48953fe..5d53e3b 100644 --- a/engine/include/pdfengine/pdf_document.hpp +++ b/engine/include/pdfengine/pdf_document.hpp @@ -118,6 +118,9 @@ public: [[nodiscard]] virtual std::expected, EngineError> getFonts(int startPage = 0, int endPage = -1) const = 0; + [[nodiscard]] virtual std::expected, EngineError> + getFontData(const std::string& internalFontId) const = 0; + virtual std::expected applyEdits(const std::string& editsJson) = 0; [[nodiscard]] virtual std::expected, EngineError> diff --git a/engine/src/fonts/cache/glyph_cache.cpp b/engine/src/fonts/cache/glyph_cache.cpp index a865940..5856e1f 100644 --- a/engine/src/fonts/cache/glyph_cache.cpp +++ b/engine/src/fonts/cache/glyph_cache.cpp @@ -13,12 +13,12 @@ GlyphCache::GlyphCache(std::size_t capacity) GlyphCache::~GlyphCache() = default; std::optional GlyphCache::get(const FontFace& fontFace, unsigned int glyphIndex, unsigned int fontSize) { - FT_Face face = fontFace.getFace(); - if (!face) { + uint64_t fontId = fontFace.getId(); + if (fontId == 0) { return std::nullopt; } - GlyphCacheKey key{face, glyphIndex, fontSize}; + GlyphCacheKey key{fontId, glyphIndex, fontSize}; std::size_t shard_idx = getShardIndex(key); auto& shard = *shards_[shard_idx]; @@ -37,12 +37,12 @@ std::optional GlyphCache::get(const FontFace& fontFace, unsigned in } void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsigned int fontSize, const GlyphBitmap& bitmap) { - FT_Face face = fontFace.getFace(); - if (!face) { + uint64_t fontId = fontFace.getId(); + if (fontId == 0) { return; } - GlyphCacheKey key{face, glyphIndex, fontSize}; + GlyphCacheKey key{fontId, glyphIndex, fontSize}; std::size_t shard_idx = getShardIndex(key); auto& shard = *shards_[shard_idx]; diff --git a/engine/src/fonts/cache/glyph_cache.hpp b/engine/src/fonts/cache/glyph_cache.hpp index 3f2c258..3c8b619 100644 --- a/engine/src/fonts/cache/glyph_cache.hpp +++ b/engine/src/fonts/cache/glyph_cache.hpp @@ -12,12 +12,12 @@ namespace pdfengine::fonts { struct GlyphCacheKey { - FT_Face face; + uint64_t fontId; unsigned int glyphIndex; unsigned int fontSize; bool operator==(const GlyphCacheKey& other) const { - return face == other.face && + return fontId == other.fontId && glyphIndex == other.glyphIndex && fontSize == other.fontSize; } @@ -25,7 +25,7 @@ struct GlyphCacheKey { struct GlyphCacheKeyHash { std::size_t operator()(const GlyphCacheKey& key) const { - std::size_t h1 = std::hash{}(static_cast(key.face)); + std::size_t h1 = std::hash{}(key.fontId); std::size_t h2 = std::hash{}(key.glyphIndex); std::size_t h3 = std::hash{}(key.fontSize); // Combine hashes using standard boost hash_combine algorithm diff --git a/engine/src/fonts/face/font_face.cpp b/engine/src/fonts/face/font_face.cpp index 296159f..7245894 100644 --- a/engine/src/fonts/face/font_face.cpp +++ b/engine/src/fonts/face/font_face.cpp @@ -1,16 +1,17 @@ #include "fonts/face/font_face.hpp" +#include "fonts/face/free_type_manager.hpp" #include +#include namespace pdfengine::fonts { -FontFace::FontFace() - : ft_library_(nullptr), - face_(nullptr) { +static std::atomic g_font_id_counter{1}; - if (FT_Init_FreeType(&ft_library_)) { - std::cerr << "Failed to initialize FreeType\n"; - } +FontFace::FontFace() + : font_id_(g_font_id_counter.fetch_add(1, std::memory_order_relaxed)), + face_(nullptr), + mutex_(std::make_unique()) { } FontFace::~FontFace() { @@ -18,17 +19,14 @@ FontFace::~FontFace() { if (face_) { FT_Done_Face(face_); } - - if (ft_library_) { - FT_Done_FreeType(ft_library_); - } } FontFace::FontFace(FontFace&& other) noexcept - : ft_library_(other.ft_library_), + : font_id_(other.font_id_), face_(other.face_), + mutex_(std::move(other.mutex_)), font_data_(std::move(other.font_data_)) { - other.ft_library_ = nullptr; + other.font_id_ = 0; other.face_ = nullptr; } @@ -37,13 +35,11 @@ FontFace& FontFace::operator=(FontFace&& other) noexcept { if (face_) { FT_Done_Face(face_); } - if (ft_library_) { - FT_Done_FreeType(ft_library_); - } - ft_library_ = other.ft_library_; + font_id_ = other.font_id_; face_ = other.face_; + mutex_ = std::move(other.mutex_); font_data_ = std::move(other.font_data_); - other.ft_library_ = nullptr; + other.font_id_ = 0; other.face_ = nullptr; } return *this; @@ -58,7 +54,7 @@ bool FontFace::loadFromFile(const std::string& path) { font_data_.clear(); if (FT_New_Face( - ft_library_, + FreeTypeManager::instance().getLibrary(), path.c_str(), 0, &face_)) { @@ -91,7 +87,7 @@ bool FontFace::loadFromMemory(const std::vector& data) { font_data_ = data; if (FT_New_Memory_Face( - ft_library_, + FreeTypeManager::instance().getLibrary(), font_data_.data(), static_cast(font_data_.size()), 0, @@ -113,6 +109,14 @@ FT_Face FontFace::getFace() const { return face_; } +uint64_t FontFace::getId() const { + return font_id_; +} + +std::mutex& FontFace::getMutex() const { + return *mutex_; +} + std::optional FontFace::renderGlyph(unsigned int glyphIndex, unsigned int fontSize) { if (!face_) { return std::nullopt; diff --git a/engine/src/fonts/face/font_face.hpp b/engine/src/fonts/face/font_face.hpp index 025b611..303c7ac 100644 --- a/engine/src/fonts/face/font_face.hpp +++ b/engine/src/fonts/face/font_face.hpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include FT_FREETYPE_H @@ -26,13 +28,16 @@ public: bool loadFromMemory(const std::vector& data); FT_Face getFace() const; + uint64_t getId() const; + std::mutex& getMutex() const; // Renders a glyph by index and size, returning a GlyphBitmap on success. std::optional renderGlyph(unsigned int glyphIndex, unsigned int fontSize); private: - FT_Library ft_library_; + uint64_t font_id_; FT_Face face_; + std::unique_ptr mutex_; std::vector font_data_; // Keeps the loaded memory buffer alive for FT_Face }; diff --git a/engine/src/fonts/face/free_type_manager.cpp b/engine/src/fonts/face/free_type_manager.cpp new file mode 100644 index 0000000..59a70d4 --- /dev/null +++ b/engine/src/fonts/face/free_type_manager.cpp @@ -0,0 +1,28 @@ +#include "fonts/face/free_type_manager.hpp" +#include + +namespace pdfengine::fonts { + +FreeTypeManager& FreeTypeManager::instance() { + static FreeTypeManager instance; + return instance; +} + +FreeTypeManager::FreeTypeManager() : library_(nullptr) { + if (FT_Init_FreeType(&library_)) { + std::cerr << "Failed to initialize FreeType\n"; + } +} + +FreeTypeManager::~FreeTypeManager() { + if (library_) { + FT_Done_FreeType(library_); + library_ = nullptr; + } +} + +FT_Library FreeTypeManager::getLibrary() const { + return library_; +} + +} // namespace pdfengine::fonts diff --git a/engine/src/fonts/face/free_type_manager.hpp b/engine/src/fonts/face/free_type_manager.hpp new file mode 100644 index 0000000..1b04377 --- /dev/null +++ b/engine/src/fonts/face/free_type_manager.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include FT_FREETYPE_H + +namespace pdfengine::fonts { + +class FreeTypeManager { +public: + static FreeTypeManager& instance(); + + FT_Library getLibrary() const; + + // Delete copy/move constructors and assignment operators for singleton + FreeTypeManager(const FreeTypeManager&) = delete; + FreeTypeManager& operator=(const FreeTypeManager&) = delete; + FreeTypeManager(FreeTypeManager&&) = delete; + FreeTypeManager& operator=(FreeTypeManager&&) = delete; + +private: + FreeTypeManager(); + ~FreeTypeManager(); + + FT_Library library_; +}; + +} // namespace pdfengine::fonts diff --git a/engine/src/fonts/loader/font_resolver.cpp b/engine/src/fonts/loader/font_resolver.cpp new file mode 100644 index 0000000..a103b67 --- /dev/null +++ b/engine/src/fonts/loader/font_resolver.cpp @@ -0,0 +1,64 @@ +#include "fonts/loader/font_resolver.hpp" +#include "fonts/pdf_fonts/font_loader.hpp" +#include + +namespace pdfengine::fonts::loader { + +FontResolver::FontResolver(std::shared_ptr document) + : document_(std::move(document)) {} + +std::expected, std::string> FontResolver::resolveFont(const FontInfo& fontInfo) { + if (!document_) { + return std::unexpected("No document attached to FontResolver"); + } + + if (fontInfo.isEmbedded) { + auto dataRes = document_->getFontData(fontInfo.internalFontId); + if (!dataRes) { + spdlog::error("Failed to extract embedded font data for ID: {}", fontInfo.internalFontId); + return std::unexpected("Failed to extract embedded font data"); + } + + const auto& bytes = dataRes.value(); + + // Route to the appropriate FontLoader method based on FontInfo type + if (fontInfo.type == "TrueType" || fontInfo.type == "Type1") { + // Treat embedded Type1/TrueType uniformly through the TrueType loader for now + // since FontLoader::loadTrueTypeFromMemory delegates to FreeType which handles both. + // A more strictly compliant PDF parser would differentiate, but FreeType is unified. + auto font = pdf_fonts::FontLoader::loadTrueTypeFromMemory(fontInfo.normalizedFamily, bytes); + if (font) { + return font; + } + } else if (fontInfo.type == "CIDFontType0" || fontInfo.type == "CIDFontType2") { + pdf_fonts::FontType subtype = (fontInfo.type == "CIDFontType0") + ? pdf_fonts::FontType::CIDFontType0 + : pdf_fonts::FontType::CIDFontType2; + + auto font = pdf_fonts::FontLoader::loadCIDFontFromMemory(fontInfo.normalizedFamily, subtype, bytes); + if (font) { + return font; + } + } + + return std::unexpected("Failed to parse extracted font data"); + } else { + // Handle System Fallback + spdlog::info("Resolving system fallback font for: {}", fontInfo.fontName); + + if (fontInfo.type == "CIDFontType0" || fontInfo.type == "CIDFontType2") { + pdf_fonts::FontType subtype = (fontInfo.type == "CIDFontType0") + ? pdf_fonts::FontType::CIDFontType0 + : pdf_fonts::FontType::CIDFontType2; + auto font = pdf_fonts::FontLoader::loadCIDFontSystemFallback(fontInfo.normalizedFamily, subtype); + if (font) return font; + } else { + auto font = pdf_fonts::FontLoader::loadType1SystemFallback(fontInfo.normalizedFamily); + if (font) return font; + } + + return std::unexpected("Failed to load system fallback font"); + } +} + +} // namespace pdfengine::fonts::loader diff --git a/engine/src/fonts/loader/font_resolver.hpp b/engine/src/fonts/loader/font_resolver.hpp new file mode 100644 index 0000000..30f9e21 --- /dev/null +++ b/engine/src/fonts/loader/font_resolver.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include "fonts/pdf_fonts/font.hpp" +#include +#include +#include +#include + +namespace pdfengine::fonts::loader { + +class FontResolver { +public: + explicit FontResolver(std::shared_ptr document); + + // Resolves a FontInfo object into a fully loaded Font ready for shaping. + // If the font is embedded, it extracts the raw bytes from the PdfDocument. + // If it's a system fallback, it uses FontLoader to load it from the OS. + std::expected, std::string> resolveFont(const FontInfo& fontInfo); + +private: + std::shared_ptr document_; +}; + +} // namespace pdfengine::fonts::loader diff --git a/engine/src/fonts/pdf_fonts/types/cid_font.cpp b/engine/src/fonts/pdf_fonts/types/cid_font.cpp index b27b975..ab1bb6a 100644 --- a/engine/src/fonts/pdf_fonts/types/cid_font.cpp +++ b/engine/src/fonts/pdf_fonts/types/cid_font.cpp @@ -126,7 +126,8 @@ uint32_t CIDFont::decodeToUnicode(uint32_t charCode) const { if (descriptor_) { std::string fontName = descriptor_->getFontName(); std::string lowerName = fontName; - std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), ::tolower); + std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); std::string registry = "None"; if (lowerName.find("simsun") != std::string::npos || diff --git a/engine/src/fonts/shaping/hb_shaper.cpp b/engine/src/fonts/shaping/hb_shaper.cpp index 9f5d3ce..fb282b1 100644 --- a/engine/src/fonts/shaping/hb_shaper.cpp +++ b/engine/src/fonts/shaping/hb_shaper.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace pdfengine::fonts { @@ -21,6 +22,9 @@ std::vector HbShaper::shapeRun( return result; } + // Acquire lock to prevent concurrent mutation of FT_Face's active pixel size + std::lock_guard lock(font.getMutex()); + // Set the pixel size on the FreeType face before shaping. // This ensures HarfBuzz measures everything using the requested font size context. if (FT_Set_Pixel_Sizes(ftFace, 0, fontSize)) { @@ -53,8 +57,23 @@ std::vector HbShaper::shapeRun( hb_buffer_set_direction(hbBuffer, HB_DIRECTION_TTB); } + // Enable professional typography features (Kerning and Ligatures) + hb_feature_t features[2]; + + // Enable kerning + features[0].tag = HB_TAG('k', 'e', 'r', 'n'); + features[0].value = 1; + features[0].start = 0; + features[0].end = static_cast(-1); + + // Enable standard ligatures + features[1].tag = HB_TAG('l', 'i', 'g', 'a'); + features[1].value = 1; + features[1].start = 0; + features[1].end = static_cast(-1); + // Shape the text inside the buffer using the font. - hb_shape(hbFont, hbBuffer, nullptr, 0); + hb_shape(hbFont, hbBuffer, features, 2); // Retrieve the results. unsigned int glyphCount = 0; diff --git a/engine/src/parser/pdfium_document.cpp b/engine/src/parser/pdfium_document.cpp index 350694a..72024bd 100644 --- a/engine/src/parser/pdfium_document.cpp +++ b/engine/src/parser/pdfium_document.cpp @@ -1509,6 +1509,54 @@ std::expected, EngineError> PdfiumDocument::getFonts(int s #endif } +std::expected, EngineError> PdfiumDocument::getFontData(const std::string& internalFontId) const { +#ifdef PDFENGINE_WITH_PDFIUM + ensure_pdfium_initialized(); + if (!doc_) { + return std::unexpected(EngineError::Unknown); + } + + int numPages = FPDF_GetPageCount(doc_); + for (int i = 0; i < numPages; ++i) { + FPDF_PAGE page = FPDF_LoadPage(doc_, i); + if (!page) continue; + + int objectCount = FPDFPage_CountObjects(page); + for (int j = 0; j < objectCount; ++j) { + FPDF_PAGEOBJECT obj = FPDFPage_GetObject(page, j); + if (!obj || FPDFPageObj_GetType(obj) != FPDF_PAGEOBJ_TEXT) continue; + + FPDF_FONT font = FPDFTextObj_GetFont(obj); + if (!font) continue; + + unsigned long nameLen = FPDFFont_GetBaseFontName(font, nullptr, 0); + if (nameLen > 0) { + std::vector nameBuf(nameLen); + if (FPDFFont_GetBaseFontName(font, nameBuf.data(), nameLen) > 0) { + std::string fontName(nameBuf.data()); + // internalFontId is constructed using fontName as the prefix + if (internalFontId.find(fontName) == 0) { + size_t buflen = FPDFFont_GetFontData(font, nullptr, 0); + if (buflen > 0) { + std::vector buffer(buflen); + if (FPDFFont_GetFontData(font, buffer.data(), buflen) > 0) { + FPDF_ClosePage(page); + return buffer; + } + } + } + } + } + } + FPDF_ClosePage(page); + } + return std::unexpected(EngineError::FileNotFound); +#else + (void)internalFontId; + return std::unexpected(EngineError::Unknown); +#endif +} + void PdfiumDocument::invalidateCaches() { { std::lock_guard lock(fontsMutex_); diff --git a/engine/src/parser/pdfium_document.hpp b/engine/src/parser/pdfium_document.hpp index abe0d53..68b020a 100644 --- a/engine/src/parser/pdfium_document.hpp +++ b/engine/src/parser/pdfium_document.hpp @@ -77,6 +77,7 @@ public: std::expected, EngineError> getPage(int pageIndex) override; std::expected, EngineError> getFonts(int startPage = 0, int endPage = -1) const override; + std::expected, EngineError> getFontData(const std::string& internalFontId) const override; void invalidateCaches(); std::expected applyEdits(const std::string& editsJson) override; From 31243722d362d285cb0e71f8753a60d7562047ad Mon Sep 17 00:00:00 2001 From: azeeee05 Date: Mon, 8 Jun 2026 11:22:15 +0530 Subject: [PATCH 4/7] search fuunction done --- frontend/src/App.tsx | 38 ++++++-- frontend/src/components/Toolbar.tsx | 8 ++ frontend/src/index.css | 6 ++ frontend/src/lib/gatewayService.ts | 40 +++++++- frontend/src/lib/wasmLoader.ts | 3 +- frontend/src/viewer/AnnotationLayer.tsx | 20 +++- frontend/src/viewer/OverlayLayer.tsx | 94 +++++++++++++++++-- frontend/src/viewer/PDFViewer.tsx | 19 +++- frontend/src/viewer/SearchOverlayLayer.tsx | 67 +++++++------- gateway/app/routers/documents.py | 102 ++++++++++++++++++++- gateway/app/services/store.py | 13 ++- scripts/build_cpp.ps1 | 19 ++-- 12 files changed, 365 insertions(+), 64 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6282708..f78705e 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -5,7 +5,7 @@ import { PDFViewer } from './viewer/PDFViewer'; import type { PDFViewerRef } from './viewer/PDFViewer'; import type { Annotation } from './viewer/AnnotationLayer'; import { gatewayService } from './lib/gatewayService'; -import type { DocumentInfo } from './lib/gatewayService'; +import type { DocumentInfo, SearchResult } from './lib/gatewayService'; import { wasmLoader } from './lib/wasmLoader'; import './App.css'; @@ -29,16 +29,30 @@ function App() { // Search State const [searchQuery, setSearchQuery] = useState(''); + const [searchResults, setSearchResults] = useState([]); const [searchResultCount, setSearchResultCount] = useState(0); const [searchCurrentMatch, setSearchCurrentMatch] = useState(0); - const handleSearch = (query: string) => { + const handleSearch = async (query: string) => { setSearchQuery(query); - // Mock search results for Phase 0 - if (query) { - setSearchResultCount(5); + if (!query || !selectedDocId) { + setSearchResults([]); + setSearchResultCount(0); setSearchCurrentMatch(0); - } else { + return; + } + + try { + const results = await gatewayService.searchDocument(selectedDocId, query); + setSearchResults(results); + setSearchResultCount(results.length); + setSearchCurrentMatch(0); + if (results.length > 0) { + viewerRef.current?.scrollToPage(results[0].pageIndex); + } + } catch (err) { + console.error("Search failed", err); + setSearchResults([]); setSearchResultCount(0); setSearchCurrentMatch(0); } @@ -46,13 +60,17 @@ function App() { const handleSearchNext = () => { if (searchResultCount > 0) { - setSearchCurrentMatch((prev) => (prev + 1) % searchResultCount); + const nextMatch = (searchCurrentMatch + 1) % searchResultCount; + setSearchCurrentMatch(nextMatch); + viewerRef.current?.scrollToPage(searchResults[nextMatch].pageIndex); } }; const handleSearchPrev = () => { if (searchResultCount > 0) { - setSearchCurrentMatch((prev) => (prev - 1 + searchResultCount) % searchResultCount); + const prevMatch = (searchCurrentMatch - 1 + searchResultCount) % searchResultCount; + setSearchCurrentMatch(prevMatch); + viewerRef.current?.scrollToPage(searchResults[prevMatch].pageIndex); } }; @@ -198,11 +216,15 @@ function App() { ref={viewerRef} documentId={activeDoc.id} totalPages={activeDoc.totalPages} + pageWidth={activeDoc.pageWidth} + pageHeight={activeDoc.pageHeight} zoom={zoom} rotation={rotation} activeTool={activeTool} annotations={annotations} searchQuery={searchQuery} + searchResults={searchResults} + searchCurrentMatch={searchCurrentMatch} onAnnotationAdded={handleAnnotationAdded} onPageVisible={setCurrentPage} /> diff --git a/frontend/src/components/Toolbar.tsx b/frontend/src/components/Toolbar.tsx index 943d30f..ceca69f 100644 --- a/frontend/src/components/Toolbar.tsx +++ b/frontend/src/components/Toolbar.tsx @@ -160,6 +160,14 @@ export const Toolbar: React.FC = ({ Highlight + +