diff --git a/engine/src/fonts/cache/glyph_cache.cpp b/engine/src/fonts/cache/glyph_cache.cpp index 8f5ce26..be1f623 100644 --- a/engine/src/fonts/cache/glyph_cache.cpp +++ b/engine/src/fonts/cache/glyph_cache.cpp @@ -3,7 +3,11 @@ namespace pdfengine::fonts { GlyphCache::GlyphCache(std::size_t capacity) - : capacity_(capacity) {} + : capacity_(capacity) { + for (std::size_t i = 0; i < NUM_SHARDS; ++i) { + shards_.push_back(std::make_unique()); + } +} GlyphCache::~GlyphCache() = default; @@ -14,17 +18,19 @@ std::optional GlyphCache::get(const FontFace& fontFace, unsigned in } GlyphCacheKey key{face, glyphIndex, fontSize}; + std::size_t shard_idx = getShardIndex(key); + auto& shard = *shards_[shard_idx]; - std::lock_guard lock(mutex_); - auto it = cache_map_.find(key); - if (it == cache_map_.end()) { - misses_++; + std::lock_guard lock(shard.mutex_); + auto it = shard.cache_map_.find(key); + if (it == shard.cache_map_.end()) { + shard.misses_++; return std::nullopt; // Cache miss } - hits_++; + shard.hits_++; // Cache hit: move the referenced key to the front of the LRU list - lru_list_.splice(lru_list_.begin(), lru_list_, it->second.second); + shard.lru_list_.splice(shard.lru_list_.begin(), shard.lru_list_, it->second.second); return it->second.first; } @@ -36,61 +42,81 @@ void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsig } GlyphCacheKey key{face, glyphIndex, fontSize}; + std::size_t shard_idx = getShardIndex(key); + auto& shard = *shards_[shard_idx]; - std::lock_guard lock(mutex_); - auto it = cache_map_.find(key); - if (it != cache_map_.end()) { + std::lock_guard lock(shard.mutex_); + auto it = shard.cache_map_.find(key); + if (it != shard.cache_map_.end()) { // Element already exists: update bitmap and move it to the front it->second.first = bitmap; - lru_list_.splice(lru_list_.begin(), lru_list_, it->second.second); + shard.lru_list_.splice(shard.lru_list_.begin(), shard.lru_list_, it->second.second); return; } + // Capacity per shard + std::size_t shard_capacity = (capacity_ + NUM_SHARDS - 1) / NUM_SHARDS; + // Evict oldest element if at capacity - if (cache_map_.size() >= capacity_ && capacity_ > 0) { - GlyphCacheKey oldest = lru_list_.back(); - cache_map_.erase(oldest); - lru_list_.pop_back(); + if (shard.cache_map_.size() >= shard_capacity && shard_capacity > 0) { + GlyphCacheKey oldest = shard.lru_list_.back(); + shard.cache_map_.erase(oldest); + shard.lru_list_.pop_back(); } // Insert new element - if (capacity_ > 0) { - lru_list_.push_front(key); - cache_map_[key] = std::make_pair(bitmap, lru_list_.begin()); + if (shard_capacity > 0) { + shard.lru_list_.push_front(key); + shard.cache_map_[key] = std::make_pair(bitmap, shard.lru_list_.begin()); } } std::size_t GlyphCache::size() const { - std::lock_guard lock(mutex_); - return cache_map_.size(); + std::size_t total = 0; + for (const auto& shard : shards_) { + std::lock_guard lock(shard->mutex_); + total += shard->cache_map_.size(); + } + return total; } std::size_t GlyphCache::capacity() const { - std::lock_guard lock(mutex_); return capacity_; } void GlyphCache::clear() { - std::lock_guard lock(mutex_); - cache_map_.clear(); - lru_list_.clear(); - hits_ = 0; - misses_ = 0; + for (auto& shard : shards_) { + std::lock_guard lock(shard->mutex_); + shard->cache_map_.clear(); + shard->lru_list_.clear(); + shard->hits_ = 0; + shard->misses_ = 0; + } } double GlyphCache::hitRate() const { - std::lock_guard lock(mutex_); - std::size_t total = hits_ + misses_; + std::size_t total_hits = 0; + std::size_t total_misses = 0; + + for (const auto& shard : shards_) { + std::lock_guard lock(shard->mutex_); + total_hits += shard->hits_; + total_misses += shard->misses_; + } + + std::size_t total = total_hits + total_misses; if (total == 0) { return 0.0; } - return static_cast(hits_) / total; + return static_cast(total_hits) / total; } void GlyphCache::resetStats() { - std::lock_guard lock(mutex_); - hits_ = 0; - misses_ = 0; + for (auto& shard : shards_) { + std::lock_guard lock(shard->mutex_); + shard->hits_ = 0; + shard->misses_ = 0; + } } } // namespace pdfengine::fonts diff --git a/engine/src/fonts/cache/glyph_cache.hpp b/engine/src/fonts/cache/glyph_cache.hpp index b497243..e58d60c 100644 --- a/engine/src/fonts/cache/glyph_cache.hpp +++ b/engine/src/fonts/cache/glyph_cache.hpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace pdfengine::fonts { @@ -66,17 +67,28 @@ public: private: std::size_t capacity_; - std::size_t hits_ = 0; - std::size_t misses_ = 0; - std::list lru_list_; using CacheIterator = std::list::iterator; - std::unordered_map< - GlyphCacheKey, - std::pair, - GlyphCacheKeyHash - > cache_map_; - mutable std::mutex mutex_; + + struct Shard { + std::size_t hits_ = 0; + std::size_t misses_ = 0; + std::list lru_list_; + std::unordered_map< + GlyphCacheKey, + std::pair, + GlyphCacheKeyHash + > cache_map_; + mutable std::mutex mutex_; + }; + + static constexpr std::size_t NUM_SHARDS = 16; + std::vector> shards_; + + // Helper to get shard index based on key hash + std::size_t getShardIndex(const GlyphCacheKey& key) const { + return GlyphCacheKeyHash{}(key) % NUM_SHARDS; + } }; } // namespace pdfengine::fonts diff --git a/engine/src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp b/engine/src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp index 080fb19..2b93589 100644 --- a/engine/src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp +++ b/engine/src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp @@ -78,7 +78,7 @@ uint32_t CjkCollectionDB::resolveCID(const std::string& collection, uint32_t cid else if (collection == "Adobe-Korea1" || collection.find("Korea") != std::string::npos) { // Standard Korean Hangul Syllables mapping (U+AC00 block) // CIDs 101 to 150 map to the first segment of KS X 1001 Hangul syllables - if (cid >= 101 && cid <= 150) { + if (cid >= 101 && cid <= 160) { switch (cid) { case 101: return 0xAC00; // 가 case 102: return 0xAC01; // 각 @@ -130,6 +130,16 @@ uint32_t CjkCollectionDB::resolveCID(const std::string& collection, uint32_t cid case 148: return 0xAC8C; // 게 case 149: return 0xAC8D; // 겐 case 150: return 0xAC90; // 겔 + case 151: return 0xAC94; // 겝 + case 152: return 0xAC9F; // 겟 + case 153: return 0xACA0; // 겠 + case 154: return 0xACA1; // 겡 + case 155: return 0xACA8; // 겯 + case 156: return 0xACA9; // 결 + case 157: return 0xACB8; // 겸 + case 158: return 0xACB9; // 겹 + case 159: return 0xACBC; // 겻 + case 160: return 0xACBD; // 겼 default: break; } } @@ -137,7 +147,7 @@ uint32_t CjkCollectionDB::resolveCID(const std::string& collection, uint32_t cid else if (collection == "Adobe-CNS1" || collection.find("CNS1") != std::string::npos) { // Standard Traditional Chinese mappings (U+4E00 block) // CIDs 100 onwards maps core Traditional Chinese characters - if (cid >= 100 && cid <= 130) { + if (cid >= 100 && cid <= 140) { switch (cid) { case 100: return 0x4E00; // 一 case 101: return 0x4E03; // 七 @@ -170,6 +180,16 @@ uint32_t CjkCollectionDB::resolveCID(const std::string& collection, uint32_t cid case 128: return 0x4ED5; // 仕 case 129: return 0x4ED6; // 他 case 130: return 0x4ED7; // 仗 + case 131: return 0x4ED8; // 付 + case 132: return 0x4ED9; // 仙 + case 133: return 0x4EDD; // 仝 + case 134: return 0x4EDE; // 仞 + case 135: return 0x4EDF; // 仟 + case 136: return 0x4EE1; // 仡 + case 137: return 0x4EE3; // 代 + case 138: return 0x4EE4; // 令 + case 139: return 0x4EE5; // 以 + case 140: return 0x4F01; // 企 default: break; } } diff --git a/engine/src/parser/pdfium_document.cpp b/engine/src/parser/pdfium_document.cpp index 89bc305..3a5da60 100644 --- a/engine/src/parser/pdfium_document.cpp +++ b/engine/src/parser/pdfium_document.cpp @@ -15,7 +15,7 @@ #include #include -namespace { +namespace pdfengine::parser { std::vector utf8_to_utf16le(const std::string& utf8) { std::vector utf16; @@ -56,15 +56,32 @@ std::string utf16le_to_utf8(const char16_t* utf16, size_t length) { for (size_t i = 0; i < length; ++i) { char16_t c = utf16[i]; if (c == 0) break; - if (c < 0x80) { - utf8 += static_cast(c); - } else if (c < 0x800) { - utf8 += static_cast(0xC0 | (c >> 6)); - utf8 += static_cast(0x80 | (c & 0x3F)); + + uint32_t cp = c; + if (c >= 0xD800 && c <= 0xDBFF) { // High surrogate + if (i + 1 < length) { + char16_t low = utf16[i + 1]; + if (low >= 0xDC00 && low <= 0xDFFF) { // Low surrogate + cp = 0x10000 + (((c - 0xD800) << 10) | (low - 0xDC00)); + i++; // Consume the low surrogate + } + } + } + + if (cp < 0x80) { + utf8 += static_cast(cp); + } else if (cp < 0x800) { + utf8 += static_cast(0xC0 | (cp >> 6)); + utf8 += static_cast(0x80 | (cp & 0x3F)); + } else if (cp < 0x10000) { + utf8 += static_cast(0xE0 | (cp >> 12)); + utf8 += static_cast(0x80 | ((cp >> 6) & 0x3F)); + utf8 += static_cast(0x80 | (cp & 0x3F)); } else { - utf8 += static_cast(0xE0 | (c >> 12)); - utf8 += static_cast(0x80 | ((c >> 6) & 0x3F)); - utf8 += static_cast(0x80 | (c & 0x3F)); + utf8 += static_cast(0xF0 | (cp >> 18)); + utf8 += static_cast(0x80 | ((cp >> 12) & 0x3F)); + utf8 += static_cast(0x80 | ((cp >> 6) & 0x3F)); + utf8 += static_cast(0x80 | (cp & 0x3F)); } } return utf8; @@ -90,7 +107,9 @@ std::string code_point_to_utf8(unsigned int cp) { } return utf8; } +} // namespace pdfengine::parser +namespace { #ifdef PDFENGINE_WITH_PDFIUM struct VectorWriter : public FPDF_FILEWRITE { @@ -414,6 +433,117 @@ void deduceFontMetadata(pdfengine::FontInfo& f) { spdlog::info("CID font detected: '{}' with CMap '{}', registry '{}'", f.fontName, f.cmapName, f.cidSystemInfo); } } + +// --------------------------------------------------------------------------- +// FontPdfData — real values extracted from the PDF font dictionary via PDFium. +// Populated by buildFontPdfDataMap() and used to override deduceFontMetadata() +// results in PdfiumPage::getFonts(). +// --------------------------------------------------------------------------- +struct FontPdfData { + bool valid = false; + bool isEmbedded = false; + int flags = 0; // PDF /Flags from FontDescriptor + double ascent = 0.0; // in PDF 1000-unit space + double descent = 0.0; // in PDF 1000-unit space + bool hasUnicodeMapping = false; // at least one char decoded to valid Unicode +}; + +// Builds a map from font-name to FontPdfData by scanning page text objects. +// +// Phase 1 — page object walk: +// Uses FPDFTextObj_GetFont() + FPDFFont_Get*() to read the actual PDF font +// dictionary fields: embedded status, descriptor flags and metrics. +// These values replace the corresponding fields that deduceFontMetadata() +// previously guessed from the font name string. +// +// Phase 2 — text-page character scan: +// For each font encountered in Phase 1, tests whether FPDFText_GetUnicode() +// returns a valid codepoint for at least one character belonging to that font. +// If yes, the font has an active Unicode mapping (ToUnicode CMap or built-in +// encoding). This replaces the previous always-true heuristic. +static std::unordered_map +buildFontPdfDataMap(FPDF_PAGE page, FPDF_TEXTPAGE textPage) { + std::unordered_map result; + if (!page) return result; + + // ---- Phase 1: page object walk to collect real font properties ---- + int objectCount = FPDFPage_CountObjects(page); + for (int i = 0; i < objectCount; ++i) { + FPDF_PAGEOBJECT obj = FPDFPage_GetObject(page, i); + if (!obj) continue; + if (FPDFPageObj_GetType(obj) != FPDF_PAGEOBJ_TEXT) continue; + + FPDF_FONT font = FPDFTextObj_GetFont(obj); + if (!font) continue; + + // Retrieve font name — first call returns required buffer size. + unsigned long nameLen = FPDFFont_GetBaseFontName(font, nullptr, 0); + if (nameLen == 0) continue; + std::vector nameBuf(nameLen); + if (FPDFFont_GetBaseFontName(font, nameBuf.data(), nameLen) == 0) continue; + std::string fontName(nameBuf.data()); + if (fontName.empty()) continue; + + // Each font name is processed only once. + if (result.count(fontName)) continue; + + FontPdfData data; + data.valid = true; + // FPDFFont_GetIsEmbedded returns 1 when the PDF contains an embedded + // font stream (/FontFile, /FontFile2, or /FontFile3 in the descriptor). + data.isEmbedded = (FPDFFont_GetIsEmbedded(font) != 0); + // FPDFFont_GetFlags returns the actual /Flags integer from the PDF + // FontDescriptor dictionary (not the text-rendering flags). + data.flags = FPDFFont_GetFlags(font); + + // FPDFFont_GetAscent/Descent are scaled by font_size. + // Passing 1000.0 recovers the raw PDF 1000-unit-space values. + float rawAscent = 0.0f; + if (FPDFFont_GetAscent(font, 1000.0f, &rawAscent) && rawAscent != 0.0f) { + data.ascent = static_cast(rawAscent); + } + float rawDescent = 0.0f; + if (FPDFFont_GetDescent(font, 1000.0f, &rawDescent) && rawDescent != 0.0f) { + data.descent = static_cast(rawDescent); + } + + result[fontName] = data; + } + + // ---- Phase 2: Unicode mapping probe via text-page character scan ---- + // For each known font, test whether PDFium can return a non-trivial Unicode + // value for at least one character that belongs to it. A valid decode + // implies the font has an active ToUnicode CMap or a built-in encoding map. + if (textPage && !result.empty()) { + int charCount = FPDFText_CountChars(textPage); + if (charCount > 0 && charCount < 1000000) { + for (int ci = 0; ci < charCount; ++ci) { + int fi = 0; + unsigned long flen = + FPDFText_GetFontInfo(textPage, ci, nullptr, 0, &fi); + if (flen == 0) continue; + std::vector fbuf(flen); + if (FPDFText_GetFontInfo(textPage, ci, fbuf.data(), flen, &fi) == 0) + continue; + std::string fname(fbuf.data()); + + auto it = result.find(fname); + // Skip fonts not found in Phase 1, or already confirmed. + if (it == result.end() || it->second.hasUnicodeMapping) continue; + + unsigned int cp = FPDFText_GetUnicode(textPage, ci); + // Require a printable codepoint — exclude NUL, control chars, + // and the Unicode replacement character (U+FFFD). + if (cp > 0x0020 && cp != 0xFFFD) { + it->second.hasUnicodeMapping = true; + } + } + } + } + + return result; +} + #endif } @@ -942,6 +1072,11 @@ std::expected, EngineError> PdfiumPage::getFonts() const { return std::unexpected(EngineError::Unknown); } + // Build real font data from the PDF font dictionary via page object walk. + // This provides accurate values for isEmbedded, subtype, flags, ascent, + // descent, and hasUnicodeMapping — replacing name-based heuristics. + auto fontPdfDataMap = buildFontPdfDataMap(page_, textPage_); + std::vector pageFonts; int charCount = FPDFText_CountChars(textPage_); @@ -965,12 +1100,13 @@ std::expected, EngineError> PdfiumPage::getFonts() const { }; // Detect vertical writing mode per font by analyzing character origin positions. - // In vertical writing mode, consecutive characters of the same font move primarily - // in the Y direction. We track this per font name. + // We filter out large spatial jumps (e.g. line breaks or paragraph changes) + // by comparing the coordinate delta to the character's font size. struct FontPositionStats { - double totalDeltaX = 0.0; - double totalDeltaY = 0.0; - int samples = 0; + int horizontalSteps = 0; + int verticalSteps = 0; + double totalFilteredDeltaX = 0.0; + double totalFilteredDeltaY = 0.0; double prevX = 0.0, prevY = 0.0; bool hasPrev = false; }; @@ -993,9 +1129,25 @@ std::expected, EngineError> PdfiumPage::getFonts() const { if (FPDFText_GetCharOrigin(textPage_, i, &ox, &oy)) { auto& stats = fontStats[fontName]; if (stats.hasPrev) { - stats.totalDeltaX += std::abs(ox - stats.prevX); - stats.totalDeltaY += std::abs(oy - stats.prevY); - stats.samples++; + double dx = std::abs(ox - stats.prevX); + double dy = std::abs(oy - stats.prevY); + double fontSize = FPDFText_GetFontSize(textPage_, i); + + // Filter out non-consecutive jumps (e.g., line or paragraph breaks). + // We allow a generously large multiplier (3.0) for wide tracking, + // but enforce a minimum of 30.0 units for very small text. + double maxJump = (std::max)(fontSize * 3.0, 30.0); + + if (dx < maxJump && dy < maxJump) { + stats.totalFilteredDeltaX += dx; + stats.totalFilteredDeltaY += dy; + + if (dy > dx * 1.5) { + stats.verticalSteps++; + } else if (dx > dy * 1.5) { + stats.horizontalSteps++; + } + } } stats.prevX = ox; stats.prevY = oy; @@ -1004,16 +1156,21 @@ std::expected, EngineError> PdfiumPage::getFonts() const { } } - // Determine which fonts are vertical (Y movement dominates X movement) + // Determine which fonts are vertical std::unordered_map fontIsVertical; for (const auto& [fname, stats] : fontStats) { - if (stats.samples > 0) { - // Vertical if Y movement is substantially greater than X movement - // Use a threshold: Y > 2 * X and meaningful Y movement - bool vertical = (stats.totalDeltaY > 2.0 * stats.totalDeltaX) && - (stats.totalDeltaY > 0.5); - fontIsVertical[fname] = vertical; + bool vertical = false; + + // Primary heuristic: explicit step counts for consecutive characters + if (stats.verticalSteps > 0 || stats.horizontalSteps > 0) { + vertical = stats.verticalSteps > stats.horizontalSteps; + } else { + // Fallback: if no clear steps were isolated, rely on filtered totals + vertical = (stats.totalFilteredDeltaY > 2.0 * stats.totalFilteredDeltaX) && + (stats.totalFilteredDeltaY > 0.5); } + + fontIsVertical[fname] = vertical; } // Second pass: build FontInfo list @@ -1034,10 +1191,68 @@ std::expected, EngineError> PdfiumPage::getFonts() const { FontInfo f; f.fontName = fontName; f.flags = static_cast(charInfos[i].flags); - - // Deduce advanced metadata from font name heuristics + + // Phase A: heuristic baseline — fills encoding, cmapName, cidSystemInfo, + // normalizedFamily, subset tag, and initial values for all other fields. deduceFontMetadata(f); + // Phase B: override heuristic fields with real PDF dictionary values + // wherever buildFontPdfDataMap() was able to extract them. + auto pdfDataIt = fontPdfDataMap.find(fontName); + if (pdfDataIt != fontPdfDataMap.end()) { + const FontPdfData& pd = pdfDataIt->second; + + // --- Embedding status (replaces subset-tag heuristic) --- + f.isEmbedded = pd.isEmbedded; + + // --- Font descriptor flags (real /Flags value) --- + f.flags = static_cast(pd.flags); + + // --- Descriptor metrics (real ascent/descent) --- + if (pd.ascent != 0.0) f.ascent = pd.ascent; + if (pd.descent != 0.0) f.descent = pd.descent; + + // CapHeight is not exposed by the PDFium public API. + // Use the exact specification values for the standard 14 fonts; + // for everything else, estimate from real ascent (71% is the + // empirical Latin cap-height ratio across common typefaces). + if (f.ascent > 0.0) { + auto lname = fontName; + std::transform(lname.begin(), lname.end(), lname.begin(), ::tolower); + if (lname.find("times") != std::string::npos) f.capHeight = 662.0; + else if (lname.find("courier") != std::string::npos) f.capHeight = 562.0; + else if (lname.find("symbol") != std::string::npos) f.capHeight = 673.0; + else if (lname.find("helvetica") != std::string::npos) f.capHeight = 728.0; + else f.capHeight = f.ascent * 0.71; + } + + // --- Unicode mapping (replaces always-true heuristic) --- + f.hasToUnicode = pd.hasUnicodeMapping; + + // --- sourceType derived from real embedding status --- + if (f.isEmbedded) { + // Font stream is present in the PDF — always report as Embedded. + f.sourceType = "Embedded"; + f.substitutedFrom = ""; + f.substitutedTo = ""; + } + // If not embedded, keep the substitution info already set by + // deduceFontMetadata() (SystemFallback or Substituted). + + // --- Recalculate stable identifier with corrected data --- + if (f.isSubset && !f.subsetTag.empty()) { + f.internalFontId = f.subsetTag + "_" + f.fontName; + } else { + f.internalFontId = f.fontName + "_" + f.type + "_" + std::to_string(f.flags); + } + + spdlog::debug( + "Font '{}': isEmbedded={} type='{}' ascent={:.1f} descent={:.1f} " + "capHeight={:.1f} hasToUnicode={} flags={}", + fontName, f.isEmbedded, f.type, f.ascent, f.descent, + f.capHeight, f.hasToUnicode, f.flags); + } + // Override isVertical with position-based detection if not already set by name heuristic if (!f.isVertical) { auto vit = fontIsVertical.find(fontName); diff --git a/engine/src/parser/pdfium_document.hpp b/engine/src/parser/pdfium_document.hpp index 71b1b08..7526a05 100644 --- a/engine/src/parser/pdfium_document.hpp +++ b/engine/src/parser/pdfium_document.hpp @@ -85,4 +85,9 @@ private: mutable std::mutex fontsMutex_; }; +// Exposed for testing +std::vector utf8_to_utf16le(const std::string& utf8); +std::string utf16le_to_utf8(const char16_t* utf16, size_t length); +std::string code_point_to_utf8(unsigned int cp); + } \ No newline at end of file diff --git a/engine/tests/document_test.cpp b/engine/tests/document_test.cpp index 65f22d6..41d2ab9 100644 --- a/engine/tests/document_test.cpp +++ b/engine/tests/document_test.cpp @@ -1,12 +1,16 @@ #include #include #include +#include "parser/pdfium_document.hpp" +#include "fonts/pdf_fonts/encoding/cjk_collection_db.hpp" #include #include #include #include #include - +#include +#include +#include "fonts/cache/glyph_cache.hpp" #ifndef TEST_CORPUS_DIR #define TEST_CORPUS_DIR "../../corpus" #endif @@ -636,6 +640,23 @@ TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) { } } + // Test Part 2b: Vertical Font Detection Heuristic explicit validation + { + auto path1 = getCorpusPath("fonts", "vertical_identity_v.pdf"); + if (std::filesystem::exists(path1)) { + auto docRes = PdfDocument::loadFromFile(path1.string()); + ASSERT_TRUE(docRes.has_value()); + auto fontsRes = (*docRes)->getFonts(); + ASSERT_TRUE(fontsRes.has_value()); + + bool foundVertical = false; + for (const auto& f : *fontsRes) { + if (f.isVertical) foundVertical = true; + } + EXPECT_TRUE(foundVertical) << "Failed to detect vertical font in vertical_identity_v.pdf"; + } + } + // Test Part 3: Font Size and Glyph Bounds Handling { auto path = getCorpusPath("fonts", "utf-8.pdf"); @@ -680,4 +701,340 @@ TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) { } } +// ========================================================================= +// Tests validating real PDFium font dictionary introspection. +// These tests verify that isEmbedded, type, ascent, descent, capHeight, and +// hasToUnicode are now derived from actual PDF font objects rather than from +// font-name heuristics (the behaviour that predated this change). +// ========================================================================= + +// Verify that embedded fonts report isEmbedded=true and that sourceType is +// set to "Embedded" from the real FPDFFont_GetIsEmbedded() result. +// A subset-embedded font (ABCDEF+FontName prefix) is the clearest case +// because the old heuristic relied solely on the prefix tag for embedding +// detection, while real PDFium checks for the /FontFile stream. +TEST(FontDiagnosticsTest, RealPDFiumEmbeddingAndTypeAccuracy) { + SKIP_IF_NO_PDFIUM(); + + // text_font.pdf has an embedded subset TrueType font \u2014 best candidate for + // verifying that isEmbedded comes from the PDF font stream, not the name tag. + auto path = getCorpusPath("fonts", "text_font.pdf"); + if (!std::filesystem::exists(path)) { + GTEST_SKIP() << "text_font.pdf not found in corpus."; + } + + auto docRes = PdfDocument::loadFromFile(path.string()); + ASSERT_TRUE(docRes.has_value()) << "Failed to open text_font.pdf"; + auto doc = *docRes; + + auto pageRes = doc->getPage(0); + ASSERT_TRUE(pageRes.has_value()); + auto fontsRes = (*pageRes)->getFonts(); + ASSERT_TRUE(fontsRes.has_value()); + const auto& fonts = *fontsRes; + ASSERT_FALSE(fonts.empty()) << "text_font.pdf must expose at least one font"; + + for (const auto& f : fonts) { + // Core invariant: every font must have a non-empty name and type. + EXPECT_FALSE(f.fontName.empty()); + EXPECT_FALSE(f.type.empty()); + + // type must be one of the four valid PDF font subtypes. + static const std::vector kValidTypes = { + "Type1", "TrueType", "CIDFontType0", "CIDFontType2" + }; + bool typeValid = std::find(kValidTypes.begin(), kValidTypes.end(), f.type) + != kValidTypes.end(); + EXPECT_TRUE(typeValid) << "Unexpected type '" << f.type << "' for font '" << f.fontName << "'"; + + // Subset-prefixed fonts MUST be reported as embedded by PDFium + // (the /FontFile stream is required by the PDF spec when a subset tag is present). + if (f.isSubset) { + EXPECT_TRUE(f.isEmbedded) + << "Subset font '" << f.fontName + << "' must be embedded (FPDFFont_GetIsEmbedded should return 1)"; + EXPECT_EQ(f.sourceType, "Embedded") + << "sourceType must be 'Embedded' when isEmbedded=true"; + EXPECT_TRUE(f.substitutedFrom.empty()); + EXPECT_TRUE(f.substitutedTo.empty()); + } + + // isEmbedded=true and sourceType="Embedded" must be consistent. + if (f.isEmbedded) { + EXPECT_EQ(f.sourceType, "Embedded"); + } + } +} + +// Verify that font descriptor metrics (ascent, descent, capHeight) come from +// the real PDF FontDescriptor via FPDFFont_GetAscent/Descent(), not from the +// hardcoded fallback table. The critical invariant is sign correctness: +// ascent must be positive, descent must be negative. +TEST(FontDiagnosticsTest, RealPDFiumMetricsAccuracy) { + SKIP_IF_NO_PDFIUM(); + + // Use the largest font corpus file; it contains the most diverse fonts. + auto path = getCorpusPath("fonts", "text_font.pdf"); + if (!std::filesystem::exists(path)) { + GTEST_SKIP() << "text_font.pdf not found in corpus."; + } + + auto docRes = PdfDocument::loadFromFile(path.string()); + ASSERT_TRUE(docRes.has_value()); + auto fontsRes = (*docRes)->getFonts(); + ASSERT_TRUE(fontsRes.has_value()); + + for (const auto& f : *fontsRes) { + // ascent and descent from FPDFFont_GetAscent/Descent(font, 1000.0f, …) + // are in PDF 1000-unit space. ascent is above the baseline (positive), + // descent is below (negative). + EXPECT_GT(f.ascent, 0.0) + << "ascent must be positive for font '" << f.fontName << "'"; + EXPECT_LT(f.descent, 0.0) + << "descent must be negative for font '" << f.fontName << "'"; + EXPECT_GT(f.capHeight, 0.0) + << "capHeight must be positive for font '" << f.fontName << "'"; + + // capHeight must not exceed ascent (sanity: caps never taller than ascender). + EXPECT_LE(f.capHeight, f.ascent + 1.0) // +1 for float rounding + << "capHeight should not exceed ascent for font '" << f.fontName << "'"; + + // Values must be in a plausible PDF 1000-unit-space range. + // Standard fonts typically have ascent in [400, 1200]. + EXPECT_LT(f.ascent, 1500.0) << "Implausibly large ascent for '" << f.fontName << "'"; + EXPECT_GT(f.descent, -1500.0) << "Implausibly deep descent for '" << f.fontName << "'"; + } +} + +// Verify that hasToUnicode reflects actual Unicode decode capability rather +// than the old always-true heuristic. +// +// with_tounicode.pdf \u2014 PDF containing a font that has a /ToUnicode stream; +// PDFium should decode characters successfully. +// no_tounicode.pdf \u2014 PDF containing a font with no /ToUnicode stream and no +// standard encoding; PDFium cannot map char codes to Unicode. +// latin_extended.pdf \u2014 Standard Latin font; must decode to Unicode via built-in +// encoding (WinAnsiEncoding or similar). +TEST(FontDiagnosticsTest, ToUnicodePresenceAccuracy) { + SKIP_IF_NO_PDFIUM(); + + // Case 1: font WITH ToUnicode \u2014 hasToUnicode must be true + { + auto path = getCorpusPath("fonts", "with_tounicode.pdf"); + if (std::filesystem::exists(path)) { + auto docRes = PdfDocument::loadFromFile(path.string()); + if (docRes.has_value()) { + auto pageRes = (*docRes)->getPage(0); + if (pageRes.has_value()) { + auto fontsRes = (*pageRes)->getFonts(); + if (fontsRes.has_value() && !fontsRes->empty()) { + bool anyTrue = false; + for (const auto& f : *fontsRes) { + if (f.hasToUnicode) { anyTrue = true; break; } + } + EXPECT_TRUE(anyTrue) + << "At least one font in with_tounicode.pdf must have hasToUnicode=true"; + } + } + } + } + } + + // Case 2: font WITHOUT ToUnicode or decodable encoding \u2014 hasToUnicode must be false + { + auto path = getCorpusPath("fonts", "no_tounicode.pdf"); + if (std::filesystem::exists(path)) { + auto docRes = PdfDocument::loadFromFile(path.string()); + if (docRes.has_value()) { + auto pageRes = (*docRes)->getPage(0); + if (pageRes.has_value()) { + auto fontsRes = (*pageRes)->getFonts(); + if (fontsRes.has_value() && !fontsRes->empty()) { + // All fonts in a no-tounicode document should fail Unicode decode. + for (const auto& f : *fontsRes) { + EXPECT_FALSE(f.hasToUnicode) + << "Font '" << f.fontName + << "' in no_tounicode.pdf must have hasToUnicode=false"; + } + } + } + } + } + } + + // Case 3: standard Latin font \u2014 must decode to Unicode via built-in encoding + { + auto path = getCorpusPath("fonts", "latin_extended.pdf"); + if (std::filesystem::exists(path)) { + auto docRes = PdfDocument::loadFromFile(path.string()); + if (docRes.has_value()) { + auto pageRes = (*docRes)->getPage(0); + if (pageRes.has_value()) { + auto fontsRes = (*pageRes)->getFonts(); + if (fontsRes.has_value() && !fontsRes->empty()) { + bool anyTrue = false; + for (const auto& f : *fontsRes) { + if (f.hasToUnicode) { anyTrue = true; break; } + } + EXPECT_TRUE(anyTrue) + << "At least one font in latin_extended.pdf must decode to Unicode"; + } + } + } + } + } +} +// ========================================================================= +// Tests for UTF-16 Surrogate Pairs (Emoji and CJK Ext-B) +// ========================================================================= + +TEST(UtfConversionTest, EmojiSurrogatePairs) { + // 😀 U+1F600 -> UTF-8: F0 9F 98 80 + std::string utf8_grinning = "\xF0\x9F\x98\x80"; + auto utf16 = pdfengine::parser::utf8_to_utf16le(utf8_grinning); + // Should be D83D DE00 + null terminator + ASSERT_EQ(utf16.size(), 3); + EXPECT_EQ(utf16[0], 0xD83D); + EXPECT_EQ(utf16[1], 0xDE00); + EXPECT_EQ(utf16[2], 0x0000); + + // Convert back to UTF-8 + std::string utf8_out = pdfengine::parser::utf16le_to_utf8(reinterpret_cast(utf16.data()), utf16.size() - 1); + EXPECT_EQ(utf8_out, utf8_grinning); + + // 🚀 U+1F680 -> UTF-8: F0 9F 9A 80 + std::string utf8_rocket = "\xF0\x9F\x9A\x80"; + utf16 = pdfengine::parser::utf8_to_utf16le(utf8_rocket); + ASSERT_EQ(utf16.size(), 3); + EXPECT_EQ(utf16[0], 0xD83D); + EXPECT_EQ(utf16[1], 0xDE80); + EXPECT_EQ(utf16[2], 0x0000); + + utf8_out = pdfengine::parser::utf16le_to_utf8(reinterpret_cast(utf16.data()), utf16.size() - 1); + EXPECT_EQ(utf8_out, utf8_rocket); +} + +TEST(UtfConversionTest, CJKExtensionB) { + // 𠀀 U+20000 -> UTF-8: F0 A0 80 80 + std::string utf8_cjk = "\xF0\xA0\x80\x80"; + auto utf16 = pdfengine::parser::utf8_to_utf16le(utf8_cjk); + // Should be D840 DC00 + null terminator + ASSERT_EQ(utf16.size(), 3); + EXPECT_EQ(utf16[0], 0xD840); + EXPECT_EQ(utf16[1], 0xDC00); + EXPECT_EQ(utf16[2], 0x0000); + + // Convert back to UTF-8 + std::string utf8_out = pdfengine::parser::utf16le_to_utf8(reinterpret_cast(utf16.data()), utf16.size() - 1); + EXPECT_EQ(utf8_out, utf8_cjk); +} + +TEST(UtfConversionTest, RoundtripMixed) { + // "A😀B𠀀C" -> 41 F0 9F 98 80 42 F0 A0 80 80 43 + std::string mixed = "A\xF0\x9F\x98\x80""B\xF0\xA0\x80\x80""C"; + auto utf16 = pdfengine::parser::utf8_to_utf16le(mixed); + std::string mixed_out = pdfengine::parser::utf16le_to_utf8(reinterpret_cast(utf16.data()), utf16.size() - 1); + EXPECT_EQ(mixed_out, mixed); +} + +// ========================================================================= +// Tests for CJK CID Resolution +// ========================================================================= + +TEST(CjkResolutionTest, AdobeCNS1) { + // Basic mapping checks for the core Adobe-CNS1 block (Traditional Chinese) + using pdfengine::fonts::pdf_fonts::CjkCollectionDB; + + // Test existing block (100-130) + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 100), 0x4E00); // 一 + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 112), 0x4E2D); // 中 + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 130), 0x4ED7); // 仗 + + // Test the newly expanded block (131-140) + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 131), 0x4ED8); // 付 + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 135), 0x4EDF); // 仟 + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 140), 0x4F01); // 企 + + // Test alternative naming + EXPECT_EQ(CjkCollectionDB::resolveCID("Identity-H-CNS1", 137), 0x4EE3); // 代 +} + +TEST(CjkResolutionTest, AdobeKorea1) { + using pdfengine::fonts::pdf_fonts::CjkCollectionDB; + + // Test existing Korean block (101-150) + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 101), 0xAC00); // 가 + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 119), 0xAC1C); // 개 + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 150), 0xAC90); // 겔 + + // Test newly added Hangul block (151-160) + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 151), 0xAC94); // 겝 + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 156), 0xACA9); // 결 + EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 160), 0xACBD); // 겼 + + // Test alternative naming + EXPECT_EQ(CjkCollectionDB::resolveCID("UniKS-UTF16-H-Korea1", 153), 0xACA0); // 겠 +} + +// ========================================================================= +// GlyphCache Concurrency & Benchmark Test +// ========================================================================= + +TEST(GlyphCacheTest, ConcurrencyBench) { + using namespace pdfengine::fonts; + + FontFace face; + // Load a common system font for testing cache keys + bool loaded = face.loadFromFile("C:\\Windows\\Fonts\\arial.ttf"); + if (!loaded) { + GTEST_SKIP() << "Skipping benchmark: Arial font not found."; + } + + GlyphCache cache(1000); + + auto run_benchmark = [&](int num_threads, int ops_per_thread) { + std::atomic start_flag{0}; + std::vector threads; + + for (int i = 0; i < num_threads; ++i) { + threads.emplace_back([&, i]() { + while (start_flag.load() == 0) { std::this_thread::yield(); } + + for (int op = 0; op < ops_per_thread; ++op) { + unsigned int glyphIndex = (op + i) % 2000; + unsigned int fontSize = 12 + (op % 5); + + auto hit = cache.get(face, glyphIndex, fontSize); + if (!hit) { + GlyphBitmap bmp; + bmp.width = 10; bmp.height = 10; + cache.insert(face, glyphIndex, fontSize, bmp); + } + } + }); + } + + auto start_time = std::chrono::high_resolution_clock::now(); + start_flag.store(1); + for (auto& t : threads) { t.join(); } + auto end_time = std::chrono::high_resolution_clock::now(); + + std::chrono::duration diff = end_time - start_time; + return diff.count(); + }; + + run_benchmark(2, 1000); // warmup + cache.clear(); + + double time_10 = run_benchmark(10, 10000); + std::cout << "[ BENCHMARK ] 10 Threads Time: " << time_10 << " seconds (" << (100000.0 / time_10) << " ops/sec)\n"; + + cache.clear(); + + double time_50 = run_benchmark(50, 10000); + std::cout << "[ BENCHMARK ] 50 Threads Time: " << time_50 << " seconds (" << (500000.0 / time_50) << " ops/sec)\n"; + + EXPECT_LE(cache.size(), 1000 + 16); // Accommodate shard capacity rounding +} + } diff --git a/api_validate.py b/gateway/tests/api_validate.py similarity index 100% rename from api_validate.py rename to gateway/tests/api_validate.py diff --git a/generate_corpus.py b/tools/generate_corpus.py similarity index 100% rename from generate_corpus.py rename to tools/generate_corpus.py diff --git a/scripts/validate_concurrency.py b/tools/validate_concurrency.py similarity index 100% rename from scripts/validate_concurrency.py rename to tools/validate_concurrency.py diff --git a/scripts/validate_memory.py b/tools/validate_memory.py similarity index 100% rename from scripts/validate_memory.py rename to tools/validate_memory.py diff --git a/verify_utf8.py b/tools/verify_utf8.py similarity index 100% rename from verify_utf8.py rename to tools/verify_utf8.py