maskantech/pdf
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://gitea.maskantech.in/gitea_admin/pdf into furqan

Browse Source
This commit is contained in:
Furqan-14
2026-06-02 11:40:40 +05:30
parent 638ea5ba1c e580423c41
commit 4e8fcb8aaa
11 changed files with 705 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/fonts/cache/glyph_cache.cpp
Vendored
+58 -32
View File
@@ -3,7 +3,11 @@
namespace pdfengine::fonts {
GlyphCache::GlyphCache(std::size_t capacity)
: capacity_(capacity) {}
: capacity_(capacity) {
for (std::size_t i = 0; i < NUM_SHARDS; ++i) {
shards_.push_back(std::make_unique<Shard>());
}
}
GlyphCache::~GlyphCache() = default;
@@ -14,17 +18,19 @@ std::optional<GlyphBitmap> GlyphCache::get(const FontFace& fontFace, unsigned in
}
GlyphCacheKey key{face, glyphIndex, fontSize};
std::size_t shard_idx = getShardIndex(key);
auto& shard = *shards_[shard_idx];
std::lock_guard<std::mutex> lock(mutex_);
auto it = cache_map_.find(key);
if (it == cache_map_.end()) {
misses_++;
std::lock_guard<std::mutex> lock(shard.mutex_);
auto it = shard.cache_map_.find(key);
if (it == shard.cache_map_.end()) {
shard.misses_++;
return std::nullopt; // Cache miss
}
hits_++;
shard.hits_++;
// Cache hit: move the referenced key to the front of the LRU list
lru_list_.splice(lru_list_.begin(), lru_list_, it->second.second);
shard.lru_list_.splice(shard.lru_list_.begin(), shard.lru_list_, it->second.second);
return it->second.first;
}
@@ -36,61 +42,81 @@ void GlyphCache::insert(const FontFace& fontFace, unsigned int glyphIndex, unsig
}
GlyphCacheKey key{face, glyphIndex, fontSize};
std::size_t shard_idx = getShardIndex(key);
auto& shard = *shards_[shard_idx];
std::lock_guard<std::mutex> lock(mutex_);
auto it = cache_map_.find(key);
if (it != cache_map_.end()) {
std::lock_guard<std::mutex> lock(shard.mutex_);
auto it = shard.cache_map_.find(key);
if (it != shard.cache_map_.end()) {
// Element already exists: update bitmap and move it to the front
it->second.first = bitmap;
lru_list_.splice(lru_list_.begin(), lru_list_, it->second.second);
shard.lru_list_.splice(shard.lru_list_.begin(), shard.lru_list_, it->second.second);
return;
}
// Capacity per shard
std::size_t shard_capacity = (capacity_ + NUM_SHARDS - 1) / NUM_SHARDS;
// Evict oldest element if at capacity
if (cache_map_.size() >= capacity_ && capacity_ > 0) {
GlyphCacheKey oldest = lru_list_.back();
cache_map_.erase(oldest);
lru_list_.pop_back();
if (shard.cache_map_.size() >= shard_capacity && shard_capacity > 0) {
GlyphCacheKey oldest = shard.lru_list_.back();
shard.cache_map_.erase(oldest);
shard.lru_list_.pop_back();
}
// Insert new element
if (capacity_ > 0) {
lru_list_.push_front(key);
cache_map_[key] = std::make_pair(bitmap, lru_list_.begin());
if (shard_capacity > 0) {
shard.lru_list_.push_front(key);
shard.cache_map_[key] = std::make_pair(bitmap, shard.lru_list_.begin());
}
}
std::size_t GlyphCache::size() const {
std::lock_guard<std::mutex> lock(mutex_);
return cache_map_.size();
std::size_t total = 0;
for (const auto& shard : shards_) {
std::lock_guard<std::mutex> lock(shard->mutex_);
total += shard->cache_map_.size();
}
return total;
}
std::size_t GlyphCache::capacity() const {
std::lock_guard<std::mutex> lock(mutex_);
return capacity_;
}
void GlyphCache::clear() {
std::lock_guard<std::mutex> lock(mutex_);
cache_map_.clear();
lru_list_.clear();
hits_ = 0;
misses_ = 0;
for (auto& shard : shards_) {
std::lock_guard<std::mutex> lock(shard->mutex_);
shard->cache_map_.clear();
shard->lru_list_.clear();
shard->hits_ = 0;
shard->misses_ = 0;
}
}
double GlyphCache::hitRate() const {
std::lock_guard<std::mutex> lock(mutex_);
std::size_t total = hits_ + misses_;
std::size_t total_hits = 0;
std::size_t total_misses = 0;
for (const auto& shard : shards_) {
std::lock_guard<std::mutex> lock(shard->mutex_);
total_hits += shard->hits_;
total_misses += shard->misses_;
}
std::size_t total = total_hits + total_misses;
if (total == 0) {
return 0.0;
}
return static_cast<double>(hits_) / total;
return static_cast<double>(total_hits) / total;
}
void GlyphCache::resetStats() {
std::lock_guard<std::mutex> lock(mutex_);
hits_ = 0;
misses_ = 0;
for (auto& shard : shards_) {
std::lock_guard<std::mutex> lock(shard->mutex_);
shard->hits_ = 0;
shard->misses_ = 0;
}
}
} // namespace pdfengine::fonts
engine/src/fonts/cache/glyph_cache.hpp
Vendored
+21 -9
View File
@@ -7,6 +7,7 @@
#include <list>
#include <cstddef>
#include <mutex>
#include <memory>
namespace pdfengine::fonts {
@@ -66,17 +67,28 @@ public:
private:
std::size_t capacity_;
std::size_t hits_ = 0;
std::size_t misses_ = 0;
std::list<GlyphCacheKey> lru_list_;
using CacheIterator = std::list<GlyphCacheKey>::iterator;
std::unordered_map<
GlyphCacheKey,
std::pair<GlyphBitmap, CacheIterator>,
GlyphCacheKeyHash
> cache_map_;
mutable std::mutex mutex_;
struct Shard {
std::size_t hits_ = 0;
std::size_t misses_ = 0;
std::list<GlyphCacheKey> lru_list_;
std::unordered_map<
GlyphCacheKey,
std::pair<GlyphBitmap, CacheIterator>,
GlyphCacheKeyHash
> cache_map_;
mutable std::mutex mutex_;
};
static constexpr std::size_t NUM_SHARDS = 16;
std::vector<std::unique_ptr<Shard>> shards_;
// Helper to get shard index based on key hash
std::size_t getShardIndex(const GlyphCacheKey& key) const {
return GlyphCacheKeyHash{}(key) % NUM_SHARDS;
}
};
} // namespace pdfengine::fonts
engine/src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp
+22 -2
View File
@@ -78,7 +78,7 @@ uint32_t CjkCollectionDB::resolveCID(const std::string& collection, uint32_t cid
else if (collection == "Adobe-Korea1" || collection.find("Korea") != std::string::npos) {
// Standard Korean Hangul Syllables mapping (U+AC00 block)
// CIDs 101 to 150 map to the first segment of KS X 1001 Hangul syllables
if (cid >= 101 && cid <= 150) {
if (cid >= 101 && cid <= 160) {
switch (cid) {
case 101: return 0xAC00; // 가
case 102: return 0xAC01; // 각
@@ -130,6 +130,16 @@ uint32_t CjkCollectionDB::resolveCID(const std::string& collection, uint32_t cid
case 148: return 0xAC8C; // 게
case 149: return 0xAC8D; // 겐
case 150: return 0xAC90; // 겔
case 151: return 0xAC94; // 겝
case 152: return 0xAC9F; // 겟
case 153: return 0xACA0; // 겠
case 154: return 0xACA1; // 겡
case 155: return 0xACA8; // 겯
case 156: return 0xACA9; // 결
case 157: return 0xACB8; // 겸
case 158: return 0xACB9; // 겹
case 159: return 0xACBC; // 겻
case 160: return 0xACBD; // 겼
default: break;
}
}
@@ -137,7 +147,7 @@ uint32_t CjkCollectionDB::resolveCID(const std::string& collection, uint32_t cid
else if (collection == "Adobe-CNS1" || collection.find("CNS1") != std::string::npos) {
// Standard Traditional Chinese mappings (U+4E00 block)
// CIDs 100 onwards maps core Traditional Chinese characters
if (cid >= 100 && cid <= 130) {
if (cid >= 100 && cid <= 140) {
switch (cid) {
case 100: return 0x4E00; // 一
case 101: return 0x4E03; // 七
@@ -170,6 +180,16 @@ uint32_t CjkCollectionDB::resolveCID(const std::string& collection, uint32_t cid
case 128: return 0x4ED5; // 仕
case 129: return 0x4ED6; // 他
case 130: return 0x4ED7; // 仗
case 131: return 0x4ED8; // 付
case 132: return 0x4ED9; // 仙
case 133: return 0x4EDD; // 仝
case 134: return 0x4EDE; // 仞
case 135: return 0x4EDF; // 仟
case 136: return 0x4EE1; // 仡
case 137: return 0x4EE3; // 代
case 138: return 0x4EE4; // 令
case 139: return 0x4EE5; // 以
case 140: return 0x4F01; // 企
default: break;
}
}
engine/src/parser/pdfium_document.cpp
+241 -26
View File
@@ -15,7 +15,7 @@
#include <spdlog/spdlog.h>
#include <csetjmp>
namespace {
namespace pdfengine::parser {
std::vector<unsigned short> utf8_to_utf16le(const std::string& utf8) {
std::vector<unsigned short> utf16;
@@ -56,15 +56,32 @@ std::string utf16le_to_utf8(const char16_t* utf16, size_t length) {
for (size_t i = 0; i < length; ++i) {
char16_t c = utf16[i];
if (c == 0) break;
if (c < 0x80) {
utf8 += static_cast<char>(c);
} else if (c < 0x800) {
utf8 += static_cast<char>(0xC0 | (c >> 6));
utf8 += static_cast<char>(0x80 | (c & 0x3F));
uint32_t cp = c;
if (c >= 0xD800 && c <= 0xDBFF) { // High surrogate
if (i + 1 < length) {
char16_t low = utf16[i + 1];
if (low >= 0xDC00 && low <= 0xDFFF) { // Low surrogate
cp = 0x10000 + (((c - 0xD800) << 10) | (low - 0xDC00));
i++; // Consume the low surrogate
}
}
}
if (cp < 0x80) {
utf8 += static_cast<char>(cp);
} else if (cp < 0x800) {
utf8 += static_cast<char>(0xC0 | (cp >> 6));
utf8 += static_cast<char>(0x80 | (cp & 0x3F));
} else if (cp < 0x10000) {
utf8 += static_cast<char>(0xE0 | (cp >> 12));
utf8 += static_cast<char>(0x80 | ((cp >> 6) & 0x3F));
utf8 += static_cast<char>(0x80 | (cp & 0x3F));
} else {
utf8 += static_cast<char>(0xE0 | (c >> 12));
utf8 += static_cast<char>(0x80 | ((c >> 6) & 0x3F));
utf8 += static_cast<char>(0x80 | (c & 0x3F));
utf8 += static_cast<char>(0xF0 | (cp >> 18));
utf8 += static_cast<char>(0x80 | ((cp >> 12) & 0x3F));
utf8 += static_cast<char>(0x80 | ((cp >> 6) & 0x3F));
utf8 += static_cast<char>(0x80 | (cp & 0x3F));
}
}
return utf8;
@@ -90,7 +107,9 @@ std::string code_point_to_utf8(unsigned int cp) {
}
return utf8;
}
} // namespace pdfengine::parser
namespace {
#ifdef PDFENGINE_WITH_PDFIUM
struct VectorWriter : public FPDF_FILEWRITE {
@@ -410,6 +429,117 @@ void deduceFontMetadata(pdfengine::FontInfo& f) {
f.capHeight = 728.0;
}
}
// ---------------------------------------------------------------------------
// FontPdfData — real values extracted from the PDF font dictionary via PDFium.
// Populated by buildFontPdfDataMap() and used to override deduceFontMetadata()
// results in PdfiumPage::getFonts().
// ---------------------------------------------------------------------------
struct FontPdfData {
bool valid = false;
bool isEmbedded = false;
int flags = 0; // PDF /Flags from FontDescriptor
double ascent = 0.0; // in PDF 1000-unit space
double descent = 0.0; // in PDF 1000-unit space
bool hasUnicodeMapping = false; // at least one char decoded to valid Unicode
};
// Builds a map from font-name to FontPdfData by scanning page text objects.
//
// Phase 1 — page object walk:
// Uses FPDFTextObj_GetFont() + FPDFFont_Get*() to read the actual PDF font
// dictionary fields: embedded status, descriptor flags and metrics.
// These values replace the corresponding fields that deduceFontMetadata()
// previously guessed from the font name string.
//
// Phase 2 — text-page character scan:
// For each font encountered in Phase 1, tests whether FPDFText_GetUnicode()
// returns a valid codepoint for at least one character belonging to that font.
// If yes, the font has an active Unicode mapping (ToUnicode CMap or built-in
// encoding). This replaces the previous always-true heuristic.
static std::unordered_map<std::string, FontPdfData>
buildFontPdfDataMap(FPDF_PAGE page, FPDF_TEXTPAGE textPage) {
std::unordered_map<std::string, FontPdfData> result;
if (!page) return result;
// ---- Phase 1: page object walk to collect real font properties ----
int objectCount = FPDFPage_CountObjects(page);
for (int i = 0; i < objectCount; ++i) {
FPDF_PAGEOBJECT obj = FPDFPage_GetObject(page, i);
if (!obj) continue;
if (FPDFPageObj_GetType(obj) != FPDF_PAGEOBJ_TEXT) continue;
FPDF_FONT font = FPDFTextObj_GetFont(obj);
if (!font) continue;
// Retrieve font name — first call returns required buffer size.
unsigned long nameLen = FPDFFont_GetBaseFontName(font, nullptr, 0);
if (nameLen == 0) continue;
std::vector<char> nameBuf(nameLen);
if (FPDFFont_GetBaseFontName(font, nameBuf.data(), nameLen) == 0) continue;
std::string fontName(nameBuf.data());
if (fontName.empty()) continue;
// Each font name is processed only once.
if (result.count(fontName)) continue;
FontPdfData data;
data.valid = true;
// FPDFFont_GetIsEmbedded returns 1 when the PDF contains an embedded
// font stream (/FontFile, /FontFile2, or /FontFile3 in the descriptor).
data.isEmbedded = (FPDFFont_GetIsEmbedded(font) != 0);
// FPDFFont_GetFlags returns the actual /Flags integer from the PDF
// FontDescriptor dictionary (not the text-rendering flags).
data.flags = FPDFFont_GetFlags(font);
// FPDFFont_GetAscent/Descent are scaled by font_size.
// Passing 1000.0 recovers the raw PDF 1000-unit-space values.
float rawAscent = 0.0f;
if (FPDFFont_GetAscent(font, 1000.0f, &rawAscent) && rawAscent != 0.0f) {
data.ascent = static_cast<double>(rawAscent);
}
float rawDescent = 0.0f;
if (FPDFFont_GetDescent(font, 1000.0f, &rawDescent) && rawDescent != 0.0f) {
data.descent = static_cast<double>(rawDescent);
}
result[fontName] = data;
}
// ---- Phase 2: Unicode mapping probe via text-page character scan ----
// For each known font, test whether PDFium can return a non-trivial Unicode
// value for at least one character that belongs to it. A valid decode
// implies the font has an active ToUnicode CMap or a built-in encoding map.
if (textPage && !result.empty()) {
int charCount = FPDFText_CountChars(textPage);
if (charCount > 0 && charCount < 1000000) {
for (int ci = 0; ci < charCount; ++ci) {
int fi = 0;
unsigned long flen =
FPDFText_GetFontInfo(textPage, ci, nullptr, 0, &fi);
if (flen == 0) continue;
std::vector<char> fbuf(flen);
if (FPDFText_GetFontInfo(textPage, ci, fbuf.data(), flen, &fi) == 0)
continue;
std::string fname(fbuf.data());
auto it = result.find(fname);
// Skip fonts not found in Phase 1, or already confirmed.
if (it == result.end() || it->second.hasUnicodeMapping) continue;
unsigned int cp = FPDFText_GetUnicode(textPage, ci);
// Require a printable codepoint — exclude NUL, control chars,
// and the Unicode replacement character (U+FFFD).
if (cp > 0x0020 && cp != 0xFFFD) {
it->second.hasUnicodeMapping = true;
}
}
}
}
return result;
}
#endif
void parseHexColor(const std::string& hex, unsigned int& r, unsigned int& g, unsigned int& b) {
@@ -1011,6 +1141,11 @@ std::expected<std::vector<FontInfo>, EngineError> PdfiumPage::getFonts() const {
return std::unexpected(EngineError::Unknown);
}
// Build real font data from the PDF font dictionary via page object walk.
// This provides accurate values for isEmbedded, subtype, flags, ascent,
// descent, and hasUnicodeMapping — replacing name-based heuristics.
auto fontPdfDataMap = buildFontPdfDataMap(page_, textPage_);
std::vector<FontInfo> pageFonts;
int charCount = FPDFText_CountChars(textPage_);
@@ -1034,12 +1169,13 @@ std::expected<std::vector<FontInfo>, EngineError> PdfiumPage::getFonts() const {
};
// Detect vertical writing mode per font by analyzing character origin positions.
// In vertical writing mode, consecutive characters of the same font move primarily
// in the Y direction. We track this per font name.
// We filter out large spatial jumps (e.g. line breaks or paragraph changes)
// by comparing the coordinate delta to the character's font size.
struct FontPositionStats {
double totalDeltaX = 0.0;
double totalDeltaY = 0.0;
int samples = 0;
int horizontalSteps = 0;
int verticalSteps = 0;
double totalFilteredDeltaX = 0.0;
double totalFilteredDeltaY = 0.0;
double prevX = 0.0, prevY = 0.0;
bool hasPrev = false;
};
@@ -1062,9 +1198,25 @@ std::expected<std::vector<FontInfo>, EngineError> PdfiumPage::getFonts() const {
if (FPDFText_GetCharOrigin(textPage_, i, &ox, &oy)) {
auto& stats = fontStats[fontName];
if (stats.hasPrev) {
stats.totalDeltaX += std::abs(ox - stats.prevX);
stats.totalDeltaY += std::abs(oy - stats.prevY);
stats.samples++;
double dx = std::abs(ox - stats.prevX);
double dy = std::abs(oy - stats.prevY);
double fontSize = FPDFText_GetFontSize(textPage_, i);
// Filter out non-consecutive jumps (e.g., line or paragraph breaks).
// We allow a generously large multiplier (3.0) for wide tracking,
// but enforce a minimum of 30.0 units for very small text.
double maxJump = (std::max)(fontSize * 3.0, 30.0);
if (dx < maxJump && dy < maxJump) {
stats.totalFilteredDeltaX += dx;
stats.totalFilteredDeltaY += dy;
if (dy > dx * 1.5) {
stats.verticalSteps++;
} else if (dx > dy * 1.5) {
stats.horizontalSteps++;
}
}
}
stats.prevX = ox;
stats.prevY = oy;
@@ -1073,16 +1225,21 @@ std::expected<std::vector<FontInfo>, EngineError> PdfiumPage::getFonts() const {
}
}
// Determine which fonts are vertical (Y movement dominates X movement)
// Determine which fonts are vertical
std::unordered_map<std::string, bool> fontIsVertical;
for (const auto& [fname, stats] : fontStats) {
if (stats.samples > 0) {
// Vertical if Y movement is substantially greater than X movement
// Use a threshold: Y > 2 * X and meaningful Y movement
bool vertical = (stats.totalDeltaY > 2.0 * stats.totalDeltaX) &&
(stats.totalDeltaY > 0.5);
fontIsVertical[fname] = vertical;
bool vertical = false;
// Primary heuristic: explicit step counts for consecutive characters
if (stats.verticalSteps > 0 || stats.horizontalSteps > 0) {
vertical = stats.verticalSteps > stats.horizontalSteps;
} else {
// Fallback: if no clear steps were isolated, rely on filtered totals
vertical = (stats.totalFilteredDeltaY > 2.0 * stats.totalFilteredDeltaX) &&
(stats.totalFilteredDeltaY > 0.5);
}
fontIsVertical[fname] = vertical;
}
// Second pass: build FontInfo list
@@ -1103,10 +1260,68 @@ std::expected<std::vector<FontInfo>, EngineError> PdfiumPage::getFonts() const {
FontInfo f;
f.fontName = fontName;
f.flags = static_cast<uint32_t>(charInfos[i].flags);
// Deduce advanced metadata from font name heuristics
// Phase A: heuristic baseline — fills encoding, cmapName, cidSystemInfo,
// normalizedFamily, subset tag, and initial values for all other fields.
deduceFontMetadata(f);
// Phase B: override heuristic fields with real PDF dictionary values
// wherever buildFontPdfDataMap() was able to extract them.
auto pdfDataIt = fontPdfDataMap.find(fontName);
if (pdfDataIt != fontPdfDataMap.end()) {
const FontPdfData& pd = pdfDataIt->second;
// --- Embedding status (replaces subset-tag heuristic) ---
f.isEmbedded = pd.isEmbedded;
// --- Font descriptor flags (real /Flags value) ---
f.flags = static_cast<uint32_t>(pd.flags);
// --- Descriptor metrics (real ascent/descent) ---
if (pd.ascent != 0.0) f.ascent = pd.ascent;
if (pd.descent != 0.0) f.descent = pd.descent;
// CapHeight is not exposed by the PDFium public API.
// Use the exact specification values for the standard 14 fonts;
// for everything else, estimate from real ascent (71% is the
// empirical Latin cap-height ratio across common typefaces).
if (f.ascent > 0.0) {
auto lname = fontName;
std::transform(lname.begin(), lname.end(), lname.begin(), ::tolower);
if (lname.find("times") != std::string::npos) f.capHeight = 662.0;
else if (lname.find("courier") != std::string::npos) f.capHeight = 562.0;
else if (lname.find("symbol") != std::string::npos) f.capHeight = 673.0;
else if (lname.find("helvetica") != std::string::npos) f.capHeight = 728.0;
else f.capHeight = f.ascent * 0.71;
}
// --- Unicode mapping (replaces always-true heuristic) ---
f.hasToUnicode = pd.hasUnicodeMapping;
// --- sourceType derived from real embedding status ---
if (f.isEmbedded) {
// Font stream is present in the PDF — always report as Embedded.
f.sourceType = "Embedded";
f.substitutedFrom = "";
f.substitutedTo = "";
}
// If not embedded, keep the substitution info already set by
// deduceFontMetadata() (SystemFallback or Substituted).
// --- Recalculate stable identifier with corrected data ---
if (f.isSubset && !f.subsetTag.empty()) {
f.internalFontId = f.subsetTag + "_" + f.fontName;
} else {
f.internalFontId = f.fontName + "_" + f.type + "_" + std::to_string(f.flags);
}
spdlog::debug(
"Font '{}': isEmbedded={} type='{}' ascent={:.1f} descent={:.1f} "
"capHeight={:.1f} hasToUnicode={} flags={}",
fontName, f.isEmbedded, f.type, f.ascent, f.descent,
f.capHeight, f.hasToUnicode, f.flags);
}
// Override isVertical with position-based detection if not already set by name heuristic
if (!f.isVertical) {
auto vit = fontIsVertical.find(fontName);
engine/src/parser/pdfium_document.hpp
+5
View File
@@ -85,4 +85,9 @@ private:
mutable std::mutex fontsMutex_;
};
// Exposed for testing
std::vector<unsigned short> utf8_to_utf16le(const std::string& utf8);
std::string utf16le_to_utf8(const char16_t* utf16, size_t length);
std::string code_point_to_utf8(unsigned int cp);
}
engine/tests/document_test.cpp
+358 -1
View File
@@ -1,12 +1,16 @@
#include <gtest/gtest.h>
#include <pdfengine/pdf_document.hpp>
#include <pdfengine/pdf_engine.hpp>
#include "parser/pdfium_document.hpp"
#include "fonts/pdf_fonts/encoding/cjk_collection_db.hpp"
#include <filesystem>
#include <fstream>
#include <vector>
#include <string>
#include <thread>
#include <atomic>
#include <chrono>
#include "fonts/cache/glyph_cache.hpp"
#ifndef TEST_CORPUS_DIR
#define TEST_CORPUS_DIR "../../corpus"
#endif
@@ -641,6 +645,23 @@ TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) {
}
}
// Test Part 2b: Vertical Font Detection Heuristic explicit validation
{
auto path1 = getCorpusPath("fonts", "vertical_identity_v.pdf");
if (std::filesystem::exists(path1)) {
auto docRes = PdfDocument::loadFromFile(path1.string());
ASSERT_TRUE(docRes.has_value());
auto fontsRes = (*docRes)->getFonts();
ASSERT_TRUE(fontsRes.has_value());
bool foundVertical = false;
for (const auto& f : *fontsRes) {
if (f.isVertical) foundVertical = true;
}
EXPECT_TRUE(foundVertical) << "Failed to detect vertical font in vertical_identity_v.pdf";
}
}
// Test Part 3: Font Size and Glyph Bounds Handling
{
auto path = getCorpusPath("fonts", "utf-8.pdf");
@@ -685,4 +706,340 @@ TEST(FontDiagnosticsTest, DeepIntrospectionAndFontSizeVerification) {
}
}
// =========================================================================
// Tests validating real PDFium font dictionary introspection.
// These tests verify that isEmbedded, type, ascent, descent, capHeight, and
// hasToUnicode are now derived from actual PDF font objects rather than from
// font-name heuristics (the behaviour that predated this change).
// =========================================================================
// Verify that embedded fonts report isEmbedded=true and that sourceType is
// set to "Embedded" from the real FPDFFont_GetIsEmbedded() result.
// A subset-embedded font (ABCDEF+FontName prefix) is the clearest case
// because the old heuristic relied solely on the prefix tag for embedding
// detection, while real PDFium checks for the /FontFile stream.
TEST(FontDiagnosticsTest, RealPDFiumEmbeddingAndTypeAccuracy) {
SKIP_IF_NO_PDFIUM();
// text_font.pdf has an embedded subset TrueType font \u2014 best candidate for
// verifying that isEmbedded comes from the PDF font stream, not the name tag.
auto path = getCorpusPath("fonts", "text_font.pdf");
if (!std::filesystem::exists(path)) {
GTEST_SKIP() << "text_font.pdf not found in corpus.";
}
auto docRes = PdfDocument::loadFromFile(path.string());
ASSERT_TRUE(docRes.has_value()) << "Failed to open text_font.pdf";
auto doc = *docRes;
auto pageRes = doc->getPage(0);
ASSERT_TRUE(pageRes.has_value());
auto fontsRes = (*pageRes)->getFonts();
ASSERT_TRUE(fontsRes.has_value());
const auto& fonts = *fontsRes;
ASSERT_FALSE(fonts.empty()) << "text_font.pdf must expose at least one font";
for (const auto& f : fonts) {
// Core invariant: every font must have a non-empty name and type.
EXPECT_FALSE(f.fontName.empty());
EXPECT_FALSE(f.type.empty());
// type must be one of the four valid PDF font subtypes.
static const std::vector<std::string> kValidTypes = {
"Type1", "TrueType", "CIDFontType0", "CIDFontType2"
};
bool typeValid = std::find(kValidTypes.begin(), kValidTypes.end(), f.type)
!= kValidTypes.end();
EXPECT_TRUE(typeValid) << "Unexpected type '" << f.type << "' for font '" << f.fontName << "'";
// Subset-prefixed fonts MUST be reported as embedded by PDFium
// (the /FontFile stream is required by the PDF spec when a subset tag is present).
if (f.isSubset) {
EXPECT_TRUE(f.isEmbedded)
<< "Subset font '" << f.fontName
<< "' must be embedded (FPDFFont_GetIsEmbedded should return 1)";
EXPECT_EQ(f.sourceType, "Embedded")
<< "sourceType must be 'Embedded' when isEmbedded=true";
EXPECT_TRUE(f.substitutedFrom.empty());
EXPECT_TRUE(f.substitutedTo.empty());
}
// isEmbedded=true and sourceType="Embedded" must be consistent.
if (f.isEmbedded) {
EXPECT_EQ(f.sourceType, "Embedded");
}
}
}
// Verify that font descriptor metrics (ascent, descent, capHeight) come from
// the real PDF FontDescriptor via FPDFFont_GetAscent/Descent(), not from the
// hardcoded fallback table. The critical invariant is sign correctness:
// ascent must be positive, descent must be negative.
TEST(FontDiagnosticsTest, RealPDFiumMetricsAccuracy) {
SKIP_IF_NO_PDFIUM();
// Use the largest font corpus file; it contains the most diverse fonts.
auto path = getCorpusPath("fonts", "text_font.pdf");
if (!std::filesystem::exists(path)) {
GTEST_SKIP() << "text_font.pdf not found in corpus.";
}
auto docRes = PdfDocument::loadFromFile(path.string());
ASSERT_TRUE(docRes.has_value());
auto fontsRes = (*docRes)->getFonts();
ASSERT_TRUE(fontsRes.has_value());
for (const auto& f : *fontsRes) {
// ascent and descent from FPDFFont_GetAscent/Descent(font, 1000.0f, …)
// are in PDF 1000-unit space. ascent is above the baseline (positive),
// descent is below (negative).
EXPECT_GT(f.ascent, 0.0)
<< "ascent must be positive for font '" << f.fontName << "'";
EXPECT_LT(f.descent, 0.0)
<< "descent must be negative for font '" << f.fontName << "'";
EXPECT_GT(f.capHeight, 0.0)
<< "capHeight must be positive for font '" << f.fontName << "'";
// capHeight must not exceed ascent (sanity: caps never taller than ascender).
EXPECT_LE(f.capHeight, f.ascent + 1.0) // +1 for float rounding
<< "capHeight should not exceed ascent for font '" << f.fontName << "'";
// Values must be in a plausible PDF 1000-unit-space range.
// Standard fonts typically have ascent in [400, 1200].
EXPECT_LT(f.ascent, 1500.0) << "Implausibly large ascent for '" << f.fontName << "'";
EXPECT_GT(f.descent, -1500.0) << "Implausibly deep descent for '" << f.fontName << "'";
}
}
// Verify that hasToUnicode reflects actual Unicode decode capability rather
// than the old always-true heuristic.
//
// with_tounicode.pdf \u2014 PDF containing a font that has a /ToUnicode stream;
// PDFium should decode characters successfully.
// no_tounicode.pdf \u2014 PDF containing a font with no /ToUnicode stream and no
// standard encoding; PDFium cannot map char codes to Unicode.
// latin_extended.pdf \u2014 Standard Latin font; must decode to Unicode via built-in
// encoding (WinAnsiEncoding or similar).
TEST(FontDiagnosticsTest, ToUnicodePresenceAccuracy) {
SKIP_IF_NO_PDFIUM();
// Case 1: font WITH ToUnicode \u2014 hasToUnicode must be true
{
auto path = getCorpusPath("fonts", "with_tounicode.pdf");
if (std::filesystem::exists(path)) {
auto docRes = PdfDocument::loadFromFile(path.string());
if (docRes.has_value()) {
auto pageRes = (*docRes)->getPage(0);
if (pageRes.has_value()) {
auto fontsRes = (*pageRes)->getFonts();
if (fontsRes.has_value() && !fontsRes->empty()) {
bool anyTrue = false;
for (const auto& f : *fontsRes) {
if (f.hasToUnicode) { anyTrue = true; break; }
}
EXPECT_TRUE(anyTrue)
<< "At least one font in with_tounicode.pdf must have hasToUnicode=true";
}
}
}
}
}
// Case 2: font WITHOUT ToUnicode or decodable encoding \u2014 hasToUnicode must be false
{
auto path = getCorpusPath("fonts", "no_tounicode.pdf");
if (std::filesystem::exists(path)) {
auto docRes = PdfDocument::loadFromFile(path.string());
if (docRes.has_value()) {
auto pageRes = (*docRes)->getPage(0);
if (pageRes.has_value()) {
auto fontsRes = (*pageRes)->getFonts();
if (fontsRes.has_value() && !fontsRes->empty()) {
// All fonts in a no-tounicode document should fail Unicode decode.
for (const auto& f : *fontsRes) {
EXPECT_FALSE(f.hasToUnicode)
<< "Font '" << f.fontName
<< "' in no_tounicode.pdf must have hasToUnicode=false";
}
}
}
}
}
}
// Case 3: standard Latin font \u2014 must decode to Unicode via built-in encoding
{
auto path = getCorpusPath("fonts", "latin_extended.pdf");
if (std::filesystem::exists(path)) {
auto docRes = PdfDocument::loadFromFile(path.string());
if (docRes.has_value()) {
auto pageRes = (*docRes)->getPage(0);
if (pageRes.has_value()) {
auto fontsRes = (*pageRes)->getFonts();
if (fontsRes.has_value() && !fontsRes->empty()) {
bool anyTrue = false;
for (const auto& f : *fontsRes) {
if (f.hasToUnicode) { anyTrue = true; break; }
}
EXPECT_TRUE(anyTrue)
<< "At least one font in latin_extended.pdf must decode to Unicode";
}
}
}
}
}
}
// =========================================================================
// Tests for UTF-16 Surrogate Pairs (Emoji and CJK Ext-B)
// =========================================================================
TEST(UtfConversionTest, EmojiSurrogatePairs) {
// 😀 U+1F600 -> UTF-8: F0 9F 98 80
std::string utf8_grinning = "\xF0\x9F\x98\x80";
auto utf16 = pdfengine::parser::utf8_to_utf16le(utf8_grinning);
// Should be D83D DE00 + null terminator
ASSERT_EQ(utf16.size(), 3);
EXPECT_EQ(utf16[0], 0xD83D);
EXPECT_EQ(utf16[1], 0xDE00);
EXPECT_EQ(utf16[2], 0x0000);
// Convert back to UTF-8
std::string utf8_out = pdfengine::parser::utf16le_to_utf8(reinterpret_cast<const char16_t*>(utf16.data()), utf16.size() - 1);
EXPECT_EQ(utf8_out, utf8_grinning);
// 🚀 U+1F680 -> UTF-8: F0 9F 9A 80
std::string utf8_rocket = "\xF0\x9F\x9A\x80";
utf16 = pdfengine::parser::utf8_to_utf16le(utf8_rocket);
ASSERT_EQ(utf16.size(), 3);
EXPECT_EQ(utf16[0], 0xD83D);
EXPECT_EQ(utf16[1], 0xDE80);
EXPECT_EQ(utf16[2], 0x0000);
utf8_out = pdfengine::parser::utf16le_to_utf8(reinterpret_cast<const char16_t*>(utf16.data()), utf16.size() - 1);
EXPECT_EQ(utf8_out, utf8_rocket);
}
TEST(UtfConversionTest, CJKExtensionB) {
// 𠀀 U+20000 -> UTF-8: F0 A0 80 80
std::string utf8_cjk = "\xF0\xA0\x80\x80";
auto utf16 = pdfengine::parser::utf8_to_utf16le(utf8_cjk);
// Should be D840 DC00 + null terminator
ASSERT_EQ(utf16.size(), 3);
EXPECT_EQ(utf16[0], 0xD840);
EXPECT_EQ(utf16[1], 0xDC00);
EXPECT_EQ(utf16[2], 0x0000);
// Convert back to UTF-8
std::string utf8_out = pdfengine::parser::utf16le_to_utf8(reinterpret_cast<const char16_t*>(utf16.data()), utf16.size() - 1);
EXPECT_EQ(utf8_out, utf8_cjk);
}
TEST(UtfConversionTest, RoundtripMixed) {
// "A😀B𠀀C" -> 41 F0 9F 98 80 42 F0 A0 80 80 43
std::string mixed = "A\xF0\x9F\x98\x80""B\xF0\xA0\x80\x80""C";
auto utf16 = pdfengine::parser::utf8_to_utf16le(mixed);
std::string mixed_out = pdfengine::parser::utf16le_to_utf8(reinterpret_cast<const char16_t*>(utf16.data()), utf16.size() - 1);
EXPECT_EQ(mixed_out, mixed);
}
// =========================================================================
// Tests for CJK CID Resolution
// =========================================================================
TEST(CjkResolutionTest, AdobeCNS1) {
// Basic mapping checks for the core Adobe-CNS1 block (Traditional Chinese)
using pdfengine::fonts::pdf_fonts::CjkCollectionDB;
// Test existing block (100-130)
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 100), 0x4E00); // 一
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 112), 0x4E2D); // 中
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 130), 0x4ED7); // 仗
// Test the newly expanded block (131-140)
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 131), 0x4ED8); // 付
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 135), 0x4EDF); // 仟
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-CNS1", 140), 0x4F01); // 企
// Test alternative naming
EXPECT_EQ(CjkCollectionDB::resolveCID("Identity-H-CNS1", 137), 0x4EE3); // 代
}
TEST(CjkResolutionTest, AdobeKorea1) {
using pdfengine::fonts::pdf_fonts::CjkCollectionDB;
// Test existing Korean block (101-150)
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 101), 0xAC00); // 가
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 119), 0xAC1C); // 개
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 150), 0xAC90); // 겔
// Test newly added Hangul block (151-160)
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 151), 0xAC94); // 겝
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 156), 0xACA9); // 결
EXPECT_EQ(CjkCollectionDB::resolveCID("Adobe-Korea1", 160), 0xACBD); // 겼
// Test alternative naming
EXPECT_EQ(CjkCollectionDB::resolveCID("UniKS-UTF16-H-Korea1", 153), 0xACA0); // 겠
}
// =========================================================================
// GlyphCache Concurrency & Benchmark Test
// =========================================================================
TEST(GlyphCacheTest, ConcurrencyBench) {
using namespace pdfengine::fonts;
FontFace face;
// Load a common system font for testing cache keys
bool loaded = face.loadFromFile("C:\\Windows\\Fonts\\arial.ttf");
if (!loaded) {
GTEST_SKIP() << "Skipping benchmark: Arial font not found.";
}
GlyphCache cache(1000);
auto run_benchmark = [&](int num_threads, int ops_per_thread) {
std::atomic<int> start_flag{0};
std::vector<std::thread> threads;
for (int i = 0; i < num_threads; ++i) {
threads.emplace_back([&, i]() {
while (start_flag.load() == 0) { std::this_thread::yield(); }
for (int op = 0; op < ops_per_thread; ++op) {
unsigned int glyphIndex = (op + i) % 2000;
unsigned int fontSize = 12 + (op % 5);
auto hit = cache.get(face, glyphIndex, fontSize);
if (!hit) {
GlyphBitmap bmp;
bmp.width = 10; bmp.height = 10;
cache.insert(face, glyphIndex, fontSize, bmp);
}
}
});
}
auto start_time = std::chrono::high_resolution_clock::now();
start_flag.store(1);
for (auto& t : threads) { t.join(); }
auto end_time = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> diff = end_time - start_time;
return diff.count();
};
run_benchmark(2, 1000); // warmup
cache.clear();
double time_10 = run_benchmark(10, 10000);
std::cout << "[ BENCHMARK ] 10 Threads Time: " << time_10 << " seconds (" << (100000.0 / time_10) << " ops/sec)\n";
cache.clear();
double time_50 = run_benchmark(50, 10000);
std::cout << "[ BENCHMARK ] 50 Threads Time: " << time_50 << " seconds (" << (500000.0 / time_50) << " ops/sec)\n";
EXPECT_LE(cache.size(), 1000 + 16); // Accommodate shard capacity rounding
}
}