Harden conversion engine: typed errors, output validation, OCR cache, warning gate

Reviewed-on: https://gitea.maskantech.in/gitea_admin/pdf/pulls/83

.dockerignore

@@ -0,0 +1,31 @@
+**/.git
+**/.github
+**/.venv
+**/node_modules
+**/dist
+**/build
+**/out
+**/__pycache__
+**/*.pyc
+**/*.pyo
+**/*.pyd
+**/*.log
+**/.DS_Store
+**/Thumbs.db
+**/.vscode
+**/.idea
+**/coverage
+**/tmp
+corpus/
+**/.pytest_cache
+**/.mypy_cache
+**/.ruff_cache
+**/CMakeUserPresets.json
+**/compile_commands.json
+**/vcpkg
+**/third_party/pdfium/depot_tools
+**/third_party/pdfium/checkout
+**/third_party/pdfium/install
+**/third_party/skia/depot_tools
+**/third_party/skia/checkout
+**/third_party/skia/install

.github/workflows/ci.yml

@@ -1,162 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [main, develop]
-  pull_request:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ci-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Rule R2 — PDFium boundary check
-        run: bash scripts/check_pdfium_boundary.sh
-
-      - name: Install clang-format (pinned)
-        run: pipx install clang-format==22.1.5
-
-      - name: clang-format
-        run: |
-          clang-format --version
-          find engine \( -name '*.cpp' -o -name '*.cc' -o -name '*.h' -o -name '*.hpp' \) \
-            -print0 | xargs -0 clang-format --dry-run --Werror
-
-  build:
-    needs: lint
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            preset: linux-debug
-          - os: macos-latest
-            preset: macos-debug
-            experimental: true
-          - os: windows-latest
-            preset: windows-debug
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental == true }}
-    env:
-      VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/.vcpkg-cache
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Ninja
-        uses: seanmiddleditch/gha-setup-ninja@v5
-
-      - name: Set up MSVC environment
-        if: runner.os == 'Windows'
-        uses: ilammy/msvc-dev-cmd@v1
-
-      - name: Locate vcpkg
-        shell: bash
-        run: |
-          echo "VCPKG_ROOT=$VCPKG_INSTALLATION_ROOT" >> "$GITHUB_ENV"
-          git -C "$VCPKG_INSTALLATION_ROOT" fetch --quiet origin || true
-
-      - name: Create vcpkg binary cache dir
-        shell: bash
-        run: mkdir -p "$VCPKG_DEFAULT_BINARY_CACHE"
-
-      - name: Cache vcpkg artifacts
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.VCPKG_DEFAULT_BINARY_CACHE }}
-          key: vcpkg-${{ matrix.os }}-${{ hashFiles('vcpkg.json') }}
-          restore-keys: vcpkg-${{ matrix.os }}-
-
-      - name: Pin vcpkg dependency baseline
-        shell: bash
-        run: |
-          if ! grep -q '"builtin-baseline"' vcpkg.json; then
-            "$VCPKG_ROOT/vcpkg" x-update-baseline --add-initial-baseline
-          fi
-
-      - name: Configure
-        run: cmake --preset ${{ matrix.preset }}
-
-      - name: Build
-        run: cmake --build --preset ${{ matrix.preset }}
-
-      - name: Test
-        run: ctest --preset ${{ matrix.preset }}
-
-  gateway:
-    needs: lint
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: gateway
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-          cache: pip
-          cache-dependency-path: gateway/pyproject.toml
-
-      - name: Install gateway (editable, with dev extras)
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install -e ".[dev]"
-
-      - name: Ruff — lint
-        run: python -m ruff check .
-
-      - name: Ruff — format check
-        run: python -m ruff format --check .
-
-      - name: Pytest
-        run: python -m pytest
-
-
-  wasm:
-    needs: lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Read pinned emsdk version
-        id: emsdk-version
-        run: |
-          version=$(grep '^EMSDK_VERSION=' wasm/emsdk.pinned | cut -d= -f2)
-          if [ -z "$version" ]; then
-            echo "::error::EMSDK_VERSION not found in wasm/emsdk.pinned"
-            exit 1
-          fi
-          echo "version=$version" >> "$GITHUB_OUTPUT"
-
-      - name: Install Ninja
-        uses: seanmiddleditch/gha-setup-ninja@v5
-
-      - name: Set up Emscripten ${{ steps.emsdk-version.outputs.version }}
-        uses: mymindstorm/setup-emsdk@v14
-        with:
-          version: ${{ steps.emsdk-version.outputs.version }}
-          actions-cache-folder: emsdk-cache-${{ steps.emsdk-version.outputs.version }}
-
-      - name: Set up Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: "20"
-
-      - name: Configure (WASM)
-
-        run: cmake --preset wasm
-
-      - name: Build
-        run: cmake --build --preset wasm
-
-      - name: Smoke test
-        run: node wasm/hello.test.mjs

.gitignore

@@ -22,6 +22,11 @@ CMakeUserPresets.json
 /third_party/pdfium/install/
 /third_party/pdfium/.gclient*
 
+# Skia from-source build (depot_tools / GN / Ninja)
+/third_party/skia/depot_tools/
+/third_party/skia/checkout/
+/third_party/skia/install/
+
 # IDE / editor
 /.vs/
 /.vscode/
@@ -51,10 +56,88 @@ node_modules/
 /frontend/.vite/
 
 # WASM artifacts
-*.wasm
+
 *.wasm.map
 
 # Logs / misc
 *.log
 
-PDF Editor Timeline.xlsx
+# Compiled Python bindings (binary build outputs)
+gateway/*.pyd
+gateway/*.so
+gateway/*.dylib
+
+# Corpora — keep convert metadata; ignore large samples / archives
+/corpus/*
+!/corpus/convert/
+/corpus/convert/**
+!/corpus/convert/real/
+!/corpus/convert/public/
+!/corpus/convert/third_party/
+/corpus/convert/real/**
+!/corpus/convert/real/pack.json
+!/corpus/convert/real/*.md
+/corpus/convert/public/**
+!/corpus/convert/public/README.md
+!/corpus/convert/public/pack_public.json
+!/corpus/convert/public/manifests/
+!/corpus/convert/public/manifests/**
+!/corpus/convert/public/reports/
+!/corpus/convert/public/reports/**/*.md
+!/corpus/convert/public/reports/**/*.json
+/corpus/convert/public/samples/
+/corpus/convert/third_party/**
+!/corpus/convert/third_party/NOTICE
+!/corpus/convert/third_party/LICENSE_CHECKLIST.md
+!/corpus/convert/third_party/checksums.sha256
+corpus/fuzz/
+# Layout ML weights (download via scripts/convert/fetch_layout_onnx.py)
+/gateway/models/layout/**/*.onnx
+/gateway/models/layout/**/*.pdmodel
+/gateway/models/layout/**/*.pdiparams
+!/gateway/models/layout/**/README.md
+!/gateway/models/layout/**/.gitkeep
+# Arabic OCR rec weights (download via scripts/convert/fetch_ocr_arabic_onnx.py)
+/gateway/models/ocr/**/*.onnx
+!/gateway/models/ocr/**/README.md
+!/gateway/models/ocr/**/.gitkeep
+!/gateway/models/ocr/**/*.txt
+# DocLayNet / Pub* lab archives
+**/DocLayNet*.zip
+**/PubLayNet*
+**/PubTabNet*
+# Large-corpus regression baseline (derived from the gitignored corpus above)
+tests/regression/baseline-large/
+# Fuzzer working artifacts (crashes, leaks, coverage)
+engine/fuzz/artifacts/
+crash-*
+leak-*
+timeout-*
+
+PDF Editor Timeline.xlsx
+# Operator convert score log (Phase 0); never commit customer DOCX
+convert-scores.jsonl
+**/convert-scores.jsonl
+
+# Local environment config / secrets
+/third_party/pdfium-wasm/
+
+# AI Large Model Weights & Vector Indexes (never commit binary model weights into Git)
+models/**/*.onnx
+models/**/*.bin
+models/**/*.pt
+models/**/*.safetensors
+models/**/*.index
+!models/**/.gitkeep
+.github
+
+# Runtime job store. gateway/data/jobs.sqlite3 is a live SQLite database the
+# service writes to on every job — including every test run — so a tracked copy
+# shows up as a modified file after any use of the gateway, and can carry job
+# rows (customer filenames among them) into the repository. It is state, not
+# source; the schema is created on startup.
+gateway/data/jobs.sqlite3
+gateway/data/jobs.sqlite3-wal
+gateway/data/jobs.sqlite3-shm
+gateway/data/jobs_spool/
+!gateway/data/jobs_spool/.gitkeep

CMakeLists.txt

@@ -32,29 +32,54 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
 if(EMSCRIPTEN)
-    message(STATUS "PdfEngine ${PROJECT_VERSION} — WASM hello-world configuration (Phase 0)")
-    add_subdirectory(wasm)
-    return()
-endif()
-
-# Only the top-level project drives testing/install defaults.
-if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR)
-    message(FATAL_ERROR "In-source builds are not allowed. Use a preset: "
-                        "cmake --preset <platform>-debug")
+    set(PDFENGINE_WASM ON CACHE INTERNAL "Building for WebAssembly")
 endif()
 
 option(PDFENGINE_BUILD_TESTS      "Build engine unit/smoke tests"                ON)
 option(PDFENGINE_ENABLE_SANITIZERS "Build with AddressSanitizer/UBSan"           OFF)
 option(PDFENGINE_WARNINGS_AS_ERRORS "Treat compiler warnings as errors"          OFF)
 option(PDFENGINE_WITH_PDFIUM      "Link the PDFium static lib (build it first)"  OFF)
+option(PDFENGINE_WITH_SKIA        "Link the Skia static lib (build it first)"    OFF)
+option(PDFENGINE_WITH_QPDF        "Link QPDF for content stream extraction"      OFF)
+option(PDFENGINE_FUZZING          "Build libFuzzer harnesses (requires Clang)"   OFF)
+
+if(PDFENGINE_FUZZING)
+    # The fuzz harness instruments the engine; tests/bindings are not part of it.
+    set(PDFENGINE_BUILD_TESTS OFF CACHE BOOL "Build engine unit/smoke tests" FORCE)
+    if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+        message(FATAL_ERROR "PDFENGINE_FUZZING requires a Clang/clang-cl toolchain "
+                            "(set in the 'fuzz-*' preset).")
+    endif()
+endif()
+
+if(PDFENGINE_WASM)
+    set(PDFENGINE_BUILD_TESTS OFF CACHE BOOL "Build engine unit/smoke tests" FORCE)
+    set(PDFENGINE_ENABLE_SANITIZERS OFF CACHE BOOL "ASan/UBSan" FORCE)
+    # PDFium under WASM is opt-in via the wasm-pdfium preset (links the prebuilt wasm32
+    # libpdfium.a). The plain 'wasm' preset leaves PDFENGINE_WITH_PDFIUM=OFF (mock engine).
+endif()
 
 include(CompilerWarnings)
 include(Sanitizers)
-include(pdfium)        # defines pdfium::pdfium when PDFENGINE_WITH_PDFIUM is ON
+
+if(PDFENGINE_WITH_PDFIUM)
+    include(pdfium)        # defines pdfium::pdfium when PDFENGINE_WITH_PDFIUM is ON
+endif()
+if(PDFENGINE_WITH_SKIA)
+    include(skia)          # defines skia::skia when PDFENGINE_WITH_SKIA is ON
+endif()
 
 find_package(freetype  CONFIG REQUIRED)
 find_package(harfbuzz  CONFIG REQUIRED)
 find_package(spdlog    CONFIG REQUIRED)
+find_package(nlohmann_json CONFIG REQUIRED)
+find_package(qpdf CONFIG REQUIRED)
+find_package(ZLIB REQUIRED)
+find_package(JPEG REQUIRED)
+if(WIN32 AND DEFINED VCPKG_TARGET_TRIPLET)
+    link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/lib")
+    link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/debug/lib")
+endif()
 
 if(PDFENGINE_BUILD_TESTS)
     find_package(GTest CONFIG REQUIRED)
@@ -62,8 +87,22 @@ if(PDFENGINE_BUILD_TESTS)
     include(GoogleTest)
 endif()
 
-add_subdirectory(engine)
+if(PDFENGINE_WASM)
+    message(STATUS "PdfEngine ${PROJECT_VERSION} — WASM configuration (Phase 1/2 Enabled)")
+    add_subdirectory(engine)
+    add_subdirectory(wasm)
+else()
+    # Only the top-level project drives testing/install defaults.
+    if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR)
+        message(FATAL_ERROR "In-source builds are not allowed. Use a preset: "
+                            "cmake --preset <platform>-debug")
+    endif()
 
+    add_subdirectory(engine)
+    if(NOT PDFENGINE_FUZZING)
+        add_subdirectory(bindings)
+    endif()
+endif()
 
 message(STATUS "PdfEngine ${PROJECT_VERSION} configured")
 message(STATUS "  C++ standard ......... ${CMAKE_CXX_STANDARD}")
@@ -71,3 +110,5 @@ message(STATUS "  Build tests .......... ${PDFENGINE_BUILD_TESTS}")
 message(STATUS "  Sanitizers ........... ${PDFENGINE_ENABLE_SANITIZERS}")
 message(STATUS "  Warnings as errors ... ${PDFENGINE_WARNINGS_AS_ERRORS}")
 message(STATUS "  Link PDFium .......... ${PDFENGINE_WITH_PDFIUM}")
+message(STATUS "  Link Skia ............ ${PDFENGINE_WITH_SKIA}")
+message(STATUS "  Link QPDF ............ ${PDFENGINE_WITH_QPDF}")

CMakePresets.json

@@ -102,12 +102,47 @@
       "displayName": "WASM • Emscripten hello-world (Phase 0, Rule R5)",
       "generator": "Ninja",
       "binaryDir": "${sourceDir}/out/build/${presetName}",
-      "toolchainFile": "${sourceDir}/cmake/toolchains/wasm.cmake",
+      "toolchainFile": "${sourceDir}/cmake/toolchains/vcpkg-wasm.cmake",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
         "PDFENGINE_BUILD_TESTS": "OFF",
         "PDFENGINE_WITH_PDFIUM": "OFF"
       }
+    },
+
+    {
+      "name": "wasm-pdfium",
+      "displayName": "WASM • Emscripten + PDFium (live-preview engine)",
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/out/build/${presetName}",
+      "toolchainFile": "${sourceDir}/cmake/toolchains/vcpkg-wasm.cmake",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "PDFENGINE_BUILD_TESTS": "OFF",
+        "PDFENGINE_WITH_PDFIUM": "ON",
+        "PDFENGINE_WITH_QPDF": "ON"
+      }
+    },
+
+    {
+      "name": "fuzz-linux",
+      "displayName": "Linux • libFuzzer (Clang + ASan)",
+      "inherits": "base",
+      "condition": { "type": "equals", "lhs": "${hostSystemName}", "rhs": "Linux" },
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "PDFENGINE_FUZZING": "ON",
+        "PDFENGINE_WITH_PDFIUM": "ON",
+        "PDFENGINE_FUZZ_SANITIZERS": "fuzzer,address,undefined"
+      }
+    },
+    {
+      "name": "fuzz-linux-nosan",
+      "displayName": "Linux • libFuzzer (Clang, coverage-only — for non-ASan PDFium)",
+      "inherits": "fuzz-linux",
+      "cacheVariables": { "PDFENGINE_FUZZ_SANITIZERS": "fuzzer" }
     }
   ],
 
@@ -121,7 +156,10 @@
     { "name": "macos-debug",     "configurePreset": "macos-debug" },
     { "name": "macos-release",   "configurePreset": "macos-release" },
     { "name": "macos-asan",      "configurePreset": "macos-asan" },
-    { "name": "wasm",            "configurePreset": "wasm" }
+    { "name": "wasm",            "configurePreset": "wasm" },
+    { "name": "wasm-pdfium",     "configurePreset": "wasm-pdfium" },
+    { "name": "fuzz-linux",       "configurePreset": "fuzz-linux" },
+    { "name": "fuzz-linux-nosan", "configurePreset": "fuzz-linux-nosan" }
   ],
 
   "testPresets": [

CMakeUserPresets.json.template

@@ -0,0 +1,34 @@
+{
+  "version": 6,
+  "cmakeMinimumRequired": { "major": 3, "minor": 25, "patch": 0 },
+  "configurePresets": [
+    {
+      "name": "win-local",
+      "displayName": "Windows • Debug (local — build dir outside OneDrive/spaces)",
+      "inherits": "windows-debug",
+      "binaryDir": "D:/pdfeng-build/win-local",
+      "cacheVariables": {
+        "PDFENGINE_WITH_PDFIUM": "ON",
+        "PDFENGINE_WITH_QPDF": "ON"
+      }
+    },
+    {
+      "name": "win-local-pdfium",
+      "displayName": "Windows • RelWithDebInfo + PDFium (static CRT — build dir outside OneDrive/spaces)",
+      "inherits": "windows-release",
+      "binaryDir": "D:/pdfeng-build/win-local-pdfium",
+      "cacheVariables": {
+        "PDFENGINE_WITH_PDFIUM": "ON",
+        "PDFENGINE_WITH_QPDF": "ON"
+      }
+    }
+  ],
+  "buildPresets": [
+    { "name": "win-local", "configurePreset": "win-local" },
+    { "name": "win-local-pdfium", "configurePreset": "win-local-pdfium" }
+  ],
+  "testPresets": [
+    { "name": "win-local", "inherits": "common", "configurePreset": "win-local" },
+    { "name": "win-local-pdfium", "inherits": "common", "configurePreset": "win-local-pdfium" }
+  ]
+}

CONVERSION_ENGINE_DEEP_DIVE.md

@@ -0,0 +1,517 @@
+# Document Conversion Engine: End-to-End Deep Dive
+
+**Investigation date:** 2026-09-09  
+**Scope:** `pdf/` repository, with emphasis on `gateway/app/services/convert`, the
+PDF engine binding, OCR/layout models, writers, configuration, and conversion tests.  
+**Evidence standard:** real documents were passed through the public
+`run_conversion` path and the resulting Office files were parsed and rendered.
+Numbers below are measurements on the Windows development host, not vendor
+claims or a declaration of production readiness.
+
+## Current State
+
+### Executive finding
+
+The engine is a composed, mostly in-process conversion stack. It is effective for
+simple digital PDFs, text extraction, Office document generation, and searchable
+PDF creation. It is not a general PDF layout clone. A PDF has positioned drawing
+operators, while DOCX/XLSX require a semantic flow/grid model; the conversion
+therefore reconstructs an intermediate document model (IDM) and necessarily loses
+some semantics and positioning.
+
+The most visible historical defect, interleaved text on multi-column PDFs, had a
+specific upstream cause and is fixed in the current tree. The remaining quality
+ceiling is dominated by table reconstruction, OCR cost/ambiguity, and the choice
+of ReportLab as the Office-to-PDF fallback. A successful HTTP response is not
+equivalent to faithful content or layout: the measured corpus still contains
+low-order similarity and three failing assertions.
+
+### Execution topology
+
+The current request path is:
+
+```text
+HTTP /v1/convert/*
+  -> filename/MIME/magic-byte/size validation
+  -> converter registry and optional automatic PDF policy
+  -> bounded worker admission (queue + shared conversion slots)
+  -> conversion/document-cache scopes
+  -> PDF route (digital, scanned, image-based, mixed)
+  -> per-page geometry and content recovery
+       C++ pdfengine ordered glyphs (preferred)
+       pypdf visitor/content-stream geometry (fallback)
+       line grouping -> paragraphs -> reading order -> tables/images
+       optional ONNX layout regions and table separators
+  -> OCR for scan/hybrid pages
+       PDF render -> RapidOCR Latin pass -> optional Arabic pass
+       confidence/script merge -> IDM blocks and metadata
+  -> IDM optimization (headers/footers, wrapped rows, figures)
+  -> format-specific formatter/writer
+  -> OOXML/PDF validity checks, quality score and warnings
+  -> content-addressed result cache
+```
+
+`run_conversion` binds cancellation, deadlines, options, automatic-routing
+decisions, and a per-conversion read/raster/geometry cache. The public converter
+registry currently covers PDF, DOCX, XLSX, PPTX, HTML, Markdown, text, CSV,
+JSON, and image paths; the exact pair list is exposed by the convert info route.
+
+### Component inventory
+
+| Layer | Implementation in this tree | Observed role and limitation |
+| --- | --- | --- |
+| PDF parser/render core | C++23 `pdfengine` pybind module; PDFium, FreeType, HarfBuzz, optional Skia/QPDF | Gives authoritative glyphs, fonts, page geometry, rendering, and editing on this host. Native availability is not guaranteed on every Python deployment. |
+| Python PDF parser | `pypdf==5.3.0` | Text extraction, page count, content-stream fallback, XObject inspection. Its inferred whitespace and broken ToUnicode output cannot be treated as ground truth. |
+| Raster path | `pypdfium2==5.13.0` declared in `pyproject.toml` | Required for hosts without the C++ renderer. It is absent from the active venv, so raster-specific tests skip there; the C++ binding supplies rendering for the current audit. |
+| OCR | `rapidocr-onnxruntime==1.2.3`, `onnxruntime==1.28.0` | Shared Latin detector/recognizer; optional PP-OCRv5 Arabic recognizer. Inference is serialized because RapidOCR sessions mutate shared state. |
+| Layout ML | PP-DocLayoutV3-derived ONNX, 130,502,330 bytes FP32 or 34,694,580 bytes INT8 | Region labels narrow heuristic decisions; it does not reconstruct a document by itself. It runs on CPU in the measured environment. |
+| Table ML | Optional `structure.onnx` (Table Transformer/SLANet operator-supplied) | Not vendored (`shipped: false` in `models/MANIFEST.json`); heuristic lattice/stream/rectangle detectors remain the default. |
+| IDM | `app/services/convert/idm/model.py` and layout pipeline | Stores pages, BBoxes, paragraphs, headings, headers/footers, figures, and cell grids. It is a useful interchange contract, but not a complete PDF display list or OOXML style graph. |
+| Office writers | `python-docx==1.1.2`, `openpyxl==3.1.5`, `python-pptx` code, XML sanitization | Good for creating valid Office files; cannot infer all original PDF semantics (fonts, fields, anchored objects, section behavior). |
+| Office -> PDF | Gotenberg/LibreOffice attempt, then in-process ReportLab 4.2.5 | Gotenberg is best fidelity when reachable. The benchmark used the ReportLab fallback, which is concurrent-safe but not a Word/Excel print-layout clone. |
+| Images | Pillow 10.4.0, PDFium render/XObject extraction | Images are extracted, resized/compacted, or the complete page is embedded when reconstruction would lose visible content. |
+| Font recognition | `FontRecognitionService` facade, classifier and embedding store | The checked-in `FontClassifier` uses aspect ratio/dark-pixel heuristics and `EmbeddingStore` uses a deterministic 16x8 grayscale projection. `load_model()` reads metadata but does not load an OpenCLIP ONNX session; `model.onnx` and `fonts.index` are not present in the repository. This is metadata assistance, not a validated font-identification model. |
+
+The declared dependency set is intentionally permissive (pypdf BSD, pypdfium2
+Apache/BSD, python-docx/openpyxl MIT, ReportLab BSD, Pillow HPND). `mammoth==1.8.0`
+is installed in this venv for DOCX/HTML fallback paths; `fitz`/PyMuPDF is absent,
+which avoids an AGPL transitive dependency but removes that optional path.
+
+### Runtime configuration and resource policy
+
+The startup script defaults to automatic reconstruction, layout ML (INT8 when
+available), Arabic OCR with adaptive gating, a 1,800 second maximum conversion
+timeout, 22 seconds per OCR page budget, 45 seconds per-page timeout, a 1,200
+second OCR wall budget, four conversion workers, four simultaneous conversion
+slots, and a queue of roughly two batches. Result caching uses namespace
+`conversion-engine-v2` and includes effective options and routing mode in the key.
+
+These are protection mechanisms, not throughput guarantees. A single process can
+still hold multiple ONNX sessions and large page rasters, and OCR inference is
+deliberately serialized. Horizontal scaling or process isolation is needed for
+untrusted large-document workloads.
+
+### Jobs, storage, and failure boundaries
+
+Synchronous and asynchronous routes share a process-wide `ThreadPoolExecutor`
+and bounded semaphores. Asynchronous job state is persisted in SQLite with WAL
+and `synchronous=NORMAL`; results larger than 512 KB are spooled to a local
+`jobs_spool` directory, while smaller results are stored as BLOBs. Jobs older
+than three hours are purged, and jobs left queued/running after a process restart
+are marked failed. This gives useful single-node crash recovery, but SQLite and
+local spool paths are not a multi-instance object store: replicas need a shared
+database/object store or a queue service before horizontal scaling.
+
+The timeout helper releases the HTTP caller on deadline by abandoning a worker
+thread (`ThreadPoolExecutor.shutdown(wait=False)`). Python cannot kill that
+thread, so a native render/OCR operation may continue in the background until it
+returns. Cooperative cancellation is checked at page/OCR boundaries, which
+limits normal work, but a hung native call can still consume CPU/RAM after a 504.
+Production deployments should put untrusted conversions behind process/container
+boundaries with an OS-level kill and cgroup limits.
+
+Gotenberg/LibreOffice is attempted for Office-to-PDF with a 30-second client
+timeout and fails open to ReportLab. This makes the endpoint available when the
+microservice is absent, but it can silently change fidelity class unless the
+response warning is surfaced to callers and monitored.
+
+## Root Cause
+
+The table below ranks causes by customer impact and likelihood in the measured
+system. "Fixed" means the implementation changed and the targeted regression
+improved; it does not mean the entire class of failures is solved.
+
+| Rank | Severity | Root cause | Evidence | State |
+| --- | --- | --- | --- | --- |
+| 1 | P0 | Precise glyph coverage was compared with `len(pypdf.extract_text())`, which includes inferred spaces/newlines, while the precise walker emits text-showing characters only. Pages near the 0.80 gate fell into `_glyphs_from_visitor`, whose width calibration measured the two-column pitch rather than font advances. | Mozilla pages had coverage 0.754-0.858; stripping whitespace raised it to 0.879-1.000. Estimated spans crossed the gutter; the visible sentence was interleaved. | Fixed by comparing non-whitespace characters and retaining the precise path. |
+| 2 | P0 | Baseline clustering was page-global. Different leading in adjacent columns let rows "bridge" into one bucket; splitting columns later could not undo the vertical weld. | On the paper page, 6.97 pt copyright rows and 8.97 pt body rows were interleaved; 4.6 pt and 3.3 pt hops were under the global tolerance. Reading-order inversions fell from 189 to 1 after column-aware bands. | Fixed with `_column_band` and a bounded baseline clip. |
+| 3 | P0 | OCR is CPU-heavy and each bilingual page can run Latin and Arabic recognition, plus per-line/per-word font metadata work. A shared RapidOCR session cannot safely run concurrently. | 20-page scan: 142.747 s wall, 655.016 CPU-s, 1,769.4 MB peak RSS. Warm one-page inference was 3.021 s/17.318 CPU-s with Arabic versus 1.370 s/7.943 CPU-s with adaptive skip. | Partially fixed: inference/model locks, adaptive Arabic, page caps, budgets, and raster reuse are present. |
+| 4 | P1 | Table reconstruction is heuristic by default; ruled headers and multi-line cells are represented as fragments before grid assignment. The optional structure model is not shipped. | W3C exact table gate produced recall 0.867 (<0.90). `Participants` became `s Participant`; `Time to complete` became `Results Time to complete`; `n=1` gained a space after `=`. | Open; targeted fix is required. |
+| 5 | P1 | PDF-to-editable-Office is a semantic reflow, not a pixel-preserving transform. Header/footer detection, style inference, anchored objects, columns, fields, and arbitrary positioned graphics have no one-to-one OOXML representation. | Pixel similarity varies from 0.784 (USGS) to 0.950 (govinfo) in the ReportLab-rendered comparison; order scores are 0.232-0.674 for several digital PDFs. | Fundamental limitation; use page-image fallback or a fidelity backend when editability is not required. |
+| 6 | P1 | OCR recognition is sensitive to source DPI, contrast, skew, script, and confidence thresholds. Low thresholds (0.08 box/text) preserve faint marks but also admit noise; post-processing must distinguish symbols from rules. | Scan recall is 1.000 against expected page tokens but precision is 0.077 because OCR also emits repeated body/table text. Arabic adaptive mode improves CPU cost but whole-run wall time is noisy. | Partially fixed; needs calibrated per-document confidence and preprocessing. |
+| 7 | P1 | Quality scoring historically rewarded agreement with a corrupted pypdf text layer and ignored duplicates/order. Structure bonus could be earned by a diagram label classified as a header. | Mozilla score moved 0.9475 -> 0.8998 while the corrupted duplicate text disappeared and real prose improved. New metrics expose precision/order and store `idm.meta["text_metrics"]`. | Fixed for diagnostics; corpus gates still need independent gold/layout checks. |
+| 8 | P2 | Repeated PDF reads, geometry extraction, rasterization, and model initialization inflated latency and memory. Unbounded cache keys also allowed stale results across option/routing changes. | Per-conversion document cache and canonical cache keys were added; regression suites cover collisions and queue admission. | Fixed/mitigated. |
+| 9 | P2 | API contract mismatch unrelated to conversion: unknown font endpoint currently returns 204 while its test expects 404. | Full suite failure in `tests/test_inplace_editing.py::test_font_endpoint_rejects_unknown`. | Open decision for API owner. |
+| 10 | P2 | Timeout release is thread-based, not process-based. A native renderer or OCR call that ignores cancellation can continue after the request receives 504; local SQLite/spool storage also limits multi-instance scaling. | `run_with_timeout()` explicitly uses `shutdown(wait=False)` because Python cannot kill a thread; job store uses one local SQLite file and local spool directory. No runaway crash was observed in the audit, but the resource behavior is a design risk. | Mitigate with process/container isolation and shared durable storage. |
+
+### Content accuracy
+
+PDF text is not necessarily a trustworthy source string. Embedded fonts can have
+missing or malformed ToUnicode maps, CID encodings, zero-width markers, and text
+show operators split in ways that do not correspond to words. The current pipeline
+therefore combines C++ glyph extraction, pypdf extraction, content-stream evidence,
+OCR, and text-quality heuristics. Any stage that chooses a wrong representation can
+duplicate, drop, or reorder characters before the writer sees them.
+
+The old comparison metric made this harder to observe: ASCII-only unique-token
+intersection ignored Arabic, CJK, combining marks, symbols, repeated occurrences,
+and order. The replacement is Unicode NFKC normalization, line-break hyphen joining,
+duplicate-aware multiset matching, meaningful-symbol retention, and an ordered
+sequence score. It is still a diagnostic metric, not a semantic proof.
+
+### Layout and structure fidelity
+
+The IDM preserves BBoxes and reading order, but a flowing DOCX writer must choose
+paragraphs, columns, tables, and page breaks. Heuristics infer headings from size
+and text, detect headers/footers from repetition and position, and merge rows across
+pages. They cannot preserve every PDF transformation matrix, clipping path,
+baseline, text box, field, or z-order. Automatic page-raster fallback is the safety
+floor for complex positioned pages: visual content survives, but text is no longer
+editable or searchable as individual runs.
+
+### Formatting preservation
+
+The writers can carry bold, italic, colors, hyperlinks, bullets, numbering, and
+several paragraph attributes when those attributes exist in the source Office XML
+or can be inferred from PDF glyph metadata. PDF extraction usually lacks the
+original style names and theme inheritance. Font aliases and locally installed
+fonts determine metrics; missing fonts fall back to Helvetica/Liberation/DejaVu or
+script-specific Noto faces. Word-to-PDF currently applies a fixed paragraph
+`spaceAfter` in the ReportLab path rather than reproducing every source paragraph's
+spacing, so small but systematic vertical shifts are expected.
+
+### Tables
+
+The detector combines ruling-line (lattice), whitespace/stream, rectangle, and
+plausibility passes, then maps glyph lines into cells. A two-column spanning header
+or a row whose words sit on separate baselines can be split into fragments. Merged
+cells and nested content are represented approximately. The structure ONNX hook
+can supply separators, but the model is operator-supplied and fail-open; it is not
+part of the reproducible default deployment.
+
+### Images and graphics
+
+XObjects and rendered page images are retained where possible. Large figures are
+compacted rather than silently discarded. If a page reconstructs to no meaningful
+blocks, the pipeline renders and embeds that page as a figure, or emits an explicit
+warning. This preserves visible pixels but cannot preserve editable diagram
+objects, vector semantics, captions' relationships, or exact crop/ordering in all
+cases.
+
+## Evidence
+
+### Measurement method
+
+`tools/audit_conversion.py` invokes the same in-process `run_conversion` entry point
+used by the gateway. For each case it records input/output bytes, source and output
+page counts, warnings/errors, duplicate-aware text recall/precision/order/F1, table
+payloads, and model status. A Windows process sampler records working-set peak and
+`GetProcessTimes` CPU seconds at 20 ms intervals. DOCX/XLSX outputs are parsed with
+`python-docx`/`openpyxl`; visual comparison renders both sides at 96 DPI and reports
+`1 - mean(abs(gray_a-gray_b))/255` over the compared pages.
+
+The visual metric is intentionally simple and should be treated as a trend signal.
+It is not SSIM, does not align transformed pages, and in this run renders Office
+outputs with the in-process ReportLab fallback. It therefore does not measure what
+LibreOffice/Gotenberg would produce.
+
+### Representative real-document run
+
+The latest six-case digital run is the current `audit_results.json` artifact; the
+20-page scan row is from the preceding targeted run and will be appended after the
+final OCR run. Times are wall seconds; CPU is process CPU seconds; RSS is peak
+working set. "Order" is the ordered token similarity, not a page-layout score.
+Output size and output page count are taken directly from the audit JSON. Re-run
+the command in Appendix A after any code change to refresh all rows. Results can
+vary with cache/model warm-up and with whether the page-raster fallback is
+selected, so the harness should be run more than once when comparing changes.
+
+| Document -> target | Source pages | Rendered output pages* | Page delta* | Wall s | CPU s | Peak RSS MB | Output bytes | Warnings | Recall | Precision | Order | Pixel similarity |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| govinfo -> DOCX | 1 | 2 | +1 | 4.707 | 12.250 | 717.0 | 39,779 | 7 | 0.941 | 0.888 | 0.443 | 0.950 |
+| IRS 1040 -> DOCX | 2 | 7 | +5 | 4.531 | 13.578 | 1118.7 | 43,430 | 9 | 0.936 | 0.944 | 0.674 | 0.890 |
+| Mozilla PDF spec (14 p) -> DOCX | 14 | 28 | +14 | 29.785 | 92.750 | 1141.1 | 77,386 | 24 | 0.916 | 0.937 | 0.149 | 0.895 |
+| USGS factsheet (4 p) -> DOCX | 4 | 7 | +3 | 14.897 | 30.594 | 1346.8 | 161,987 | 13 | 0.767 | 0.793 | 0.232 | 0.784 |
+| W3C ruled table -> XLSX | 1 | 1 | 0 | 1.732 | 5.391 | 1191.7 | 5,328 | 7 | 1.000 | 1.000 | 0.990 | 0.959 |
+| Multipage table (2 p) -> XLSX | 2 | 2 | 0 | 3.280 | 11.891 | 1195.6 | 5,593 | 7 | 0.600 | 1.000 | 0.750 | 0.998 |
+| Scan pack 1 (20 p) -> DOCX | 20 | 4 | -16 | 142.747 | 655.016 | 1769.4 | 58,244 | 102 | 1.000 | 0.077 | 0.143 | 0.957 |
+
+Source characteristics for the digital cases were 3,487 chars/545 words (govinfo),
+10,416/2,223 (IRS), 82,714/13,439 (Mozilla), 14,022/1,887 (USGS), 375/65 (W3C),
+and 73/25 (multipage table) according to pypdf's source read. The scan pack has
+20 raster pages and only a tiny/invalid embedded text layer, so expected page-token
+recall was supplied independently (`ScanTokenpack1P000` ... `P019`).
+
+\*Rendered output pages are the pages produced when the audit harness sends the
+Office bytes through its in-process ReportLab visual renderer. They are not a
+claim about a user's pagination in Word. For example, the scan DOCX contains 19
+explicit page breaks (20 intended pages) even though the fallback visual render
+reports four pages; both page-break metadata and rendered page count must be
+tracked.
+
+The scan result is a useful warning: recall 1.000 alone would say "perfect," while
+precision 0.077 and order 0.143 show that the editable OCR reconstruction contains
+substantial extra/repeated material. Its pixel similarity is high because the page
+images are retained; visual success and editable-text success are different
+products.
+
+### Multi-column defect A/B evidence
+
+On the Mozilla document, the old path produced 189 reading-order inversions in the
+probe. After the coverage-gate and column-aware row fixes, the probe reports 1.
+The corrected prose sentence is no longer spliced. The old corpus quality number
+was 0.9475 and the new number 0.8998, but the output character count fell from
+83,136 to 79,223 because duplicate visitor text disappeared. A/B instrumentation
+showed recall 0.9806 -> 0.9608 and missing token *types* 41 -> 83, while missing
+real words changed 31 -> 32; the additional "missing" tokens were chart-axis
+labels that pypdf had decoded differently. This is a metric-baseline artifact,
+not evidence that the repaired sentence was lost.
+
+### Table evidence
+
+The hand-labelled table truth in `corpus/convert/table_truth.json` requires the W3C
+exact six-column grid to reach recall 0.90 and precision 0.55. The current failing
+output from `build_idm_from_pdf(table.pdf)` is:
+
+```text
+["Disability Category", "s Participant", "Ballots Completed",
+ "Ballots Incomplete/ Terminated", "Accuracy", "Results Time to complete"]
+["Blind", "5", "1", "4", "34.5%, n= 1", "1199 sec, n=1"]
+...
+["Mobility", "3", "3", "0", "95.4%, n= 3", "1416 sec, n=3"]
+```
+
+Compiled glyph inspection found a zero-width-space marker at approximately
+`x=222.28` and the following `s` at `x=222.31`, with near-identical baseline;
+`lines_from_glyphs()` separated them. Header words were spread over multiple
+baselines (`Participant`/`s`, `Results`/`Time to complete`, and similar pairs),
+and some spans contained the punctuation artifact `"= "`. The two failing table
+tests are therefore reproducible extraction defects, not a threshold problem.
+
+### OCR and model evidence
+
+The active venv reports:
+
+```text
+pypdf 5.3.0
+onnxruntime 1.28.0
+rapidocr-onnxruntime 1.2.3
+providers: AzureExecutionProvider, CPUExecutionProvider
+pypdfium2: unavailable
+```
+
+No CUDA provider was available, so the measured conversion was CPU-bound. The
+layout model is 130.5 MB FP32 (34.7 MB INT8 alternative); the Arabic recognizer is
+8.0 MB. The root `models/font` tree contains only metadata/tokenizer placeholders,
+not `model.onnx` or `fonts.index`, and the classifier code does not create an ONNX
+session. Any font-confidence value from that service must therefore be treated as
+a heuristic, not a calibrated model probability.
+
+Arabic warm-page measurements were stable enough to show the tradeoff:
+
+| Mode | Mean wall/page | Mean CPU/page | Interpretation |
+| --- | ---: | ---: | --- |
+| Arabic pass enabled | 3.021 s | 17.318 CPU-s | Better script coverage, roughly double inference work |
+| Adaptive skip on clean Latin | 1.370 s | 7.943 CPU-s | About 55% lower inference cost; known Arabic/broken-cmap pages still force the pass |
+
+Whole 20-page wall time was noisy (non-adaptive 150.389 s versus adaptive
+178.408 s in separate runs), so the warm-page measurements, not that pair alone,
+should guide capacity planning.
+
+### Test evidence
+
+Focused regression groups passed as follows: earlier cache/queue suite 96 tests;
+OCR budget/Arabic suite 32; CORS/model packaging 8; raster/quality/real-corpus
+focused run 34 passed with 16 raster-specific skips; streaming export plus
+cache/queue 7; corpus/phase/production smoke 36; and an additional metrics/OCR
+run 133. The full gateway suite completed 746 passed, 21 skipped, and 3 failed.
+
+The three failures were:
+
+1. `test_labelled_table_is_recovered[w3c_ruled_grid]`: recall 0.867, floor 0.90.
+2. `test_table_pdf_reading_order_and_spans`: the same `n= 1` and header-fragment
+   defects.
+3. `test_font_endpoint_rejects_unknown`: endpoint returned 204; test expects 404.
+
+The raster skips are environmental (`pypdfium2` is missing), not green evidence
+that image export/OCR works on a renderer-free host.
+
+## Recommended Fix
+
+### Immediate P0/P1 work
+
+1. **Finish W3C/table normalization and fragment stitching.** In the table path
+   only, remove whitespace immediately after `=` and normalize punctuation spacing
+   around `%`, `=`, and commas. Merge one-character continuation fragments when
+   their baseline and x-position overlap a neighboring fragment. Stitch header
+   fragments by column anchor and vertical proximity, preserving true multi-line
+   cells. Add a regression for the exact five bad cells and for ordinary prose so
+   the normalization cannot alter sentence spacing.
+2. **Merge the two geometry walkers.** The precise C++ walker and pypdf visitor
+   observe the same content stream. Use precise glyph bounds whenever available and
+   visitor text only as a decoding fallback, instead of calibrating visitor width
+   from cross-column span gaps. This removes the remaining unsound estimator rather
+   than merely reducing how often it runs.
+3. **Add independent layout gates.** Keep hand-labelled table truth, and add page
+   bounding-box/column anchors, header/footer repetition, page-break, image-count,
+   and cross-column order fixtures. Report recall, precision, duplicates, order,
+   and visual similarity separately; never gate only on pypdf-derived text.
+4. **Make the table structure model reproducible.** Either vendor a checked,
+   permissively licensed Table Transformer/SLANet model with checksum and measured
+   CPU/RAM cost, or explicitly document heuristic-only deployment and route
+   high-risk tables to the page-image fallback.
+
+### OCR and resource work
+
+5. Keep one process-wide session per model, but use a dedicated OCR worker pool or
+   process queue so serialized inference cannot block unrelated conversions. Bound
+   image dimensions and decoded bytes before OCR; expose queue wait, model-load,
+   render, inference, and rebuild timings per page.
+6. Calibrate DPI and preprocessing by document class (deskew, contrast, denoise,
+   adaptive threshold, crop). Use confidence distributions and script detection to
+   decide a second pass; preserve low-confidence text with a warning rather than
+   silently replacing it. Evaluate RapidOCR against PaddleOCR PP-OCRv5, EasyOCR,
+   and a CPU-friendly transformer on a labelled multilingual set before switching.
+7. Treat the current font service as a heuristic fallback until a real model is
+   shipped. If font fidelity matters, load and checksum an actual ONNX model,
+   batch line crops, and cache one prediction per style/span rather than running
+   line and word inference for every OCR block. Otherwise disable it in the hot
+   conversion path and use extracted PDF font metadata.
+
+### Fidelity and operations
+
+8. Preserve the existing page-raster fallback for complex positioned PDFs and make
+   the choice explicit in response metadata (`editable` versus `visual`). For
+   customer-facing editable PDF->Office jobs, use Gotenberg/LibreOffice or a
+   commercial fidelity provider behind the same plugin interface; retain ReportLab
+   for concurrent-safe low-cost fallback.
+9. Install `pypdfium2` in every supported runtime image and run the raster suite in
+   CI. Keep the C++ engine path as the preferred renderer, but fail loudly when a
+   host has neither renderer nor OCR capability.
+10. Resolve the font endpoint 204/404 contract, and expose structured error codes
+    for page-cap, OCR-budget, table-confidence, and visual-fallback events.
+11. Run conversion workers in separate processes/containers for hostile or very
+    large files. Apply per-tenant byte/page/time budgets, temporary-directory
+    quotas, cancellation cleanup, and a dead-letter queue. Track p95/p99 latency,
+    peak RSS, CPU saturation, cache hit rate, and output-fidelity distributions.
+
+## Implementation
+
+The following changes are present in the working tree and were validated by the
+focused suites:
+
+- `layout/pipeline.py`, `pypdf_geometry.py`, `glyphs.py`, and paragraph/order
+  helpers now use non-whitespace coverage, column-aware row bands, zero-width
+  whitespace handling, improved paragraph grouping, and figure preservation.
+- `ocr.py` serializes model load/inference safely, supports adaptive Arabic
+  recognition (`CONVERT_OCR_ARABIC_ADAPTIVE`), preserves meaningful symbols such
+  as `$` and `%`, and records whether Arabic was used per call.
+- `ocr/rebuild.py` raises the page cap to 200, separates page-cap and time-budget
+  warnings/metadata, compacts oversized figures instead of dropping them, and
+  embeds unreconstructable pages as honest page images.
+- `result_cache.py` canonicalizes effective options, namespace, and automatic
+  versus explicit resolution mode. `doc_cache.py` reuses readers, geometry,
+  rasters, and engine handles inside one conversion.
+- `routers/convert/jobs.py` and conversion scopes bound admission and shared
+  execution slots; cancellation and deadlines are checked at page/OCR boundaries.
+- `quality/text_metrics.py` and `quality/scorer.py` now expose Unicode-aware,
+  duplicate-aware, order-sensitive metrics and warnings for low precision/order;
+  the IDM retains component metrics for diagnostics.
+- PDF->PNG/JPEG/TIFF exports stream pages; TIFF is multi-frame LZW with DPI
+  metadata. DOCX raster fallback is selected from measured complexity/content
+  evidence rather than blindly reflowing every page.
+- `Dockerfile`, `models/MANIFEST.json`, and `tools/verify_models.py` package and
+  checksum runtime layout/Arabic weights. CORS defaults are explicit and
+  production-safe. `start_convert_gateway.ps1` documents OCR, model, timeout,
+  queue, and cache settings.
+- `tools/audit_conversion.py` provides repeatable end-to-end measurements. New
+  tests cover metrics, OCR resource guards, cache keys, queue admission, streaming
+  exports, model packaging, and table truth.
+
+No component was replaced solely because it was open source. The changes preserve
+the C++/PDFium core and existing plugin interfaces; alternatives are recommended
+only where the evidence shows a fundamental fidelity or reproducibility gap.
+
+## Validation Results
+
+The current state passes the broad focused checks listed in Evidence and runs the
+full gateway suite to completion without a crash or timeout. The multi-column
+order probe improved from 189 inversions to 1, and the OCR/page-cap/resource
+regressions are green. Streaming image export and OOXML validity checks are green.
+
+The result is not yet release-clean:
+
+- W3C exact table recall is 0.867 against a 0.90 floor.
+- Table reading-order/span assertions fail on the same punctuation/header
+  fragments.
+- Unknown-font endpoint status disagrees with its test contract.
+- USGS and multipage-table order/recall remain low despite high pixel similarity.
+- Scan-pack editable text has perfect supplied-token recall but only 0.077
+  precision and 0.143 order similarity; it must not be marketed as accurate OCR
+  without a richer ground truth.
+
+Expected gains from the recommended work are stated as targets, not measured
+claims: table stitching should raise W3C recall above the existing 0.90 gate and
+remove the four known header/punctuation errors; a merged geometry walker should
+keep the 1-inversion result while eliminating the remaining visitor-width risk;
+adaptive OCR already demonstrates approximately 55% lower warm-page inference
+cost on clean Latin scans; process-level isolation should make peak RSS roughly
+linear per worker instead of allowing simultaneous jobs to share one unbounded
+address space. Each target must be re-measured on the same corpus before release.
+
+## Remaining Limitations
+
+1. PDF->editable DOCX/XLSX cannot be pixel-perfect for arbitrary positioned PDF
+   content. The page-image fallback preserves appearance by giving up editability;
+   the reflow path preserves more semantics by accepting layout drift.
+2. The default table detector remains heuristic and the table structure model is
+   not shipped. Merged cells, borderless grids, nested tables, and multi-line
+   headers remain the largest content-structure risk.
+3. RapidOCR is CPU-bound in the measured host, and shared-session locking limits
+   parallel OCR throughput. Whole-document timings vary with cache/model state and
+   Windows scheduling; warm per-page numbers are more stable.
+4. The active venv lacks `pypdfium2`, so renderer-free Linux behavior and several
+   raster tests were skipped. The Docker dependency declaration and local runtime
+   are currently inconsistent until the venv is rebuilt from `pyproject.toml`.
+5. No CUDA provider was available in this audit. GPU utilization, GPU memory, and
+   multi-GPU scaling are therefore unmeasured.
+6. The advertised OpenCLIP font model and FAISS index are metadata placeholders in
+   this checkout. Font predictions are heuristic and should not be used as a
+   fidelity guarantee.
+7. Visual similarity is a grayscale pixel heuristic over at most the compared
+   pages and, for Office outputs in this audit, uses ReportLab rather than a real
+   LibreOffice/Gotenberg render.
+8. The quality score is useful for triage but is not a single acceptance metric.
+   Source text can be corrupt, OCR can add legitimate-looking alternatives, and
+   order/layout require independent annotations.
+9. The repository has a `.git_disabled` metadata directory rather than a normal
+   `.git` checkout, and much of the conversion subsystem is working-tree content.
+   Before/after comparisons in this investigation therefore come from harness
+   snapshots and controlled runs, not a pristine commit diff.
+
+## Appendix A: Reproduction Commands and Evidence Files
+
+Run from `pdf/` with the gateway venv:
+
+```powershell
+gateway\.venv\Scripts\python.exe tools\audit_conversion.py
+gateway\.venv\Scripts\python.exe tools\audit_conversion.py --no-scan
+gateway\.venv\Scripts\python.exe -m pytest gateway\tests\convert -q
+gateway\.venv\Scripts\python.exe -m pytest gateway\tests -q
+```
+
+Primary evidence and implementation references:
+
+- `audit_results.json` - machine-readable audit output (the checked-in file may
+  be targeted to one case; regenerate for the full matrix).
+- `tools/audit_conversion.py` - timing, resource, content, table, and visual
+  measurement harness.
+- `corpus/convert/table_truth.json` and `corpus/convert/baselines.json` - external
+  table labels and suite floors.
+- `gateway/app/services/convert/layout/pipeline.py` - routing, IDM construction,
+  OCR budgeting, and page-image fallback.
+- `gateway/app/services/convert/layout/pypdf_geometry.py` and `glyphs.py` - glyph
+  walkers, line grouping, and geometry evidence.
+- `gateway/app/services/convert/ocr/rebuild.py` and `gateway/app/services/ocr.py` -
+  raster/OCR/rebuild path and Arabic adaptive policy.
+- `gateway/app/services/convert/formatters/docx_formatter.py`,
+  `writers/pdf_from_docx.py`, and `backends/pdf_exporter.py` - Office writers and
+  Gotenberg/ReportLab selection.
+- `gateway/pyproject.toml`, `gateway/Dockerfile`, `gateway/models/MANIFEST.json`,
+  and `gateway/start_convert_gateway.ps1` - dependency, packaging, and runtime
+  configuration evidence.
+
+**Bottom line:** the engine now has honest diagnostics and materially better
+multi-column recovery, but it should be released with explicit fidelity classes,
+the W3C table defect fixed, renderer/OCR dependencies verified in CI, and separate
+content, structure, and visual acceptance gates.

Testing/Temporary/CTestCostData.txt

@@ -0,0 +1 @@
+---

bindings/CMakeLists.txt

@@ -0,0 +1,139 @@
+# Python bindings for PdfEngine using pybind11
+#
+
+# The .pyd ABI tag (cp312 vs cp313) must match the interpreter that runs
+# the FastAPI gateway. Prefer gateway/.venv when it exists so `import pdfengine`
+# works in uvicorn without a second Python install.
+if(WIN32)
+    set(_gateway_python "${CMAKE_SOURCE_DIR}/gateway/.venv/Scripts/python.exe")
+else()
+    set(_gateway_python "${CMAKE_SOURCE_DIR}/gateway/.venv/bin/python")
+endif()
+if(EXISTS "${_gateway_python}")
+    set(Python_EXECUTABLE "${_gateway_python}" CACHE FILEPATH "Python for pybind11" FORCE)
+    set(Python3_EXECUTABLE "${_gateway_python}" CACHE FILEPATH "Python3 for pybind11" FORCE)
+    set(PYTHON_EXECUTABLE "${_gateway_python}" CACHE FILEPATH "Python for pybind11 (legacy)" FORCE)
+    message(STATUS "pybind11: using gateway venv ${_gateway_python}")
+endif()
+
+# Drop stale FindPythonLibsNew cache from a previous configure (it can keep
+# PYTHON_MODULE_EXTENSION=.cp313-*.pyd and python313.lib even after we switch
+# the interpreter to the gateway 3.12 venv).
+unset(PYTHON_MODULE_EXTENSION CACHE)
+unset(PYTHON_LIBRARIES CACHE)
+unset(PYTHON_INCLUDE_DIRS CACHE)
+unset(PYTHON_VERSION CACHE)
+unset(PYTHON_VERSION_MAJOR CACHE)
+unset(PYTHON_VERSION_MINOR CACHE)
+unset(PYTHON_IS_DEBUG CACHE)
+unset(PYTHON_MODULE_PREFIX CACHE)
+unset(PYTHON_MODULE_DEBUG_POSTFIX CACHE)
+
+if(Python_EXECUTABLE)
+    execute_process(
+        COMMAND "${Python_EXECUTABLE}" -c "import sys, sysconfig, pathlib; base=pathlib.Path(sys.base_prefix); ver=f'{sys.version_info.major}{sys.version_info.minor}'; print(sysconfig.get_config_var('EXT_SUFFIX') or ''); print(sysconfig.get_path('include')); print(base / 'libs' / f'python{ver}.lib')"
+        OUTPUT_VARIABLE _pdfengine_py_info
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_QUIET
+    )
+    string(REPLACE "\r" "" _pdfengine_py_info "${_pdfengine_py_info}")
+    string(REPLACE "\n" ";" _pdfengine_py_info_lines "${_pdfengine_py_info}")
+    list(LENGTH _pdfengine_py_info_lines _pdfengine_py_info_len)
+    if(_pdfengine_py_info_len GREATER_EQUAL 3)
+        list(GET _pdfengine_py_info_lines 0 _pdfengine_py_ext)
+        list(GET _pdfengine_py_info_lines 1 _pdfengine_py_inc)
+        list(GET _pdfengine_py_info_lines 2 _pdfengine_py_lib)
+        file(TO_CMAKE_PATH "${_pdfengine_py_ext}" _pdfengine_py_ext)
+        file(TO_CMAKE_PATH "${_pdfengine_py_inc}" _pdfengine_py_inc)
+        file(TO_CMAKE_PATH "${_pdfengine_py_lib}" _pdfengine_py_lib)
+        if(_pdfengine_py_ext)
+            set(PYTHON_MODULE_EXTENSION "${_pdfengine_py_ext}" CACHE INTERNAL "Python extension suffix")
+            message(STATUS "pybind11: extension suffix ${_pdfengine_py_ext}")
+        endif()
+        # vcpkg ships a static python312.lib; linking the .pyd against that
+        # crashes when uvicorn loads it (no python312.dll import). Use the
+        # interpreter's own import lib / headers instead.
+        if(EXISTS "${_pdfengine_py_lib}" AND EXISTS "${_pdfengine_py_inc}/Python.h")
+            set(Python_LIBRARY "${_pdfengine_py_lib}" CACHE FILEPATH "Python import library" FORCE)
+            set(Python_LIBRARIES "${_pdfengine_py_lib}" CACHE FILEPATH "Python import library" FORCE)
+            set(Python_INCLUDE_DIR "${_pdfengine_py_inc}" CACHE PATH "Python headers" FORCE)
+            set(Python3_LIBRARY "${_pdfengine_py_lib}" CACHE FILEPATH "Python3 import library" FORCE)
+            set(Python3_INCLUDE_DIR "${_pdfengine_py_inc}" CACHE PATH "Python3 headers" FORCE)
+            get_filename_component(_pdfengine_py_root "${_pdfengine_py_inc}" DIRECTORY)
+            set(Python_ROOT_DIR "${_pdfengine_py_root}" CACHE PATH "CPython install root" FORCE)
+            message(STATUS "pybind11: CPython lib ${_pdfengine_py_lib}")
+            message(STATUS "pybind11: CPython include ${_pdfengine_py_inc}")
+        endif()
+    endif()
+endif()
+
+set(PYBIND11_FINDPYTHON ON)
+
+find_package(pybind11 CONFIG REQUIRED)
+
+# Declare the python module target. We name the target pdfengine_py to avoid
+# target collision with the static C++ library pdfengine, but we set the
+# OUTPUT_NAME to pdfengine to produce the correct importable module.
+pybind11_add_module(pdfengine_py python/pdfengine_py.cpp $<TARGET_OBJECTS:pdfengine>)
+
+set_target_properties(pdfengine_py PROPERTIES
+    OUTPUT_NAME "pdfengine"
+    ARCHIVE_OUTPUT_NAME "pdfengine_py_import"
+)
+if(_pdfengine_py_ext)
+    set_target_properties(pdfengine_py PROPERTIES SUFFIX "${_pdfengine_py_ext}")
+endif()
+
+target_link_libraries(pdfengine_py PRIVATE pdfengine::pdfengine)
+
+# OBJECT-library usage requirements are not always enough on MSVC for the
+# python module (it also consumes $<TARGET_OBJECTS:pdfengine>). Repeat the
+# libraries the engine objects reference so the .pyd link line is complete.
+target_link_libraries(pdfengine_py PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
+if(PDFENGINE_WITH_PDFIUM)
+    target_link_libraries(pdfengine_py PRIVATE pdfium::pdfium)
+endif()
+if(_pdfengine_py_lib AND EXISTS "${_pdfengine_py_lib}")
+    target_link_libraries(pdfengine_py PRIVATE "${_pdfengine_py_lib}")
+endif()
+
+target_include_directories(pdfengine_py PRIVATE 
+    "${CMAKE_SOURCE_DIR}/engine/src"
+)
+
+if(MSVC)
+    target_link_options(pdfengine_py PRIVATE "/FORCE:MULTIPLE")
+endif()
+
+# vcpkg's find_package(Python) wrapper still injects its static python312.lib.
+# A pybind module must link only the host interpreter's import lib (python312.dll).
+foreach(_py_tgt IN ITEMS Python::Module Python::Python Python3::Module Python3::Python)
+    if(TARGET ${_py_tgt} AND _pdfengine_py_lib)
+        set_property(TARGET ${_py_tgt} PROPERTY INTERFACE_LINK_LIBRARIES "${_pdfengine_py_lib}")
+        set_property(TARGET ${_py_tgt} PROPERTY IMPORTED_LOCATION "${_pdfengine_py_lib}")
+        set_property(TARGET ${_py_tgt} PROPERTY IMPORTED_IMPLIB "${_pdfengine_py_lib}")
+    endif()
+endforeach()
+get_target_property(_pdfengine_py_link pdfengine_py LINK_LIBRARIES)
+if(_pdfengine_py_link)
+    set(_pdfengine_py_kept "")
+    foreach(_lib IN LISTS _pdfengine_py_link)
+        if(_lib MATCHES "vcpkg_installed" AND _lib MATCHES "[Pp]ython")
+            message(STATUS "pybind11: dropping vcpkg CPython ${_lib}")
+            continue()
+        endif()
+        list(APPEND _pdfengine_py_kept "${_lib}")
+    endforeach()
+    set_property(TARGET pdfengine_py PROPERTY LINK_LIBRARIES "${_pdfengine_py_kept}")
+endif()
+
+# Set warnings and sanitizers for the bindings module
+pdfengine_set_warnings(pdfengine_py)
+pdfengine_enable_sanitizers(pdfengine_py)
+
+# Copy the compiled .pyd (or .so) file to the gateway/ directory so the
+# Python FastAPI app and its tests can import it immediately after building.
+add_custom_command(TARGET pdfengine_py POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:pdfengine_py> "${CMAKE_SOURCE_DIR}/gateway/"
+    COMMENT "Copying compiled Python extension to gateway/ directory"
+)

bindings/python/pdfengine_py.cpp

@@ -0,0 +1,766 @@
+#include <pdfengine/pdf_document.hpp>
+#include <pdfengine/pdf_engine.hpp>
+#include <spdlog/spdlog.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+namespace py = pybind11;
+
+namespace {
+
+void throw_on_error(pdfengine::EngineError err) {
+    switch (err) {
+    case pdfengine::EngineError::FileNotFound:
+        PyErr_SetString(PyExc_FileNotFoundError, "PDF file not found");
+        throw py::error_already_set();
+    case pdfengine::EngineError::InvalidFormat:
+        throw py::value_error("Invalid PDF format");
+    case pdfengine::EngineError::PasswordRequired:
+        throw py::value_error("Password required to open this PDF");
+    case pdfengine::EngineError::InvalidPassword:
+        throw py::value_error("Invalid password provided for this PDF");
+    case pdfengine::EngineError::PageOutOfBounds:
+        throw py::index_error("Page index out of bounds");
+    case pdfengine::EngineError::RenderFailed:
+        throw std::runtime_error("Failed to render PDF page");
+    case pdfengine::EngineError::WriteFailed:
+        throw std::runtime_error("Failed to write PDF data");
+    default:
+        throw std::runtime_error("Unknown PDF engine error");
+    }
+}
+
+template <typename T> T get_or_throw(std::expected<T, pdfengine::EngineError>&& res) {
+    if (!res.has_value()) {
+        throw_on_error(res.error());
+    }
+    return std::move(res.value());
+}
+
+void get_or_throw(std::expected<void, pdfengine::EngineError>&& res) {
+    if (!res.has_value()) {
+        throw_on_error(res.error());
+    }
+}
+
+} // namespace
+
+#include <parser/content_builder.hpp>
+#include <parser/lexer.hpp>
+#include <parser/parser.hpp>
+#include <pdfengine/content_object.hpp>
+#include <pdfengine/ocr/ocr_coordinator.hpp>
+#include <qpdf/qpdf_extractor.hpp>
+#include <qpdf/qpdf_writer.hpp>
+#include <serializer/ast_serializer.hpp>
+#include <serializer/content_serializer.hpp>
+
+static constexpr double kTjSpaceKern = -500.0;
+
+class StreamEditor {
+public:
+    StreamEditor(const std::string& filepath) : filepath_(filepath) {}
+
+    py::list extract_text_objects(int page_index) {
+        pdfengine::qpdf_layer::QpdfExtractor extractor;
+        auto stream = extractor.extractPageStream(filepath_, page_index);
+        if (!stream.has_value()) {
+            throw std::runtime_error("Failed to extract page stream");
+        }
+
+        pdfengine::Lexer lexer(stream->decodedContent);
+        auto tokens = lexer.tokenize();
+        pdfengine::ContentParser parser(tokens);
+        pdfengine::ContentBuilder builder;
+
+        auto objects = builder.build(parser.parse());
+
+        py::list result;
+        for (const auto& obj : objects) {
+            if (obj->getType() == pdfengine::ContentObjectType::Text) {
+                auto* textObj = static_cast<pdfengine::TextObject*>(obj.get());
+                py::dict d;
+                d["text"] = py::bytes(textObj->text);
+                d["fontName"] = textObj->fontName;
+                d["fontSize"] = textObj->fontSize;
+
+                py::list tm;
+                for (int i = 0; i < 6; ++i) {
+                    tm.append(textObj->tm[i]);
+                }
+                d["tm"] = tm;
+                result.append(d);
+            }
+        }
+        return result;
+    }
+
+    bool replace_text_object(int page_index, int object_index, const py::bytes& new_text_bytes,
+                             const std::string& dest_path) {
+        std::string new_text = new_text_bytes;
+        pdfengine::qpdf_layer::QpdfExtractor extractor;
+        auto stream = extractor.extractPageStream(filepath_, page_index);
+        if (!stream.has_value())
+            return false;
+
+        pdfengine::Lexer lexer(stream->decodedContent);
+        auto tokens = lexer.tokenize();
+        pdfengine::ContentParser parser(tokens);
+        auto operations = parser.parse();
+
+        int textCount = 0;
+        bool modified = false;
+        for (auto& op : operations) {
+            if (op.op == "Tj" || op.op == "'") {
+                if (op.operands.empty())
+                    continue;
+                auto& strNode = op.operands.back();
+                if (strNode->type == pdfengine::AstNodeType::String ||
+                    strNode->type == pdfengine::AstNodeType::HexString) {
+                    if (textCount == object_index) {
+                        strNode->type = pdfengine::AstNodeType::String;
+                        strNode->stringValue = new_text;
+                        modified = true;
+                        break;
+                    }
+                    textCount++;
+                }
+            } else if (op.op == "TJ") {
+                if (op.operands.empty())
+                    continue;
+                auto& arrNode = op.operands.back();
+                if (arrNode->type == pdfengine::AstNodeType::Array) {
+                    std::string combinedText;
+                    for (const auto& item : arrNode->arrayItems) {
+                        if (item->type == pdfengine::AstNodeType::String) {
+                            combinedText += item->stringValue;
+                        } else if (item->type == pdfengine::AstNodeType::HexString) {
+                            combinedText +=
+                                std::string(item->bytesValue.begin(), item->bytesValue.end());
+                        } else if (item->type == pdfengine::AstNodeType::Number) {
+                            if (item->numberValue < kTjSpaceKern)
+                                combinedText += " ";
+                        }
+                    }
+                    if (!combinedText.empty()) {
+                        if (textCount == object_index) {
+                            bool redistributed = false;
+                            if (new_text.size() == combinedText.size()) {
+                                std::vector<std::pair<pdfengine::AstNode*, std::string>> assign;
+                                size_t pos = 0;
+                                bool ok = true;
+                                for (const auto& item : arrNode->arrayItems) {
+                                    if (item->type == pdfengine::AstNodeType::String ||
+                                        item->type == pdfengine::AstNodeType::HexString) {
+                                        size_t L = (item->type == pdfengine::AstNodeType::HexString)
+                                                       ? item->bytesValue.size()
+                                                       : item->stringValue.size();
+                                        assign.emplace_back(item.get(), new_text.substr(pos, L));
+                                        pos += L;
+                                    } else if (item->type == pdfengine::AstNodeType::Number &&
+                                               item->numberValue < kTjSpaceKern) {
+                                        if (pos >= new_text.size() || new_text[pos] != ' ') {
+                                            ok = false;
+                                            break;
+                                        }
+                                        pos += 1;
+                                    }
+                                }
+                                if (ok && pos == new_text.size()) {
+                                    for (auto& [node, content] : assign) {
+                                        node->type = pdfengine::AstNodeType::String;
+                                        node->stringValue = content;
+                                    }
+                                    redistributed = true;
+                                }
+                            }
+                            if (!redistributed) {
+                                arrNode->arrayItems.clear();
+                                auto newStrNode = std::make_shared<pdfengine::AstNode>(
+                                    pdfengine::AstNodeType::String);
+                                newStrNode->stringValue = new_text;
+                                arrNode->arrayItems.push_back(std::move(newStrNode));
+                            }
+                            modified = true;
+                            break;
+                        }
+                        textCount++;
+                    }
+                }
+            }
+        }
+
+        if (!modified)
+            return false;
+
+        pdfengine::AstSerializer astSerializer;
+        std::string newRawStream = astSerializer.serialize(operations);
+
+        pdfengine::qpdf_layer::QpdfWriter writer;
+        auto res = writer.replacePageStreamAndSave(filepath_, dest_path, page_index, newRawStream);
+        return res.has_value();
+    }
+
+private:
+    std::string filepath_;
+};
+
+PYBIND11_MODULE(pdfengine, m) {
+    m.doc() = "Python bindings for the PdfEngine C++ Core SDK";
+
+    py::class_<StreamEditor>(m, "StreamEditor")
+        .def(py::init<const std::string&>(), py::arg("filepath"))
+        .def("extract_text_objects", &StreamEditor::extract_text_objects, py::arg("page_index"))
+        .def("replace_text_object", &StreamEditor::replace_text_object, py::arg("page_index"),
+             py::arg("object_index"), py::arg("new_text"), py::arg("dest_path"));
+
+    m.def("engine_version", &pdfengine::engineVersion, "Get the engine version string");
+    m.def("engine_build_info", &pdfengine::engineBuildInfo, "Get the engine build info string");
+    m.def("engine_has_pdfium", &pdfengine::engineHasPdfium,
+          "Check if the engine was built with PDFium support");
+    m.def("engine_has_skia", &pdfengine::engineHasSkia,
+          "Check if the engine was built with Skia support");
+
+    m.def(
+        "protect_pdf",
+        [](const py::bytes& input_bytes,
+           const std::string& user_password,
+           const std::string& owner_password,
+           const py::dict& perms_dict) {
+            if (user_password.empty()) {
+                throw py::value_error("User password cannot be empty");
+            }
+
+            std::string_view sv = input_bytes;
+            std::vector<uint8_t> data(sv.begin(), sv.end());
+
+            pdfengine::qpdf_layer::PdfEncryptionOptions opts;
+            opts.userPassword = user_password;
+            opts.ownerPassword = owner_password.empty() ? user_password : owner_password;
+
+            auto get_bool = [&](const char* key, bool dflt) {
+                if (perms_dict.contains(key) && !perms_dict[key].is_none()) {
+                    try { return perms_dict[key].cast<bool>(); } catch (...) {}
+                }
+                return dflt;
+            };
+
+            opts.allowPrint = get_bool("canPrint", true);
+            opts.allowPrintHighRes = get_bool("canPrintHighRes", true);
+            opts.allowModify = get_bool("canModify", true);
+            opts.allowCopy = get_bool("canCopy", true);
+            opts.allowAnnotate = get_bool("canAnnotate", true);
+            opts.allowFillForms = get_bool("canFillForms", true);
+            opts.allowAccessibility = get_bool("canExtractForAccessibility", true);
+            opts.allowAssemble = get_bool("canAssemble", true);
+            opts.keyLengthBits = 256;
+
+            pdfengine::qpdf_layer::QpdfWriter writer;
+            auto res = writer.encryptPdf(data, opts);
+
+            if (!res.has_value()) {
+                throw std::runtime_error("Encryption failed");
+            }
+
+            const auto& out_bytes = res.value();
+            return py::bytes(reinterpret_cast<const char*>(out_bytes.data()), out_bytes.size());
+        },
+        py::arg("input_bytes"),
+        py::arg("user_password"),
+        py::arg("owner_password") = "",
+        py::arg("permissions") = py::dict(),
+        "Encrypt raw PDF bytes with AES-256 and custom permissions"
+    );
+
+    m.def(
+        "unlock_pdf",
+        [](const py::bytes& input_bytes, const std::string& password) {
+            std::string_view sv = input_bytes;
+            std::vector<uint8_t> data(sv.begin(), sv.end());
+
+            pdfengine::qpdf_layer::QpdfWriter writer;
+            auto res = writer.unlockPdf(data, password);
+
+            if (!res.has_value()) {
+                throw std::runtime_error("Unlock failed");
+            }
+
+            const auto& out_bytes = res.value();
+            return py::bytes(reinterpret_cast<const char*>(out_bytes.data()), out_bytes.size());
+        },
+        py::arg("input_bytes"),
+        py::arg("password") = "",
+        "Decrypt raw PDF bytes with QPDF"
+    );
+
+    py::class_<pdfengine::Point2D>(m, "Point2D")
+        .def(py::init<double, double>(), py::arg("x") = 0.0, py::arg("y") = 0.0)
+        .def_readwrite("x", &pdfengine::Point2D::x)
+        .def_readwrite("y", &pdfengine::Point2D::y)
+        .def("__repr__", [](const pdfengine::Point2D& self) {
+            return "Point2D(x=" + std::to_string(self.x) + ", y=" + std::to_string(self.y) + ")";
+        });
+
+    py::class_<pdfengine::DevicePoint>(m, "DevicePoint")
+        .def(py::init<int, int>(), py::arg("x") = 0, py::arg("y") = 0)
+        .def_readwrite("x", &pdfengine::DevicePoint::x)
+        .def_readwrite("y", &pdfengine::DevicePoint::y)
+        .def("__repr__", [](const pdfengine::DevicePoint& self) {
+            return "DevicePoint(x=" + std::to_string(self.x) + ", y=" + std::to_string(self.y) +
+                   ")";
+        });
+
+    py::class_<pdfengine::DocumentMetadata>(m, "DocumentMetadata")
+        .def_readonly("title", &pdfengine::DocumentMetadata::title)
+        .def_readonly("author", &pdfengine::DocumentMetadata::author)
+        .def_readonly("creator", &pdfengine::DocumentMetadata::creator)
+        .def_readonly("producer", &pdfengine::DocumentMetadata::producer)
+        .def_readonly("creation_date", &pdfengine::DocumentMetadata::creationDate)
+        .def_readonly("modification_date", &pdfengine::DocumentMetadata::modificationDate)
+        .def("__repr__", [](const pdfengine::DocumentMetadata& self) {
+            return "DocumentMetadata(title='" + self.title + "', author='" + self.author + "')";
+        });
+
+    py::class_<pdfengine::DocumentPermissions>(m, "DocumentPermissions")
+        .def_readonly("is_encrypted", &pdfengine::DocumentPermissions::isEncrypted)
+        .def_readonly("encryption", &pdfengine::DocumentPermissions::encryption)
+        .def_readonly("security_revision", &pdfengine::DocumentPermissions::securityRevision)
+        .def_readonly("owner_unlocked", &pdfengine::DocumentPermissions::ownerUnlocked)
+        .def_readonly("can_print", &pdfengine::DocumentPermissions::canPrint)
+        .def_readonly("can_print_high_res", &pdfengine::DocumentPermissions::canPrintHighRes)
+        .def_readonly("can_modify", &pdfengine::DocumentPermissions::canModify)
+        .def_readonly("can_copy", &pdfengine::DocumentPermissions::canCopy)
+        .def_readonly("can_annotate", &pdfengine::DocumentPermissions::canAnnotate)
+        .def_readonly("can_fill_forms", &pdfengine::DocumentPermissions::canFillForms)
+        .def_readonly("can_extract_for_accessibility",
+                      &pdfengine::DocumentPermissions::canExtractForAccessibility)
+        .def_readonly("can_assemble", &pdfengine::DocumentPermissions::canAssemble);
+
+    py::class_<pdfengine::PageImage>(m, "PageImage")
+        .def_readonly("width", &pdfengine::PageImage::width)
+        .def_readonly("height", &pdfengine::PageImage::height)
+        .def_property_readonly("data", [](const pdfengine::PageImage& self) {
+            return py::bytes(reinterpret_cast<const char*>(self.data.data()), self.data.size());
+        });
+
+    py::class_<pdfengine::FontInfo>(m, "FontInfo")
+        .def_readonly("font_name", &pdfengine::FontInfo::fontName)
+        .def_readonly("type", &pdfengine::FontInfo::type)
+        .def_readonly("is_embedded", &pdfengine::FontInfo::isEmbedded)
+        .def_readonly("is_subset", &pdfengine::FontInfo::isSubset)
+        .def_readonly("is_vertical", &pdfengine::FontInfo::isVertical)
+        .def_readonly("encoding", &pdfengine::FontInfo::encoding)
+        .def_readonly("has_to_unicode", &pdfengine::FontInfo::hasToUnicode)
+        .def_readonly("cmap_name", &pdfengine::FontInfo::cmapName)
+        .def_readonly("cid_system_info", &pdfengine::FontInfo::cidSystemInfo)
+        .def_readonly("subset_tag", &pdfengine::FontInfo::subsetTag)
+        .def_readonly("source_type", &pdfengine::FontInfo::sourceType)
+        .def_readonly("substituted_from", &pdfengine::FontInfo::substitutedFrom)
+        .def_readonly("substituted_to", &pdfengine::FontInfo::substitutedTo)
+        .def_readonly("normalized_family", &pdfengine::FontInfo::normalizedFamily)
+        .def_readonly("internal_font_id", &pdfengine::FontInfo::internalFontId)
+        .def_readonly("flags", &pdfengine::FontInfo::flags)
+        .def_readonly("ascent", &pdfengine::FontInfo::ascent)
+        .def_readonly("descent", &pdfengine::FontInfo::descent)
+        .def_readonly("cap_height", &pdfengine::FontInfo::capHeight)
+        .def("__repr__", [](const pdfengine::FontInfo& self) {
+            return "FontInfo(font_name='" + self.fontName + "', type='" + self.type +
+                   "', is_embedded=" + (self.isEmbedded ? "True" : "False") + ")";
+        });
+
+    py::class_<pdfengine::Glyph>(m, "Glyph")
+        .def_readonly("text", &pdfengine::Glyph::text)
+        .def_readonly("unicode", &pdfengine::Glyph::unicode)
+        .def_readonly("font_name", &pdfengine::Glyph::fontName)
+        .def_readonly("flags", &pdfengine::Glyph::flags)
+        .def_readonly("font_size", &pdfengine::Glyph::fontSize)
+        .def_readonly("origin_x", &pdfengine::Glyph::originX)
+        .def_readonly("origin_y", &pdfengine::Glyph::originY)
+        .def_readonly("bbox_x", &pdfengine::Glyph::bboxX)
+        .def_readonly("bbox_y", &pdfengine::Glyph::bboxY)
+        .def_readonly("bbox_w", &pdfengine::Glyph::bboxW)
+        .def_readonly("bbox_h", &pdfengine::Glyph::bboxH)
+        .def_readonly("angle", &pdfengine::Glyph::angle)
+        .def_readonly("page_object_index", &pdfengine::Glyph::pageObjectIndex);
+
+    py::class_<pdfengine::TextRun>(m, "TextRun")
+        .def_readonly("text", &pdfengine::TextRun::text)
+        .def_readonly("font_name", &pdfengine::TextRun::fontName)
+        .def_readonly("flags", &pdfengine::TextRun::flags)
+        .def_readonly("font_size", &pdfengine::TextRun::fontSize)
+        .def_readonly("internal_font_id", &pdfengine::TextRun::internalFontId)
+        .def_readonly("is_embedded", &pdfengine::TextRun::isEmbedded)
+        .def_readonly("is_embedded_font", &pdfengine::TextRun::isEmbeddedFont)
+        .def_readonly("is_predicted_font", &pdfengine::TextRun::isPredictedFont)
+        .def_readonly("type", &pdfengine::TextRun::type)
+        .def_readonly("glyphs", &pdfengine::TextRun::glyphs)
+        .def_readonly("x", &pdfengine::TextRun::x)
+        .def_readonly("y", &pdfengine::TextRun::y)
+        .def_readonly("w", &pdfengine::TextRun::w)
+        .def_readonly("h", &pdfengine::TextRun::h)
+        .def_readonly("object_indices", &pdfengine::TextRun::objectIndices)
+        .def_readonly("fill_color", &pdfengine::TextRun::fillColor)
+        .def_readonly("para_id", &pdfengine::TextRun::paraId)
+        .def_readonly("font_fidelity", &pdfengine::TextRun::fontFidelity);
+
+    py::class_<pdfengine::TextLine>(m, "TextLine")
+        .def_readonly("runs", &pdfengine::TextLine::runs)
+        .def_readonly("baseline_y", &pdfengine::TextLine::baselineY)
+        .def_readonly("x", &pdfengine::TextLine::x)
+        .def_readonly("y", &pdfengine::TextLine::y)
+        .def_readonly("w", &pdfengine::TextLine::w)
+        .def_readonly("h", &pdfengine::TextLine::h);
+
+    py::class_<pdfengine::Paragraph>(m, "Paragraph")
+        .def_readonly("lines", &pdfengine::Paragraph::lines)
+        .def_readonly("x", &pdfengine::Paragraph::x)
+        .def_readonly("y", &pdfengine::Paragraph::y)
+        .def_readonly("w", &pdfengine::Paragraph::w)
+        .def_readonly("h", &pdfengine::Paragraph::h);
+
+    py::class_<pdfengine::PageModel>(m, "PageModel")
+        .def_readonly("paragraphs", &pdfengine::PageModel::paragraphs)
+        .def_readonly("width", &pdfengine::PageModel::width)
+        .def_readonly("height", &pdfengine::PageModel::height)
+        .def_readonly("page_index", &pdfengine::PageModel::pageIndex);
+
+    py::class_<pdfengine::PdfPage::AnnotationInfo>(m, "AnnotationInfo")
+        .def_readonly("id", &pdfengine::PdfPage::AnnotationInfo::id)
+        .def_readonly("type", &pdfengine::PdfPage::AnnotationInfo::type)
+        .def_readonly("x", &pdfengine::PdfPage::AnnotationInfo::x)
+        .def_readonly("y", &pdfengine::PdfPage::AnnotationInfo::y)
+        .def_readonly("width", &pdfengine::PdfPage::AnnotationInfo::width)
+        .def_readonly("height", &pdfengine::PdfPage::AnnotationInfo::height)
+        .def_readonly("color", &pdfengine::PdfPage::AnnotationInfo::color)
+        .def_readonly("author", &pdfengine::PdfPage::AnnotationInfo::author)
+        .def_readonly("content", &pdfengine::PdfPage::AnnotationInfo::content)
+        .def_readonly("timestamp", &pdfengine::PdfPage::AnnotationInfo::timestamp)
+        .def_readonly("page_index", &pdfengine::PdfPage::AnnotationInfo::pageIndex)
+        .def_readonly("thickness", &pdfengine::PdfPage::AnnotationInfo::thickness)
+        .def_readonly("paths", &pdfengine::PdfPage::AnnotationInfo::paths)
+        .def_readonly("field_name", &pdfengine::PdfPage::AnnotationInfo::fieldName)
+        .def_readonly("field_value", &pdfengine::PdfPage::AnnotationInfo::fieldValue)
+        .def_readonly("field_type", &pdfengine::PdfPage::AnnotationInfo::fieldType)
+        .def_readonly("field_flags", &pdfengine::PdfPage::AnnotationInfo::fieldFlags)
+        .def_readonly("field_options", &pdfengine::PdfPage::AnnotationInfo::fieldOptions)
+        .def_property_readonly("quad_points", [](const pdfengine::PdfPage::AnnotationInfo& self) {
+            py::list out;
+            for (const auto& quad : self.quadPoints) {
+                py::list quad_list;
+                for (const auto& pt : quad) {
+                    py::dict d;
+                    d["x"] = pt.x;
+                    d["y"] = pt.y;
+                    quad_list.append(d);
+                }
+                out.append(quad_list);
+            }
+            return out;
+        });
+
+    py::class_<pdfengine::PdfPage, std::shared_ptr<pdfengine::PdfPage>>(m, "PdfPage")
+        .def_property_readonly("width", &pdfengine::PdfPage::width)
+        .def_property_readonly("height", &pdfengine::PdfPage::height)
+        .def(
+            "render",
+            [](const pdfengine::PdfPage& self, int dpi) { return get_or_throw(self.render(dpi)); },
+            py::arg("dpi") = 96)
+        .def(
+            "render_region_raw",
+            [](const pdfengine::PdfPage& self, int dpi, double y_top_pt, double height_pt) {
+                auto img = get_or_throw(self.renderRegionRaw(dpi, y_top_pt, height_pt));
+                return py::make_tuple(
+                    img.width, img.height,
+                    py::bytes(reinterpret_cast<const char*>(img.data.data()), img.data.size()));
+            },
+            py::arg("dpi"), py::arg("y_top_pt"), py::arg("height_pt") = 0.0)
+        .def(
+            "render_tile",
+            [](const pdfengine::PdfPage& self, int dpi, double xPt, double yPt, double wPt,
+               double hPt) {
+                auto img = get_or_throw(self.renderTile(dpi, xPt, yPt, wPt, hPt));
+                return py::make_tuple(
+                    img.width, img.height,
+                    py::bytes(reinterpret_cast<const char*>(img.data.data()), img.data.size()));
+            },
+            py::arg("dpi"), py::arg("xPt"), py::arg("yPt"), py::arg("wPt"), py::arg("hPt"))
+        .def("extract_document_model",
+             [](const pdfengine::PdfPage& self) {
+                 return get_or_throw(self.extractDocumentModel());
+             })
+        .def("extract_text",
+             [](const pdfengine::PdfPage& self) { return get_or_throw(self.extractText()); })
+        .def("extract_annotations_text",
+             [](const pdfengine::PdfPage& self) {
+                 return get_or_throw(self.extractAnnotationsText());
+             })
+        .def("extract_annotations",
+             [](const pdfengine::PdfPage& self) { return get_or_throw(self.extractAnnotations()); })
+        .def("extract_text_with_bounds",
+             [](const pdfengine::PdfPage& self) {
+                 auto res = get_or_throw(self.extractTextWithBounds());
+                 py::list py_list;
+                 for (const auto& glyph : res) {
+                     py::dict d;
+                     d["text"] = glyph.text;
+                     d["x"] = glyph.x;
+                     d["y"] = glyph.y;
+                     d["w"] = glyph.w;
+                     d["h"] = glyph.h;
+                     d["fontSize"] = glyph.fontSize;
+                     py_list.append(d);
+                 }
+                 return py_list;
+             })
+        .def("ordered_glyphs",
+             [](const pdfengine::PdfPage& self) {
+                 auto res = get_or_throw(self.orderedGlyphs());
+                 py::list py_list;
+                 for (const auto& g : res) {
+                     py::dict d;
+                     d["text"] = g.text;
+                     d["x"] = g.x;
+                     d["y"] = g.y;
+                     d["w"] = g.w;
+                     d["h"] = g.h;
+                     d["fontSize"] = g.fontSize;
+                     py_list.append(d);
+                 }
+                 return py_list;
+             })
+        .def(
+            "hit_glyph",
+            [](const pdfengine::PdfPage& self, double x, double y) {
+                auto hit = get_or_throw(self.hitGlyph(x, y));
+                py::dict d;
+                d["glyphIndex"] = hit.glyphIndex;
+                d["caret"] = hit.caret;
+                d["line"] = hit.line;
+                return d;
+            },
+            py::arg("x"), py::arg("y"))
+        .def(
+            "select_range",
+            [](const pdfengine::PdfPage& self, double ax, double ay, double bx, double by) {
+                auto sel = get_or_throw(self.selectRange(ax, ay, bx, by));
+                py::dict d;
+                d["startGlyph"] = sel.startGlyph;
+                d["endGlyph"] = sel.endGlyph;
+                d["text"] = sel.text;
+                py::list rects;
+                for (const auto& r : sel.rects) {
+                    py::dict rd;
+                    rd["x"] = r.x;
+                    rd["y"] = r.y;
+                    rd["w"] = r.w;
+                    rd["h"] = r.h;
+                    rects.append(rd);
+                }
+                d["rects"] = rects;
+                return d;
+            },
+            py::arg("ax"), py::arg("ay"), py::arg("bx"), py::arg("by"))
+        .def("get_fonts",
+             [](const pdfengine::PdfPage& self) { return get_or_throw(self.getFonts()); })
+        .def(
+            "get_glyph_width",
+            [](const pdfengine::PdfPage& self, const std::string& fontName, uint32_t charcode,
+               double fontSize) {
+                return get_or_throw(self.getGlyphWidth(fontName, charcode, fontSize));
+            },
+            py::arg("font_name"), py::arg("charcode"), py::arg("font_size"))
+        .def("page_to_device", &pdfengine::PdfPage::pageToDevice, py::arg("page_point"),
+             py::arg("device_width"), py::arg("device_height"), py::arg("rotate") = 0)
+        .def("device_to_page", &pdfengine::PdfPage::deviceToPage, py::arg("device_point"),
+             py::arg("device_width"), py::arg("device_height"), py::arg("rotate") = 0)
+        .def("extract_display_list",
+             [](const pdfengine::PdfPage& self) {
+                 return get_or_throw(self.extractDisplayListJson());
+             })
+        .def(
+            "extract_image_xobject",
+            [](const pdfengine::PdfPage& self, const std::string& name) {
+                auto res = get_or_throw(self.extractImageXObject(name));
+                return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
+            },
+            py::arg("name"));
+
+    py::class_<pdfengine::PdfDocument, std::shared_ptr<pdfengine::PdfDocument>>(m, "PdfDocument")
+        .def_static(
+            "load_from_file",
+            [](const std::string& path, const std::string& password) {
+                return get_or_throw(pdfengine::PdfDocument::loadFromFile(path, password));
+            },
+            py::arg("path"), py::arg("password") = "")
+        .def_static(
+            "load_from_memory",
+            [](const py::bytes& bytes, const std::string& password) {
+                std::string_view sv = bytes;
+                std::vector<uint8_t> data(sv.begin(), sv.end());
+                return get_or_throw(pdfengine::PdfDocument::loadFromMemory(data, password));
+            },
+            py::arg("data"), py::arg("password") = "")
+        .def_property_readonly("page_count", &pdfengine::PdfDocument::pageCount)
+        .def_property_readonly("metadata", &pdfengine::PdfDocument::metadata)
+        .def_property_readonly("permissions", &pdfengine::PdfDocument::permissions)
+        .def("extract_outline",
+             [](const pdfengine::PdfDocument& self) {
+                 auto res = get_or_throw(self.extractOutline());
+                 py::list out;
+                 for (const auto& item : res) {
+                     py::dict d;
+                     d["title"] = item.title;
+                     d["pageIndex"] = item.pageIndex;
+                     d["level"] = item.level;
+                     out.append(d);
+                 }
+                 return out;
+             })
+        .def(
+            "get_page",
+            [](pdfengine::PdfDocument& self, int pageIndex) {
+                return get_or_throw(self.getPage(pageIndex));
+            },
+            py::arg("page_index"))
+        .def(
+            "get_fonts",
+            [](const pdfengine::PdfDocument& self, int start_page, int end_page) {
+                return get_or_throw(self.getFonts(start_page, end_page));
+            },
+            py::arg("start_page") = 0, py::arg("end_page") = -1)
+        .def(
+            "get_font_data",
+            [](const pdfengine::PdfDocument& self, const std::string& internal_font_id) {
+                auto res = self.getFontData(internal_font_id);
+                if (!res || res->empty()) {
+                    return py::bytes();
+                }
+                return py::bytes(reinterpret_cast<const char*>(res->data()), res->size());
+            },
+            py::arg("internal_font_id"))
+        .def(
+            "get_reconstructed_font_data",
+            [](pdfengine::PdfDocument& self, const std::string& internal_font_id) {
+                auto res = self.getReconstructedFontData(internal_font_id);
+                if (!res || res->empty()) {
+                    return py::bytes();
+                }
+                return py::bytes(reinterpret_cast<const char*>(res->data()), res->size());
+            },
+            py::arg("internal_font_id"))
+        .def(
+            "apply_edits",
+            [](pdfengine::PdfDocument& self, const std::string& editsJson) {
+                auto regions = get_or_throw(self.applyEdits(editsJson));
+                py::list py_regions;
+                for (const auto& r : regions) {
+                    py::dict d;
+                    d["pageIndex"] = r.pageIndex;
+                    d["x"] = r.x;
+                    d["y"] = r.y;
+                    d["width"] = r.width;
+                    d["height"] = r.height;
+                    py_regions.append(d);
+                }
+                return py_regions;
+            },
+            py::arg("edits_json"))
+        .def("last_reflow_layout",
+             [](const pdfengine::PdfDocument& self) { return self.lastReflowLayout(); })
+        .def("save_incremental",
+             [](const pdfengine::PdfDocument& self) {
+                 std::vector<uint8_t> res = get_or_throw(self.saveIncremental());
+                 return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
+             })
+        .def("save_full",
+             [](const pdfengine::PdfDocument& self) {
+                 std::vector<uint8_t> res = get_or_throw(self.saveFull());
+                 return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
+             })
+        .def("save_full_for_export", [](const pdfengine::PdfDocument& self) {
+            std::vector<uint8_t> res = get_or_throw(self.saveFullForExport());
+            return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
+        });
+
+    py::class_<pdfengine::ocr::OCRCoordinator>(m, "OCRCoordinator")
+        .def(py::init<>())
+        .def(
+            "process_document",
+            [](const pdfengine::ocr::OCRCoordinator& self, int pageIndex, double imgW, double imgH,
+               double pdfW, double pdfH, const py::list& lines_list) {
+                std::vector<pdfengine::document::RawOCRLine> cpp_lines;
+
+                auto get_str_safe = [](py::dict d, const char* key, const std::string& fallback = "") -> std::string {
+                    if (d.contains(key) && !d[key].is_none()) {
+                        try { return d[key].cast<std::string>(); } catch (...) {}
+                    }
+                    return fallback;
+                };
+                auto get_double_safe = [](py::dict d, const char* key, double fallback = 0.0) -> double {
+                    if (d.contains(key) && !d[key].is_none()) {
+                        try { return d[key].cast<double>(); } catch (...) {}
+                    }
+                    return fallback;
+                };
+                auto get_int_safe = [](py::dict d, const char* key, int fallback = 0) -> int {
+                    if (d.contains(key) && !d[key].is_none()) {
+                        try { return d[key].cast<int>(); } catch (...) {}
+                    }
+                    return fallback;
+                };
+                auto get_bool_safe = [](py::dict d, const char* key, bool fallback = false) -> bool {
+                    if (d.contains(key) && !d[key].is_none()) {
+                        try { return d[key].cast<bool>(); } catch (...) {}
+                    }
+                    return fallback;
+                };
+
+                for (auto item : lines_list) {
+                    if (item.is_none()) continue;
+                    py::dict d = item.cast<py::dict>();
+                    pdfengine::document::RawOCRLine line;
+
+                    line.text = get_str_safe(d, "text", "");
+                    line.confidence = get_double_safe(d, "confidence", 0.0);
+                    line.fontSize = get_double_safe(d, "fontSize", 12.0);
+                    line.fontName = get_str_safe(d, "fontName", "Helvetica");
+                    line.fontId = get_str_safe(d, "fontId", "");
+                    line.fontFace = get_str_safe(d, "fontFace", "");
+                    line.fontWeight = get_int_safe(d, "fontWeight", 400);
+                    line.fontStyle = get_str_safe(d, "fontStyle", "normal");
+                    line.isBold = get_bool_safe(d, "isBold", false);
+                    line.isItalic = get_bool_safe(d, "isItalic", false);
+                    line.lineSpacing = get_double_safe(d, "lineSpacing", 1.2);
+                    line.letterSpacing = get_double_safe(d, "letterSpacing", 0.0);
+
+                    if (d.contains("box") && !d["box"].is_none()) {
+                        py::dict box = d["box"].cast<py::dict>();
+                        line.x = get_double_safe(box, "x", 0.0);
+                        line.y = get_double_safe(box, "y", 0.0);
+                        line.width = get_double_safe(box, "width", 0.0);
+                        line.height = get_double_safe(box, "height", 0.0);
+                    } else {
+                        line.x = get_double_safe(d, "x", 0.0);
+                        line.y = get_double_safe(d, "y", 0.0);
+                        line.width = get_double_safe(d, "width", 0.0);
+                        line.height = get_double_safe(d, "height", 0.0);
+                    }
+
+                    spdlog::info(
+                        "[OCR_CPP_RUN] text='{}' fontName='{}' fontId='{}' "
+                        "fontFace='{}' fontWeight={} fontStyle='{}'",
+                        line.text,
+                        line.fontName.empty() ? "<empty>" : line.fontName,
+                        line.fontId.empty() ? "<empty>" : line.fontId,
+                        line.fontFace.empty() ? "<empty>" : line.fontFace,
+                        line.fontWeight,
+                        line.fontStyle.empty() ? "<empty>" : line.fontStyle
+                    );
+
+                    cpp_lines.push_back(line);
+                }
+                return self.processDocument(pageIndex, imgW, imgH, pdfW, pdfH, cpp_lines);
+            },
+            py::arg("page_index"), py::arg("img_w"), py::arg("img_h"), py::arg("pdf_w"),
+            py::arg("pdf_h"), py::arg("lines"));
+}

cmake/pdfium.cmake

@@ -7,18 +7,39 @@ if(NOT PDFENGINE_WITH_PDFIUM)
     return()
 endif()
 
-set(PDFIUM_INSTALL_DIR "${CMAKE_SOURCE_DIR}/third_party/pdfium/install"
-    CACHE PATH "Root of the PDFium install tree produced by build_pdfium.*")
+# Desktop uses the GN-built tree; WASM uses the prebuilt Emscripten static lib
+# (third_party/pdfium-wasm, fetched by get_pdfium_wasm.ps1 — a relinkable libpdfium.a).
+if(EMSCRIPTEN)
+    set(PDFIUM_INSTALL_DIR "${CMAKE_SOURCE_DIR}/third_party/pdfium-wasm"
+        CACHE PATH "Root of the wasm32 PDFium tree (prebuilt libpdfium.a + include)")
+else()
+    set(PDFIUM_INSTALL_DIR "${CMAKE_SOURCE_DIR}/third_party/pdfium/install"
+        CACHE PATH "Root of the PDFium install tree produced by build_pdfium.*")
+endif()
 
-find_path(PDFIUM_INCLUDE_DIR
-    NAMES fpdfview.h
-    PATHS "${PDFIUM_INSTALL_DIR}/include"
-    NO_DEFAULT_PATH)
+if(EMSCRIPTEN)
+    # Emscripten's toolchain sets CMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY, which makes
+    # find_library ignore custom PATHS (it only searches the emsdk sysroot). Set the
+    # prebuilt wasm32 lib + headers directly instead.
+    set(PDFIUM_INCLUDE_DIR "${PDFIUM_INSTALL_DIR}/include")
+    set(PDFIUM_LIBRARY "${PDFIUM_INSTALL_DIR}/lib/libpdfium.a")
+    if(NOT EXISTS "${PDFIUM_INCLUDE_DIR}/fpdfview.h")
+        set(PDFIUM_INCLUDE_DIR "")
+    endif()
+    if(NOT EXISTS "${PDFIUM_LIBRARY}")
+        set(PDFIUM_LIBRARY "")
+    endif()
+else()
+    find_path(PDFIUM_INCLUDE_DIR
+        NAMES fpdfview.h
+        PATHS "${PDFIUM_INSTALL_DIR}/include"
+        NO_DEFAULT_PATH)
 
-find_library(PDFIUM_LIBRARY
-    NAMES pdfium libpdfium
-    PATHS "${PDFIUM_INSTALL_DIR}/lib"
-    NO_DEFAULT_PATH)
+    find_library(PDFIUM_LIBRARY
+        NAMES pdfium libpdfium
+        PATHS "${PDFIUM_INSTALL_DIR}/lib"
+        NO_DEFAULT_PATH)
+endif()
 
 if(NOT PDFIUM_INCLUDE_DIR OR NOT PDFIUM_LIBRARY)
     message(FATAL_ERROR

cmake/skia.cmake

@@ -0,0 +1,45 @@
+# Skia integration.
+# Creates imported target skia::skia from third_party/skia/install/.
+
+if(NOT PDFENGINE_WITH_SKIA)
+    message(STATUS "Skia: disabled (PDFENGINE_WITH_SKIA=OFF). "
+                   "Engine builds without raw Skia linkage.")
+    return()
+endif()
+
+set(SKIA_INSTALL_DIR "${CMAKE_SOURCE_DIR}/third_party/skia/install"
+    CACHE PATH "Root of the Skia install tree produced by build_skia.*")
+
+find_path(SKIA_INCLUDE_DIR
+    NAMES include/core/SkCanvas.h
+    PATHS "${SKIA_INSTALL_DIR}"
+    NO_DEFAULT_PATH)
+
+find_library(SKIA_LIBRARY
+    NAMES skia libskia
+    PATHS "${SKIA_INSTALL_DIR}/lib"
+    NO_DEFAULT_PATH)
+
+if(NOT SKIA_INCLUDE_DIR OR NOT SKIA_LIBRARY)
+    message(FATAL_ERROR
+        "PDFENGINE_WITH_SKIA=ON but no Skia install tree was found under:\n"
+        "    ${SKIA_INSTALL_DIR}\n"
+        "Build Skia first (one-time, slow):\n"
+        "    Windows:  pwsh third_party/skia/build_skia.ps1\n"
+        "    Unix:     ./third_party/skia/build_skia.sh\n"
+        "See third_party/skia/README.md.")
+endif()
+
+add_library(skia::skia STATIC IMPORTED GLOBAL)
+set_target_properties(skia::skia PROPERTIES
+    IMPORTED_LOCATION "${SKIA_LIBRARY}"
+    INTERFACE_INCLUDE_DIRECTORIES "${SKIA_INCLUDE_DIR}")
+
+if(UNIX AND NOT APPLE)
+    set_property(TARGET skia::skia APPEND PROPERTY
+        INTERFACE_LINK_LIBRARIES pthread dl)
+endif()
+
+message(STATUS "Skia: found")
+message(STATUS "  include .. ${SKIA_INCLUDE_DIR}/include")
+message(STATUS "  library .. ${SKIA_LIBRARY}")

cmake/toolchains/vcpkg-wasm.cmake

@@ -0,0 +1,27 @@
+# WebAssembly vcpkg + Emscripten toolchain loader
+# Chain-loads the vcpkg toolchain which then chain-loads the Emscripten compiler toolchain.
+
+if(NOT DEFINED ENV{EMSDK})
+    message(FATAL_ERROR "EMSDK environment variable not set. Please activate Emscripten SDK first.")
+endif()
+
+if(NOT DEFINED ENV{VCPKG_ROOT})
+    message(FATAL_ERROR "VCPKG_ROOT environment variable not set. Please set VCPKG_ROOT first.")
+endif()
+
+set(VCPKG_TARGET_TRIPLET "wasm32-emscripten" CACHE STRING "vcpkg WASM triplet")
+
+# Emscripten toolchain path
+set(EMSDK_TOOLCHAIN "$ENV{EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake")
+if(NOT EXISTS "${EMSDK_TOOLCHAIN}")
+    message(FATAL_ERROR "Emscripten toolchain not found at: ${EMSDK_TOOLCHAIN}")
+endif()
+
+# Instruct vcpkg to chain-load the Emscripten toolchain file
+set(VCPKG_CHAINLOAD_TOOLCHAIN_FILE "${EMSDK_TOOLCHAIN}" CACHE STRING "vcpkg chainload toolchain")
+
+# Include the main vcpkg toolchain
+include("$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake")
+
+# Allow heap memory growth
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-sALLOW_MEMORY_GROWTH=1")

command.MD

@@ -0,0 +1,139 @@
+# 🛠️ PDF Engine — Developer Command Cheat Sheet
+
+This document serves as the single source of truth for commands across our developer roles.
+
+---
+
+## 🗺️ Developer Matrix & Focus Areas
+
+| Role | Focus Area | Code Paths | Primary Responsibilities |
+| :--- | :--- | :--- | :--- |
+| **Dev 1 — Parser & SDK** | Core PDF parser, C++ wrappers, and FastAPI integration | `engine/src/parser/`<br>`gateway/` | PDFium abstraction layers, Rule R2 compliance, Python bindings, FastAPI gateway |
+| **Dev 2 — Graphics & Render** | Frontend React viewer and browser WebAssembly layers | `frontend/`<br>`wasm/` | Rendering facade, Emscripten build pipeline, annotation tools, React UI components |
+| **Dev 3 — Fonts & Text** | Text shaping, font embedding, and subsetting | `engine/src/fonts/`<br>`engine/src/text/` | FreeType & HarfBuzz wrappers, font subsetting, text extraction layers |
+
+---
+
+## ⚡ Quick Reference Command Matrix
+
+| Task / Goal | Dev 1: Parser & SDK | Dev 2: Graphics & Render | Dev 3: Fonts & Text |
+| :--- | :--- | :--- | :--- |
+| **1. One-Time Setup** | `powershell scripts/bootstrap.ps1`<br>*(Setup Python env in `gateway/`)* | `powershell scripts/bootstrap.ps1`<br>*(Setup Node in `frontend/`)* | `powershell scripts/bootstrap.ps1`<br>*(Setup Python env in `gateway/`)* |
+| **2. Build C++ Engine** | `powershell scripts/build_cpp.ps1` | `powershell scripts/build_wasm.ps1` *(WASM)* | `powershell scripts/build_cpp.ps1` |
+| **3. Run Unit Tests** | `powershell scripts/test_cpp.ps1` | `node wasm/pdfengine.test.mjs` | `powershell scripts/test_cpp.ps1 -R "Font\|Text"` |
+| **4. Run Gateway/UI Tests**| `powershell scripts/test_gateway.ps1` | — | `powershell scripts/test_gateway.ps1` |
+| **5. Start Local Server** | `powershell scripts/start_gateway.ps1` | `cd frontend; npm run dev` | — |
+| **6. Aggregator/CI Check** | `powershell scripts/test_phase0.ps1` | `powershell scripts/test_phase0.ps1` | `powershell scripts/test_phase0.ps1` |
+
+---
+
+## 🛠️ Setup & Execution Workflows
+
+### 💻 Dev 1 — Parser & SDK Workflow
+
+#### A. Setup Python Gateway
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/bootstrap.ps1
+cd gateway
+
+# Fast path with uv (recommended: 10x faster)
+uv venv
+.venv\Scripts\Activate.ps1
+uv pip install -e ".[dev]"
+
+# (Alternative fallback with standard pip)
+# python -m venv .venv
+# .venv\Scripts\Activate.ps1
+# pip install -e ".[dev]"
+cd ..
+```
+
+#### B. Build & Local Verification
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/build_cpp.ps1
+powershell -ExecutionPolicy Bypass -File scripts/test_cpp.ps1
+powershell -ExecutionPolicy Bypass -File scripts/check_pdfium_boundary.ps1
+```
+
+#### C. Gateway Integration & Dev Loop
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/test_gateway.ps1
+powershell -ExecutionPolicy Bypass -File scripts/start_gateway.ps1
+```
+
+---
+
+### ⚛️ Dev 2 — Graphics & Render Workflow
+
+#### A. Setup Frontend UI
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/bootstrap.ps1
+cd frontend
+npm install
+cd ..
+```
+
+#### B. WASM Engine Compilation
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/build_wasm.ps1
+```
+
+#### C. Frontend Execution & Production Build
+To launch the Vite web server for visual UI prototyping:
+```powershell
+cd frontend
+npm run dev
+```
+
+To compile production bundles:
+```powershell
+cd frontend
+npm run build
+```
+
+---
+
+### 🔤 Dev 3 — Fonts & Text Workflow
+
+#### A. Setup Environment
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/bootstrap.ps1
+cd gateway
+
+# Fast path with uv (recommended: 10x faster)
+uv venv
+.venv\Scripts\Activate.ps1
+uv pip install -e ".[dev]"
+
+# (Alternative fallback with standard pip)
+# python -m venv .venv
+# .venv\Scripts\Activate.ps1
+# pip install -e ".[dev]"
+cd ..
+```
+
+#### B. Development & Test Loop
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/build_cpp.ps1
+powershell -ExecutionPolicy Bypass -File scripts/test_cpp.ps1 -R "Font|Text"
+powershell -ExecutionPolicy Bypass -File scripts/test_gateway.ps1
+```
+
+---
+
+## 🏁 Environment Verification (Aggregator Check)
+Before pushing any branches, all developers should run the full suite verification:
+```powershell
+powershell -ExecutionPolicy Bypass -File scripts/test_phase0.ps1 -Preset win-local-pdfium
+```
+
+---
+
+## 🔧 Developer Utilities
+Below is a list of other helper scripts in the repository:
+*   **Populate Test Corpus**:
+    ```powershell
+    powershell -ExecutionPolicy Bypass -File scripts/copy_test_corpus.ps1
+    ```
+    *(Copies standard testing PDFs from local PDFium source/checkout directories to the workspace `corpus/` folder).*
+

corpus/basic/clip_path.pdf

@@ -0,0 +1,50 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /Kids [3 0 R]
+  /MediaBox [0 0 100 50]
+  /Count 1
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Contents 4 0 R
+  /Parent 2 0 R
+>>
+endobj
+4 0 obj <<
+  /Length 71
+>>
+stream
+10 15 m
+40 15 l
+40 35 l
+10 35 l
+W n
+0 0 1 RG
+10 10 m
+25 40 l
+40 10 l
+s
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000156 00000 n 
+0000000225 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 5
+>>
+startxref
+347
+%%EOF

corpus/basic/dashed_lines.pdf

@@ -0,0 +1,48 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [0 0 200 100]
+  /Count 1
+  /Kids [3 0 R]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Contents 4 0 R
+>>
+endobj
+4 0 obj <<
+  /Length 95
+>>
+stream
+q
+0 0 0 rg
+10 25 m 190 25 l S
+[6 5 4 3 2 1] 5 d
+10 50 m 190 50 l S
+[] 0 d
+10 75 m 190 75 l S
+Q
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000157 00000 n 
+0000000226 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 5
+>>
+startxref
+372
+%%EOF

corpus/basic/hello_world.pdf

@@ -0,0 +1,68 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [0 0 200 200]
+  /Count 1
+  /Kids [3 0 R]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 4 0 R
+      /F2 5 0 R
+    >>
+  >>
+  /Contents 6 0 R
+>>
+endobj
+4 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Times-Roman
+>>
+endobj
+5 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+6 0 obj <<
+  % Note this object deliberately does not use /Length 83.
+>>
+stream
+BT
+20 50 Td
+/F1 12 Tf
+(Hello, world!) Tj
+0 50 Td
+/F2 16 Tf
+(Goodbye, world!) Tj
+ET
+endstream
+endobj
+xref
+0 7
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000157 00000 n 
+0000000299 00000 n 
+0000000377 00000 n 
+0000000453 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 7
+>>
+startxref
+633
+%%EOF

corpus/basic/hello_world_2_pages.pdf

@@ -0,0 +1,81 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [0 0 200 200]
+  /Count 2
+  /Kids [3 0 R 4 0 R]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 5 0 R
+      /F2 6 0 R
+    >>
+  >>
+  /Contents 7 0 R
+>>
+endobj
+4 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 5 0 R
+      /F2 6 0 R
+    >>
+  >>
+  /Contents 7 0 R
+>>
+endobj
+5 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Times-Roman
+>>
+endobj
+6 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+7 0 obj <<
+  /Length 83
+>>
+stream
+BT
+20 50 Td
+/F1 12 Tf
+(Hello, world!) Tj
+0 50 Td
+/F2 16 Tf
+(Goodbye, world!) Tj
+ET
+endstream
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000163 00000 n 
+0000000305 00000 n 
+0000000447 00000 n 
+0000000525 00000 n 
+0000000601 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 8
+>>
+startxref
+735
+%%EOF

corpus/basic/rectangles.pdf

@@ -0,0 +1,54 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [0 0 200 300]
+  /Count 1
+  /Kids [3 0 R]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Contents 4 0 R
+>>
+endobj
+4 0 obj <<
+  /Length 188
+>>
+stream
+q
+0 0 0 rg
+0 290 10 10 re B*
+10 150 50 30 re B*
+0 0 1 rg
+190 290 10 10 re B*
+70 232 50 30 re B*
+0 1 0 rg
+190 0 10 10 re B*
+130 150 50 30 re B*
+1 0 0 rg
+0 0 10 10 re B*
+70 67 50 30 re B*
+Q
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000157 00000 n 
+0000000226 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 5
+>>
+startxref
+466
+%%EOF

corpus/basic/rectangles_multi_pages.pdf

@@ -0,0 +1,122 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [ 0 0 200 250 ]
+  /Count 5
+  /Kids [ 3 0 R 5 0 R 7 0 R 9 0 R 11 0 R ]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Contents 4 0 R
+>>
+endobj
+4 0 obj <<
+  /Length 49
+>>
+stream
+q
+1 1 0 rg
+100 0 30 50 re B*
+70 67 50 30 re B*
+Q
+endstream
+endobj
+5 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Rotate 90
+  /Contents 6 0 R
+>>
+endobj
+6 0 obj <<
+  /Length 49
+>>
+stream
+q
+0 1 1 rg
+100 0 30 50 re B*
+70 67 50 30 re B*
+Q
+endstream
+endobj
+7 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Contents 8 0 R
+>>
+endobj
+8 0 obj <<
+  /Length 49
+>>
+stream
+q
+1 0 0 rg
+100 0 30 50 re B*
+70 67 50 30 re B*
+Q
+endstream
+endobj
+9 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Contents 10 0 R
+>>
+endobj
+10 0 obj <<
+  /Length 51
+>>
+stream
+q
+0 1 0 rg
+100 0 30 50 re B*
+100 150 50 30 re B*
+Q
+endstream
+endobj
+11 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Contents 12 0 R
+>>
+endobj
+12 0 obj <<
+  /Length 50
+>>
+stream
+q
+0 0 0 rg
+0 90 80 60 re B*
+100 150 50 30 re B*
+Q
+endstream
+endobj
+xref
+0 13
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000186 00000 n 
+0000000255 00000 n 
+0000000355 00000 n 
+0000000437 00000 n 
+0000000537 00000 n 
+0000000606 00000 n 
+0000000706 00000 n 
+0000000776 00000 n 
+0000000879 00000 n 
+0000000950 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 13
+>>
+startxref
+1052
+%%EOF

corpus/basic/whitespace.pdf

@@ -0,0 +1,57 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [ 0 0 200 200 ]
+  /Count 1
+  /Kids [ 3 0 R ]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 4 0 R
+    >>
+  >>
+  /Contents 5 0 R
+>>
+endobj
+4 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+5 0 obj <<
+  /Length 33
+>>
+stream
+BT
+20 100 Td
+/F1 16 Tf
+( ) Tj
+ET
+endstream
+endobj
+xref
+0 6
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000161 00000 n 
+0000000287 00000 n 
+0000000363 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 6
+>>
+startxref
+447
+%%EOF

corpus/convert/public/README.md

@@ -0,0 +1,29 @@
+# Public free eval corpus (Wave-1)
+
+Licensed third-party + DocQube self-generated smoke for PDF↔DOCX/XLSX, Path A, images, OCR.
+
+## Quick start (Windows)
+
+```powershell
+cd pdf
+..\gateway\.venv\Scripts\python.exe scripts\convert\generate_public_smoke_fixtures.py
+..\gateway\.venv\Scripts\python.exe scripts\convert\score_public_pack.py
+```
+
+Optional third-party fetch (after LICENSE_CHECKLIST sign-off):
+
+```powershell
+..\gateway\.venv\Scripts\python.exe scripts\convert\fetch_public_eval.py
+```
+
+## Layout
+
+- `manifests/` — per-family case definitions
+- `pack_public.json` — unified pytest pack
+- `samples/` — gitignored binaries (generated or fetched)
+- `reports/` — baseline / A/B JSON
+
+## Licenses
+
+See `../third_party/NOTICE` and `../third_party/LICENSE_CHECKLIST.md`.
+Default fixtures are **self-generated** (no third-party license). DocLayNet/CORD/SROIE/PubTabNet entries in manifests are placeholders until fetched.

corpus/convert/public/manifests/cord_smoke.json

@@ -0,0 +1,23 @@
+{
+  "cases": [
+    {
+      "id": "cord_placeholder",
+      "family": "cord",
+      "license": "CC-BY-4.0",
+      "source": "public/samples/cord/.gitkeep",
+      "pair": "pdf_txt",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.5
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "placeholder until fetch_public_eval.py",
+      "optional": true
+    }
+  ]
+}

corpus/convert/public/manifests/doclaynet_smoke.json

@@ -0,0 +1,49 @@
+{
+  "cases": [
+    {
+      "id": "synth_digital_001",
+      "family": "doclaynet",
+      "license": "DocQube",
+      "source": "public/samples/doclaynet/synth_digital_001.pdf",
+      "pair": "pdf_docx",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.95
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "expected_tokens": [
+        "Digital",
+        "Report",
+        "AlphaBridge"
+      ],
+      "notes": "self-gen digital layout stand-in"
+    },
+    {
+      "id": "synth_multicol_001",
+      "family": "doclaynet",
+      "license": "DocQube",
+      "source": "public/samples/doclaynet/synth_multicol_001.pdf",
+      "pair": "pdf_docx",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.8
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "expected_tokens": [
+        "ColumnLeftOne",
+        "ColumnRightOne"
+      ],
+      "notes": "two-column reading-order stress"
+    }
+  ]
+}

corpus/convert/public/manifests/images_smoke.json

@@ -0,0 +1,22 @@
+{
+  "cases": [
+    {
+      "id": "images_png_pdf",
+      "family": "images",
+      "license": "DocQube",
+      "source": "public/samples/images/smoke.png",
+      "pair": "png_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "PNG\u2192PDF"
+    }
+  ]
+}

corpus/convert/public/manifests/path_a_smoke.json

@@ -0,0 +1,45 @@
+{
+  "cases": [
+    {
+      "id": "path_a_docx",
+      "family": "path_a",
+      "license": "DocQube",
+      "source": "public/samples/path_a/path_a_sample.docx",
+      "pair": "docx_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [
+          "PathAParagraphToken"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "DOCX\u2192PDF Path A"
+    },
+    {
+      "id": "path_a_xlsx",
+      "family": "path_a",
+      "license": "DocQube",
+      "source": "public/samples/path_a/path_a_sample.xlsx",
+      "pair": "xlsx_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [
+          "HeaderA",
+          "CellAlpha"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "XLSX\u2192PDF Path A"
+    }
+  ]
+}

corpus/convert/public/manifests/sroie_smoke.json

@@ -0,0 +1,23 @@
+{
+  "cases": [
+    {
+      "id": "sroie_placeholder",
+      "family": "sroie",
+      "license": "CC-BY-4.0",
+      "source": "public/samples/sroie/.gitkeep",
+      "pair": "pdf_txt",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.5
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "placeholder until fetch_public_eval.py",
+      "optional": true
+    }
+  ]
+}

corpus/convert/public/manifests/synthetic_invoice_smoke.json

@@ -0,0 +1,42 @@
+{
+  "cases": [
+    {
+      "id": "synth_invoice_001",
+      "family": "synthetic_invoice",
+      "license": "DocQube",
+      "source": "public/samples/invoices/synth_invoice_001.pdf",
+      "pair": "pdf_xlsx",
+      "bars": {
+        "cell_match_min": 0.85,
+        "prose_tokens": [
+          "Invoice",
+          "AcmeCorp",
+          "Total"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": [
+          [
+            "Desc",
+            "Qty",
+            "Amount"
+          ],
+          [
+            "Service",
+            "1",
+            "100"
+          ],
+          [
+            "Tax",
+            "1",
+            "10"
+          ]
+        ],
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "self-gen invoice"
+    }
+  ]
+}

corpus/convert/public/pack_public.json

@@ -0,0 +1,218 @@
+{
+  "cases": [
+    {
+      "id": "synth_invoice_001",
+      "family": "synthetic_invoice",
+      "license": "DocQube",
+      "source": "public/samples/invoices/synth_invoice_001.pdf",
+      "pair": "pdf_xlsx",
+      "bars": {
+        "cell_match_min": 0.85,
+        "prose_tokens": [
+          "Invoice",
+          "AcmeCorp",
+          "Total"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": [
+          [
+            "Desc",
+            "Qty",
+            "Amount"
+          ],
+          [
+            "Service",
+            "1",
+            "100"
+          ],
+          [
+            "Tax",
+            "1",
+            "10"
+          ]
+        ],
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "self-gen invoice"
+    },
+    {
+      "id": "synth_table_001",
+      "family": "pubtabnet",
+      "license": "DocQube",
+      "source": "public/samples/pubtab/synth_table_001.pdf",
+      "pair": "pdf_xlsx",
+      "bars": {
+        "cell_match_min": 0.9,
+        "prose_tokens": [],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": [
+          [
+            "Item",
+            "Qty",
+            "Price"
+          ],
+          [
+            "Widget",
+            "2",
+            "10"
+          ],
+          [
+            "Gadget",
+            "1",
+            "25"
+          ]
+        ],
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "self-gen stand-in for PubTabNet until fetch"
+    },
+    {
+      "id": "synth_digital_001",
+      "family": "doclaynet",
+      "license": "DocQube",
+      "source": "public/samples/doclaynet/synth_digital_001.pdf",
+      "pair": "pdf_docx",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.95
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "expected_tokens": [
+        "Digital",
+        "Report",
+        "AlphaBridge"
+      ],
+      "notes": "self-gen digital layout stand-in"
+    },
+    {
+      "id": "synth_multicol_001",
+      "family": "doclaynet",
+      "license": "DocQube",
+      "source": "public/samples/doclaynet/synth_multicol_001.pdf",
+      "pair": "pdf_docx",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.8
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "expected_tokens": [
+        "ColumnLeftOne",
+        "ColumnRightOne"
+      ],
+      "notes": "two-column reading-order stress"
+    },
+    {
+      "id": "path_a_docx",
+      "family": "path_a",
+      "license": "DocQube",
+      "source": "public/samples/path_a/path_a_sample.docx",
+      "pair": "docx_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [
+          "PathAParagraphToken"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "DOCX\u2192PDF Path A"
+    },
+    {
+      "id": "path_a_xlsx",
+      "family": "path_a",
+      "license": "DocQube",
+      "source": "public/samples/path_a/path_a_sample.xlsx",
+      "pair": "xlsx_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [
+          "HeaderA",
+          "CellAlpha"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "XLSX\u2192PDF Path A"
+    },
+    {
+      "id": "images_png_pdf",
+      "family": "images",
+      "license": "DocQube",
+      "source": "public/samples/images/smoke.png",
+      "pair": "png_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "PNG\u2192PDF"
+    },
+    {
+      "id": "cord_placeholder",
+      "family": "cord",
+      "license": "CC-BY-4.0",
+      "source": "public/samples/cord/.gitkeep",
+      "pair": "pdf_txt",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.5
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "placeholder until fetch_public_eval.py",
+      "optional": true
+    },
+    {
+      "id": "sroie_placeholder",
+      "family": "sroie",
+      "license": "CC-BY-4.0",
+      "source": "public/samples/sroie/.gitkeep",
+      "pair": "pdf_txt",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.5
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "placeholder until fetch_public_eval.py",
+      "optional": true
+    }
+  ]
+}

corpus/convert/public/reports/ab_compare.json

@@ -0,0 +1,6 @@
+{
+  "ml_off_passed": 7,
+  "ml_on_passed": 7,
+  "deltas": [],
+  "regressions": 0
+}

corpus/convert/public/reports/baseline_heuristic.json

@@ -0,0 +1,64 @@
+{
+  "ml": "0",
+  "results": [
+    {
+      "id": "synth_invoice_001",
+      "pair": "pdf_xlsx",
+      "ml": "0",
+      "pass": true,
+      "cell_match": 1.0
+    },
+    {
+      "id": "synth_table_001",
+      "pair": "pdf_xlsx",
+      "ml": "0",
+      "pass": true,
+      "cell_match": 1.0
+    },
+    {
+      "id": "synth_digital_001",
+      "pair": "pdf_docx",
+      "ml": "0",
+      "pass": true,
+      "text_recall": 1.0
+    },
+    {
+      "id": "synth_multicol_001",
+      "pair": "pdf_docx",
+      "ml": "0",
+      "pass": true,
+      "text_recall": 1.0
+    },
+    {
+      "id": "path_a_docx",
+      "pair": "docx_pdf",
+      "ml": "0",
+      "pass": true
+    },
+    {
+      "id": "path_a_xlsx",
+      "pair": "xlsx_pdf",
+      "ml": "0",
+      "pass": true
+    },
+    {
+      "id": "images_png_pdf",
+      "pair": "png_pdf",
+      "ml": "0",
+      "pass": true
+    },
+    {
+      "id": "cord_placeholder",
+      "skipped": true,
+      "reason": "missing_source"
+    },
+    {
+      "id": "sroie_placeholder",
+      "skipped": true,
+      "reason": "missing_source"
+    }
+  ],
+  "passed": 7,
+  "failed": 0,
+  "skipped": 2
+}

corpus/convert/public/reports/live_real_doc_suite.json

@@ -0,0 +1,728 @@
+[
+    {
+        "file":  "digital_report_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36742,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "digital_report_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4947,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "govinfo.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  38508,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8932",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "govinfo.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  7086,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8932",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "govinfo_small.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  38508,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8932",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "govinfo_small.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  7085,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8932",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "invoice_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36748,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "invoice_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4980,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6132",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "invoice_002.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36742,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "invoice_002.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4963,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6050",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "irs_f1040_sample.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  41114,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8993",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "irs_f1040_sample.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  10851,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8993",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "merged_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36833,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "merged_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5007,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    },
+    {
+        "file":  "mozilla_pdf_spec_excerpt.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  74515,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9076",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "mozilla_pdf_spec_excerpt.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  56039,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9076",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "multicolumn_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36671,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multicolumn_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4896,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "multicolumn_002.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36671,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multicolumn_002.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4896,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "multipage_table_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36835,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7125",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multipage_table_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5593,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8000",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Page 2: layout_ml=onnx regions"
+    },
+    {
+        "file":  "simple_table_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36701,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "simple_table_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4914,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6050",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "table_gap_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36668,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: glyph/engine text we"
+    },
+    {
+        "file":  "table_gap_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4898,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "table_pipe_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36847,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "table_pipe_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5033,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    },
+    {
+        "file":  "table_simple_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36854,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "table_simple_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5051,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    },
+    {
+        "file":  "usgs_factsheet.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  3065773,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6825",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "usgs_factsheet.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  15077,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.5600",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "w3c_pdf_table.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  37075,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6096",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "w3c_pdf_table.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5335,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.5431",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=2"
+    },
+    {
+        "file":  "w3c_table.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  37075,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6096",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "w3c_table.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5336,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.5431",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=2"
+    },
+    {
+        "file":  "synth_digital_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36742,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "synth_digital_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4947,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "synth_multicol_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36739,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9175",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "synth_multicol_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4933,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9175",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "synth_invoice_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36748,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "synth_invoice_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4981,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6132",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "synth_table_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36701,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "synth_table_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4915,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6050",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "digital_report_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36742,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "digital_report_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4946,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "invoice_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36748,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "invoice_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4981,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6132",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "invoice_002.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36742,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "invoice_002.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4965,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6050",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "merged_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36833,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "merged_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5006,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    },
+    {
+        "file":  "multicolumn_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36671,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multicolumn_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4895,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "multicolumn_002.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36671,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multicolumn_002.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4895,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "multipage_table_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36835,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7125",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multipage_table_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5593,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8000",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Page 2: layout_ml=onnx regions"
+    },
+    {
+        "file":  "simple_table_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36701,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "simple_table_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4914,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6050",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "table_gap_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36668,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: glyph/engine text we"
+    },
+    {
+        "file":  "table_gap_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4898,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "table_pipe_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36847,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "table_pipe_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5032,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    },
+    {
+        "file":  "table_simple_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36854,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "table_simple_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5051,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    }
+]

corpus/convert/public/reports/score_ml_on.json

@@ -0,0 +1,64 @@
+{
+  "ml": "1",
+  "results": [
+    {
+      "id": "synth_invoice_001",
+      "pair": "pdf_xlsx",
+      "ml": "1",
+      "pass": true,
+      "cell_match": 1.0
+    },
+    {
+      "id": "synth_table_001",
+      "pair": "pdf_xlsx",
+      "ml": "1",
+      "pass": true,
+      "cell_match": 1.0
+    },
+    {
+      "id": "synth_digital_001",
+      "pair": "pdf_docx",
+      "ml": "1",
+      "pass": true,
+      "text_recall": 1.0
+    },
+    {
+      "id": "synth_multicol_001",
+      "pair": "pdf_docx",
+      "ml": "1",
+      "pass": true,
+      "text_recall": 1.0
+    },
+    {
+      "id": "path_a_docx",
+      "pair": "docx_pdf",
+      "ml": "1",
+      "pass": true
+    },
+    {
+      "id": "path_a_xlsx",
+      "pair": "xlsx_pdf",
+      "ml": "1",
+      "pass": true
+    },
+    {
+      "id": "images_png_pdf",
+      "pair": "png_pdf",
+      "ml": "1",
+      "pass": true
+    },
+    {
+      "id": "cord_placeholder",
+      "skipped": true,
+      "reason": "missing_source"
+    },
+    {
+      "id": "sroie_placeholder",
+      "skipped": true,
+      "reason": "missing_source"
+    }
+  ],
+  "passed": 7,
+  "failed": 0,
+  "skipped": 2
+}

corpus/convert/public/reports/triage.md

@@ -0,0 +1,16 @@
+# Public pack triage (A/B)
+
+- ML off passed: 7
+- ML on passed: 7
+- Regressions: 0
+
+## Suggested climb owners
+
+| Failure pattern | Climb | Files |
+| --- | --- | --- |
+| Invoice / grid cell | M2 | layout/tables.py, quality/table_metrics.py |
+| Multi-column order | M3 | reading_order.py, ml_merge.py |
+| Scan / OCR empty | M4 | ocr/rebuild.py, rapid_adapter.py |
+| Missing figures | M5 | layout/images.py |
+| Header/footer in body | S1 | headers_footers.py, docx_formatter.py |
+

corpus/convert/real/README.md

@@ -0,0 +1,16 @@
+# Wave-1 production smoke pack (curated)
+
+Internal curated PDFs used for **shippable Wave-1** acceptance bars.
+These are synthetic/corpus-derived stand-ins for business docs until customer PDFs are added.
+
+| File | Role |
+| --- | --- |
+| digital_report_001.pdf | Simple digital text → DOCX |
+| table_pipe_001.pdf / table_simple_001.pdf | Simple tables → XLSX |
+| invoice_001.pdf / invoice_002.pdf | Invoice grid + prose → XLSX |
+| merged_001.pdf | Merged-style table → XLSX |
+| multicolumn_*.pdf | Multi-column → DOCX text recall |
+| multipage_table_001.pdf | Multi-page tables |
+| table_gap_001.pdf | Gap-separated columns |
+
+License: synthetic-internal. Not ConvertAPI goldens.

corpus/convert/real/pack.json

@@ -0,0 +1,182 @@
+{
+  "cases": [
+    {
+      "id": "real_digital_docx",
+      "source": "real/digital_report_001.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "Digital",
+        "Report",
+        "AlphaBridge",
+        "AcmeCorp"
+      ],
+      "text_recall_min": 0.95
+    },
+    {
+      "id": "real_table_xlsx",
+      "source": "real/simple_table_001.pdf",
+      "pair": "pdf->xlsx",
+      "expected_cells": [
+        [
+          "Item",
+          "Qty",
+          "Price"
+        ],
+        [
+          "Widget",
+          "2",
+          "10"
+        ],
+        [
+          "Gadget",
+          "1",
+          "25"
+        ]
+      ],
+      "cell_match_min": 0.9
+    },
+    {
+      "id": "real_invoice_xlsx",
+      "source": "real/invoice_001.pdf",
+      "pair": "pdf->xlsx",
+      "expected_cells": [
+        [
+          "Desc",
+          "Qty",
+          "Amount"
+        ],
+        [
+          "Service",
+          "1",
+          "100"
+        ]
+      ],
+      "prose_tokens": [
+        "Invoice",
+        "AcmeCorp",
+        "Vendor"
+      ],
+      "cell_match_min": 0.85
+    },
+    {
+      "id": "real_two_col_docx",
+      "source": "real/multicolumn_001.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "LeftColA",
+        "LeftColB",
+        "RightColA",
+        "RightColB"
+      ],
+      "text_recall_min": 0.95
+    },
+    {
+      "id": "real_three_col_docx",
+      "source": "real/multicolumn_003.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "AlphaLeft",
+        "BravoMid",
+        "CharlieRight",
+        "AlphaLast",
+        "CharlieLast"
+      ],
+      "text_recall_min": 0.9
+    },
+    {
+      "id": "real_fourcol_xlsx",
+      "source": "real/table_fourcol_001.pdf",
+      "pair": "pdf->xlsx",
+      "expected_cells": [
+        [
+          "Category",
+          "Q1",
+          "Q2",
+          "Total"
+        ],
+        [
+          "Food",
+          "10",
+          "12",
+          "22"
+        ],
+        [
+          "Travel",
+          "5",
+          "7",
+          "12"
+        ]
+      ],
+      "cell_match_min": 0.85
+    },
+    {
+      "id": "real_arabic_docx",
+      "source": "real/arabic_line_001.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "Hello",
+        "World",
+        "TRAIL_AR",
+        "Bilingual"
+      ],
+      "text_recall_min": 0.85
+    },
+    {
+      "id": "real_scanlike_docx",
+      "source": "real/scanlike_001.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "ScanLike",
+        "ScanTokenSMOKEP000",
+        "Widget"
+      ],
+      "text_recall_min": 0.9
+    },
+    {
+      "id": "real_spanning_docx",
+      "source": "real/spanning_title_001.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "SpanningTitleAcrossTheFullPageWidthHere",
+        "SpanLeftA",
+        "SpanRightA"
+      ],
+      "text_recall_min": 0.9
+    },
+    {
+      "id": "real_rect_table_xlsx",
+      "source": "real/rect_table_001.pdf",
+      "pair": "pdf->xlsx",
+      "expected_cells": [
+        [
+          "Name",
+          "Qty",
+          "Cost"
+        ],
+        [
+          "Alpha",
+          "3",
+          "9"
+        ],
+        [
+          "Beta",
+          "4",
+          "8"
+        ]
+      ],
+      "cell_match_min": 0.7
+    },
+    {
+      "id": "real_hyphen_md",
+      "source": "real/hyphen_md_001.pdf",
+      "pair": "pdf->md",
+      "expected_tokens": [
+        "HyphenSmoke",
+        "word",
+        "Chapter",
+        "postprocess"
+      ],
+      "text_recall_min": 0.75
+    }
+  ]
+}

corpus/convert/third_party/LICENSE_CHECKLIST.md

@@ -0,0 +1,18 @@
+# License verification checklist (operator)
+
+Sign off before fetching each asset into `corpus/convert` or `gateway/models/layout`.
+
+| Asset | Upstream URL | License claimed | Verified by | Date | OK for prod train/eval? |
+| --- | --- | --- | --- | --- | --- |
+| DocLayNet | https://github.com/DS4SD/DocLayNet | CDLA-Permissive-1.0 | | | Y/N |
+| PubLayNet | https://github.com/ibm-aur-nlp/publaynet | CDLA-P + PMC CA images | | | Y/N |
+| PubTabNet | https://github.com/ibm-aur-nlp/PubTabNet | CDLA-P + PMC CA | | | Y/N |
+| CORD | https://github.com/clovaai/cord | CC-BY-4.0 | | | Y/N |
+| SROIE | https://rrc.cvc.uab.es/?ch=13 | CC-BY-4.0 (verify) | | | Y/N |
+| PP-DocLayout ONNX | PaddleOCR release / fetch script | Apache-2.0 | | | Y/N |
+| PP-OCRv5 Arabic rec ONNX | RapidAI ModelScope `arabic_PP-OCRv5_rec_mobile.onnx` + `ppocrv5_arabic_dict.txt` | Apache-2.0 (RapidAI/Paddle) | | | Y/N |
+| Self-gen fixtures | DocQube scripts | Proprietary/internal | | | Y |
+
+**Never fetch for prod:** FUNSD (NC), LayoutLMv3 weights (NC), DocLayout-YOLO (AGPL).
+
+Disk budgets: smoke ≤ 500 MB; full DocLayNet ~28 GB lab-only (`--full`).

corpus/convert/third_party/NOTICE

@@ -0,0 +1,40 @@
+DocQube convert — third-party NOTICE
+====================================
+
+Append a block below whenever you fetch a public dataset or layout model.
+Keep license names and upstream URLs accurate.
+
+How to append
+-------------
+1. Verify license on LICENSE_CHECKLIST.md
+2. Download via scripts/convert/fetch_public_eval.py or fetch_layout_onnx.py
+3. Add:
+
+   Name: <asset>
+   License: <SPDX or common name>
+   Source: <URL>
+   Used for: <eval|train|inference>
+   Date: <ISO date>
+
+Self-generated fixtures under corpus/convert (real/ / public/samples/invoices etc.)
+are owned by DocQube and require no third-party attribution.
+
+--- Upstream entries (filled by fetch scripts) ---
+
+Name: PP-DocLayoutV3 ONNX (layout.onnx / inference.onnx)
+License: Apache-2.0
+Source: https://huggingface.co/PaddlePaddle/PP-DocLayoutV3_onnx
+Used for: inference
+Date: 2026-08-27
+SHA256: 45bf71750b00739a41fc209f132eb104a4d6b5bb29483c9078164d8b87cf28ba
+Path: gateway/models/layout/v1/layout.onnx (copy of inference.onnx)
+
+Name: PP-OCRv5 Arabic recognition ONNX (rec.onnx)
+License: Apache-2.0
+Source: https://www.modelscope.cn/models/RapidAI/RapidOCR
+Used for: inference
+
+Name: PP-OCRv5 Arabic charset (arabic_dict.txt)
+License: Apache-2.0
+Source: https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/v3.9.2/paddle/PP-OCRv5/rec/arabic_PP-OCRv5_rec_mobile/ppocrv5_arabic_dict.txt
+Used for: inference

corpus/convert/third_party/checksums.sha256

@@ -0,0 +1,16 @@
+# SHA256 checksums for convert public samples and layout weights
+# Format: <sha256>  <relative-path-from-pdf/>
+# Populated by fetch_public_eval.py / fetch_layout_onnx.py / generate fixtures.
+58a051ee4dcc59e49b1c379ed36e528e1d99dc6968c9a5c11bf6934e0a8a1f36  corpus/convert/real/digital_report_001.pdf
+e67e0040b551d674ed57097d71b1d11bb2c0b45de18da902b847053f9b00d6e8  corpus/convert/real/simple_table_001.pdf
+ba2bf716c44970858e298c340fbd5e8a31c40639315fa2ce42da6bbee0c1a279  corpus/convert/real/invoice_001.pdf
+ffad883045c91f860039352ad9343fad1eb958e6a11fd49a4a2e7f8d22102a5c  corpus/convert/public/samples/invoices/synth_invoice_001.pdf
+0d9165e3d425d3782f4b41e38b890343c6f6cd032691d04f5e345f8ed1105211  corpus/convert/public/samples/pubtab/synth_table_001.pdf
+7fc1dc773d71db49bafdb0458d964e5ec68f76488f785d753f4575b1b685e40f  corpus/convert/public/samples/doclaynet/synth_digital_001.pdf
+314e13e9e4f1178dace76985b10cb41aa5db9a45aaa2e319dc21aab37a336023  corpus/convert/public/samples/doclaynet/synth_multicol_001.pdf
+e2e2f4f6671e753955036f4a0ac2b56e3534bf3901fe9feeeca0e68950a87749  corpus/convert/public/samples/path_a/path_a_sample.docx
+eb203ee73bea31647892d0892605cb65a2aad9a0abdb7a76e8b2e2e60e625775  corpus/convert/public/samples/path_a/path_a_sample.xlsx
+a33ad10e0028c1f84fd1ce229a4de2d0bdb931bc68ac01e80222dd9772008d05  corpus/convert/public/samples/images/smoke.png
+45bf71750b00739a41fc209f132eb104a4d6b5bb29483c9078164d8b87cf28ba  gateway/models/layout/v1/layout.onnx
+c1192e632d0baa9146ae5b756a0e635e3dc63c1733737ebfd1629e87144e9295  gateway/models/ocr/ar/v5/rec.onnx
+7f92f7dbb9b75a4787a83bfb4f6d14a8ab515525130c9d40a9036f61cf6999e9  gateway/models/ocr/ar/v5/arabic_dict.txt

corpus/edge-cases/annots.pdf

@@ -0,0 +1,395 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+  /AcroForm <<
+    /Fields [23 0 R]
+    /DR <<
+      /Font <<
+        /F1 7 0 R
+      >>
+    >>
+  >>
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /Count 2
+  /Kids [3 0 R 4 0 R]
+  /MediaBox [0 0 612 792]
+  /CropBox [0 0 612 792]
+  /Resources <<
+    /Font <<
+      /F1 7 0 R
+      /F2 8 0 R
+    >>
+    /ProcSet [/PDF /Text /ImageC]
+    /ExtGState <<
+      /GS0 24 0 R
+    >>
+  >>
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Contents 5 0 R
+  /Annots [15 0 R 16 0 R 17 0 R 18 0 R 19 0 R 20 0 R 21 0 R 22 0 R 23 0 R]
+>>
+endobj
+4 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Contents 6 0 R
+  /Annots [15 0 R 16 0 R 26 0 R]
+>>
+endobj
+5 0 obj <<
+  /Length 486
+>>
+stream
+BT
+70 700 Td
+/F1 18 Tf
+(Link Annotations - Page 1) Tj
+0 -65 Td
+/F2 14 Tf
+(1. Link with destination to first page) Tj
+10 -20 Td
+/F2 14 Tf
+(2. Link with destination to second page) Tj
+-12 -84 Td
+/F2 10 Tf
+(PDF Reference, Version 1.7, Section 8.4.5 defines Annotations) Tj
+2 -53 Td
+(3.  An example of Highlight with text notes) Tj
+0 -18 Td
+(https://pdfium.googlesource.com/pdfium is link in plain text, not link annotation. These are referred to) Tj
+0 -17 Td
+(as WebLinks in PDFium.)Tj
+ET
+endstream
+endobj
+6 0 obj <<
+  /Length 185
+>>
+stream
+BT
+70 700 Td
+/F1 18 Tf
+(Link Annotations - Page 2) Tj
+0 -65 Td
+/F2 14 Tf
+(1. Link with destination to first page) Tj
+10 -20 Td
+/F2 14 Tf
+(2. Link with destination to second page) Tj
+ET
+endstream
+endobj
+7 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Times-Roman
+>>
+endobj
+8 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+9 0 obj <<
+  /Type /XObject
+  /Subtype /Form
+  /FormType 1
+  /Length 18
+  /BBox [293 530 349 542]
+  /Resources <<
+    /XObject <<
+      /Form0 10 0 R
+    >>
+    /ExtGState <<
+      /GS0 25 0 R
+    >>
+  >>
+>>
+stream
+/GS0 gs
+/Form0 Do
+endstream
+endobj
+10 0 obj <<
+  /Type /XObject
+  /Subtype /Form
+  /FormType 1
+  /Group <<
+    /S /Transparency
+  >>
+  /Length 59
+  /BBox [293 530 349 542]
+>>
+stream
+1.0 1.0 0.0 rg
+293 530 m
+349 530 l
+349 542 l
+293 542 l
+h f
+endstream
+endobj
+11 0 obj <<
+  /Type /XObject
+  /Subtype /Form
+  /FormType 1
+  /Length 18
+  /BBox [83 440 178 453]
+  /Resources <<
+    /XObject <<
+      /Form0 12 0 R
+    >>
+    /ExtGState <<
+      /GS0 25 0 R
+    >>
+  >>
+>>
+stream
+/GS0 gs
+/Form0 Do
+endstream
+endobj
+12 0 obj <<
+  /Type /XObject
+  /Subtype /Form
+  /FormType 1
+  /Group <<
+    /S /Transparency
+  >>
+  /Length 57
+  /BBox [83 440 178 453]
+>>
+stream
+0.0 1.0 1.0 rg
+83 440 m
+178 440 l
+178 453 l
+83 453 l
+h f
+endstream
+endobj
+13 0 obj <<
+  /Type /XObject
+  /Subtype /Form
+  /FormType 1
+  /Length 18
+  /BBox [149 476 191 487]
+  /Resources <<
+    /XObject <<
+      /Form0 14 0 R
+    >>
+    /ExtGState <<
+      /GS0 25 0 R
+    >>
+  >>
+>>
+stream
+/GS0 gs
+/Form0 Do
+endstream
+endobj
+14 0 obj <<
+  /Type /XObject
+  /Subtype /Form
+  /FormType 1
+  /Group <<
+    /S /Transparency
+  >>
+  /Length 59
+  /BBox [149 476 191 487]
+>>
+stream
+0.0 1.0 0.0 rg
+149 476 m
+191 476 l
+191 487 l
+149 487 l
+h f
+endstream
+endobj
+15 0 obj <<
+  /Type /Annot
+  /Subtype /Link
+  /BS <<
+    /W 0
+  >>
+  /Rect [69 633 542 653]
+  /Dest [3 0 R /XYZ 200 725 0]
+  /F 4
+>>
+endobj
+16 0 obj <<
+  /Type /Annot
+  /Subtype /Link
+  /BS <<
+    /W 0
+  >>
+  /Rect [80 613 542 633]
+  /Dest [4 0 R /XYZ 200 725 0]
+  /F 4
+>>
+endobj
+17 0 obj <<
+  /Type /Annot
+  /Subtype /Link
+  /BS <<
+    /W 0
+  >>
+  /Rect [66 529 196 544]
+  /A <<
+    /Type /Action
+    /URI (https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/pdf_reference_1-7.pdf)
+    /S /URI
+  >>
+  /F 4
+>>
+endobj
+18 0 obj <<
+  /Type /Annot
+  /Subtype /Link
+  /BS <<
+    /W 0
+  >>
+  /Rect [83 440 178 453]
+  /QuadPoints [83 453 178 453 83 440 178 440]
+  /A <<
+    /Type /Action
+    /URI (https://cs.chromium.org/chromium/src/third_party/pdfium/public/fpdf_text.h)
+    /S /URI
+  >>
+  /F 4
+>>
+endobj
+19 0 obj <<
+  /Type /Annot
+  /Subtype /Highlight
+  /AP <<
+    /N 9 0 R
+  >>
+  /NM (Highlight-1)
+  /F 4
+  /QuadPoints [293 542 349 542 293 530 349 530]
+  /P 3 0 R
+  /C [1 0.90196 0]
+  /Rect [293 530 349 542]
+>>
+endobj
+20 0 obj <<
+  /Type /Annot
+  /Subtype /Highlight
+  /AP <<
+    /N 11 0 R
+  >>
+  /NM (Highlight-2)
+  /F 4
+  /QuadPoints [83 453 178 453 83 440 178 440]
+  /P 3 0 R
+  /C [0.26667 0.78431 0.96078]
+  /Rect [83 440 178 453]
+>>
+endobj
+21 0 obj <<
+  /Type /Annot
+  /Subtype /Popup
+  /Parent 22 0 R
+  /Rect [191 377 443 488]
+>>
+endobj
+22 0 obj <<
+  /Type /Annot
+  /Subtype /Highlight
+  /Popup 21 0 R
+  /AP <<
+    /N 13 0 R
+  >>
+  /NM (Highlight-With-Popup-1)
+  /Contents (Text Note)
+  /QuadPoints [149 487 191 487 149 476 191 476]
+  /P 3 0 R
+  /C [0.14902 0.90196 0]
+  /Rect [149 476 191 487]
+  /F 4
+>>
+endobj
+23 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 131072
+  /T (Combo1)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [70 350 170 380]
+  /Opt [(Highlight) (Link) (Popup) (Widget)]
+>>
+endobj
+24 0 obj <<
+  /ca 1
+  /Type /ExtGState
+  /CA 1
+  /BM /Normal
+>>
+endobj
+25 0 obj <<
+  /ca 1
+  /Type /ExtGState
+  /CA 1
+  /AIS false
+  /BM /Multiply
+>>
+endobj
+26 0 obj <<
+  /Type /Annot
+  /Subtype /Square
+  /Border [0 0 2]
+  /C [1 0 0]
+  /F 4
+  /P 3 0 R
+  /Rect [50 100 60 120]
+>>
+endobj
+xref
+0 27
+0000000000 65535 f 
+0000000015 00000 n 
+0000000169 00000 n 
+0000000439 00000 n 
+0000000583 00000 n 
+0000000685 00000 n 
+0000001223 00000 n 
+0000001460 00000 n 
+0000001538 00000 n 
+0000001614 00000 n 
+0000001864 00000 n 
+0000002087 00000 n 
+0000002337 00000 n 
+0000002557 00000 n 
+0000002808 00000 n 
+0000003031 00000 n 
+0000003171 00000 n 
+0000003311 00000 n 
+0000003558 00000 n 
+0000003842 00000 n 
+0000004059 00000 n 
+0000004286 00000 n 
+0000004384 00000 n 
+0000004659 00000 n 
+0000004849 00000 n 
+0000004920 00000 n 
+0000005006 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 27
+>>
+startxref
+5135
+%%EOF

corpus/edge-cases/bookmarks.pdf

@@ -0,0 +1,162 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+  /Outlines 8 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /Count 2
+  /Kids [
+    3 0 R
+    4 0 R
+  ]
+>>
+endobj
+% Page number 0.
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 5 0 R
+    >>
+  >>
+  /Contents [6 0 R]
+  /MediaBox [0 0 612 792]
+>>
+endobj
+% Page number 1.
+4 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources  <<
+    /Font <<
+      /F1 5 0 R
+    >>
+  >>
+  /Contents [7 0 R]
+  /MediaBox [0 0 612 792]
+>>
+endobj
+% Font resource.
+5 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Arial
+>>
+endobj
+% Content for page 0.
+6 0 obj <<
+  /Length 37
+>>
+stream
+BT
+/F1 20 Tf
+100 600 TD (Page1)Tj
+ET
+endstream
+endobj
+% Content for page 1.
+7 0 obj <<
+  /Length 37
+>>
+stream
+BT
+/F1 20 Tf
+100 600 TD (Page2)Tj
+ET
+endstream
+endobj
+% Root bookmark
+8 0 obj <<
+  /Type /Outlines
+  /Count 3
+  /First 9 0 R
+  /Last 12 0 R
+>>
+endobj
+% First child bookmark (leaf node)
+9 0 obj <<
+  /Title (A Good Beginning)
+  /Parent 8 0 R
+  /Next 10 0 R
+  /Dest (foo)
+>>
+endobj
+% Second child bookmark (open)
+10 0 obj <<
+  /Title (Open Middle)
+  /Parent 8 0 R
+  /First 11 0 R
+  /Last 11 0 R
+  /Prev 9 0 R
+  /Next 12 0 R
+  /Count 1
+  /A <<
+    /Type /Action
+    /S /URI
+    /URI (https://theplay.test)
+  >>
+>>
+endobj
+% First grandchild bookmark
+11 0 obj <<
+  /Title (Open Middle Descendant)
+  /Parent 10 0 R
+  /Dest [3 0 R /XYZ 100 200 0]
+>>
+endobj
+% Third child bookmark (closed)
+12 0 obj <<
+  /Title (A Good Closed Ending)
+  /Parent 8 0 R
+  /First 13 0 R
+  /Last 14 0 R
+  /Prev 10 0 R
+  /Count -2
+  /Dest (bar)
+>>
+endobj
+% Second grandchild bookmark
+13 0 obj <<
+  /Title (A Good Closed Ending Descendant)
+  /Parent 12 0 R
+  /Next 14 0 R
+  /Dest (bar)
+>>
+endobj
+% Third grandchild bookmark
+14 0 obj <<
+  /Title (A Good Closed Ending Descendant 2)
+  /Parent 12 0 R
+  /Prev 13 0 R
+  /Dest (bar)
+>>
+endobj
+xref
+0 15
+0000000000 65535 f 
+0000000015 00000 n 
+0000000086 00000 n 
+0000000184 00000 n 
+0000000355 00000 n 
+0000000527 00000 n 
+0000000621 00000 n 
+0000000731 00000 n 
+0000000835 00000 n 
+0000000950 00000 n 
+0000001075 00000 n 
+0000001310 00000 n 
+0000001446 00000 n 
+0000001617 00000 n 
+0000001756 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 15
+>>
+startxref
+1869
+%%EOF

corpus/edge-cases/combobox_form.pdf

@@ -0,0 +1,109 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+  /AcroForm <<
+    /Fields [ 8 0 R 9 0 R 10 0 R ]
+    /DR 4 0 R
+  >>
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /Count 1
+  /Kids [ 3 0 R ]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources 4 0 R
+  /MediaBox [ 0 0 300 600 ]
+  /Contents 7 0 R
+  /Annots [ 8 0 R 9 0 R 10 0 R ]
+>>
+endobj
+4 0 obj <<
+  /Font 5 0 R
+>>
+endobj
+5 0 obj <<
+  /F1 6 0 R
+>>
+endobj
+6 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+7 0 obj <<
+  /Length 51
+>>
+stream
+BT
+0 0 0 rg
+/F1 12 Tf
+100 450 Td
+(Test Form) Tj
+ET
+endstream
+endobj
+8 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 393216
+  /T (Combo_Editable)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [ 100 350 200 380 ]
+  /Opt [[(foo) (Foo)] [(bar) (Bar)] [(qux) (Qux)]]
+>>
+endobj
+9 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 131072
+  /T (Combo1)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [ 100 400 200 430 ]
+  /Opt [(Apple) (Banana) (Cherry) (Date) (Elderberry) (Fig) (Guava) (Honeydew)
+        (Indian Fig) (Jackfruit) (Kiwi) (Lemon) (Mango) (Nectarine) (Orange)
+        (Persimmon) (Quince) (Raspberry) (Strawberry) (Tamarind) (Ugli Fruit)
+        (Voavanga) (Wolfberry) (Xigua) (Yangmei) (Zucchini)]
+  /V (Banana)
+>>
+endobj
+10 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 131073
+  /T (Combo_ReadOnly)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [ 100 500 200 530 ]
+  /Opt [(Dog) (Elephant) (Frog)]
+>>
+endobj
+xref
+0 11
+0000000000 65535 f 
+0000000015 00000 n 
+0000000137 00000 n 
+0000000202 00000 n 
+0000000351 00000 n 
+0000000386 00000 n 
+0000000419 00000 n 
+0000000495 00000 n 
+0000000597 00000 n 
+0000000803 00000 n 
+0000001259 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 11
+>>
+startxref
+1448
+%%EOF

corpus/edge-cases/empty_xref.pdf

@@ -0,0 +1,70 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [ 0 0 200 200 ]
+  /Count 1
+  /Kids [ 3 0 R ]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 4 0 R
+      /F2 5 0 R
+    >>
+  >>
+  /Contents 6 0 R
+>>
+endobj
+4 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Times-Roman
+>>
+endobj
+5 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+6 0 obj <<
+>>
+stream
+BT
+20 50 Td
+/F1 12 Tf
+(Hello, world!) Tj
+0 50 Td
+/F2 16 Tf
+(Goodbye, world!) Tj
+ET
+endstream
+endobj
+xref
+0 7
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000161 00000 n 
+0000000303 00000 n 
+0000000381 00000 n 
+0000000457 00000 n 
+trailer<< /Root 1 0 R /Size 7 >>
+startxref
+578
+%%EOF
+xref
+0 0
+trailer<< /Root 1 0 R /Size 0 /Prev 578 >>
+startxref
+780
+%%EOF

corpus/edge-cases/listbox_form.pdf

@@ -0,0 +1,165 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+  /AcroForm <<
+    /Fields [8 0 R 9 0 R 10 0 R 11 0 R 12 0 R 13 0 R 14 0 R]
+    /DR 4 0 R
+  >>
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /Count 1
+  /Kids [3 0 R]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources 4 0 R
+  /MediaBox [0 0 300 600]
+  /Contents 7 0 R
+  /Annots [8 0 R 9 0 R 10 0 R 11 0 R 12 0 R 13 0 R 14 0 R]
+>>
+endobj
+4 0 obj <<
+  /Font 5 0 R
+>>
+endobj
+5 0 obj <<
+  /F1 6 0 R
+>>
+endobj
+6 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+7 0 obj <<
+  /Length 51
+>>
+stream
+BT
+0 0 0 rg
+/F1 12 Tf
+100 450 Td
+(Test Form) Tj
+ET
+endstream
+endobj
+8 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 0
+  /T (Listbox_SingleSelect)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [100 350 200 380]
+  /Opt [[(foo) (Foo)] [(bar) (Bar)] [(qux) (Qux)]]
+>>
+endobj
+9 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 2097152
+  /T (Listbox_MultiSelect)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [100 400 200 430]
+  /Opt [(Apple) (Banana) (Cherry) (Date) (Elderberry) (Fig) (Guava) (Honeydew)
+        (Indian Fig) (Jackfruit) (Kiwi) (Lemon) (Mango) (Nectarine) (Orange)
+        (Persimmon) (Quince) (Raspberry) (Strawberry) (Tamarind) (Ugli Fruit)
+        (Voavanga) (Wolfberry) (Xigua) (Yangmei) (Zucchini)]
+  /V (Banana)
+>>
+endobj
+10 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 1
+  /T (Listbox_ReadOnly)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [100 500 200 530]
+  /Opt [(Dog) (Elephant) (Frog)]
+>>
+endobj
+11 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 2097152
+  /T (Listbox_MultiSelectMultipleIndices)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [100 250 200 280]
+  /Opt [(Albania) (Belgium) (Croatia) (Denmark) (Estonia)]
+  /I [1 3]
+>>
+endobj
+12 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 2097152
+  /T (Listbox_MultiSelectMultipleValues)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [100 200 200 230]
+  /Opt [(Alpha) (Beta) (Gamma) (Delta) (Epsilon)]
+  /V [(Epsilon) (Gamma)]
+>>
+endobj
+13 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 2097152
+  /T (Listbox_MultiSelectMultipleMismatch)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [100 150 200 180]
+  /Opt [(Alligator) (Bear) (Cougar) (Deer) (Echidna)]
+  /V [(Alligator) (Cougar)]
+  /I [1 3 4]
+>>
+endobj
+14 0 obj <<
+  /Type /Annot
+  /Subtype /Widget
+  /FT /Ch
+  /Ff 0
+  /T (Listbox_SingleSelectLastSelected)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [100 100 200 130]
+  /Opt [(Alberta) (British Columbia) (Manitoba) (New Brunswick)
+        (Newfoundland and Labrador) (Nova Scotia) (Ontario)
+        (Prince Edward Island) (Quebec) (Saskatchewan)]
+  /V (Saskatchewan)
+  /TI 9
+>>
+endobj
+xref
+0 15
+0000000000 65535 f 
+0000000015 00000 n 
+0000000163 00000 n 
+0000000226 00000 n 
+0000000399 00000 n 
+0000000434 00000 n 
+0000000467 00000 n 
+0000000543 00000 n 
+0000000645 00000 n 
+0000000850 00000 n 
+0000001318 00000 n 
+0000001502 00000 n 
+0000001747 00000 n 
+0000001996 00000 n 
+0000002267 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 15
+>>
+startxref
+2642
+%%EOF

corpus/edge-cases/no_page_count.pdf

@@ -0,0 +1,63 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [0 0 200 200]
+  /Kids [3 0 R 3 0 R]
+>>
+endobj
+3 0 obj <<
+  /Type /Pages
+  /Kids [4 0 R 4 0 R 4 0 R]
+>>
+endobj
+4 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 5 0 R
+    >>
+  >>
+  /Contents 6 0 R
+>>
+endobj
+5 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Times-Roman
+>>
+endobj
+
+6 0 obj <<
+  /Length 44
+>>
+stream
+BT
+20 50 Td
+/F1 12 Tf
+(Hello, world!) Tj
+ET
+endstream
+endobj
+xref
+0 7
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000152 00000 n 
+0000000216 00000 n 
+0000000342 00000 n 
+0000000421 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 7
+>>
+startxref
+516
+%%EOF

corpus/edge-cases/restricted.pdf

@@ -0,0 +1,41 @@
+%PDF-1.7
+%¿÷¢þ
+1 0 obj
+<< /Extensions << /ADBE << /BaseVersion /1.7 /ExtensionLevel 8 >> >> /Pages 2 0 R /Type /Catalog >>
+endobj
+2 0 obj
+<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
+endobj
+3 0 obj
+<< /Contents 4 0 R /MediaBox [ 0 0 612 792 ] /Parent 2 0 R /Resources << /Font << /F1 5 0 R /F2 6 0 R >> >> /Type /Page >>
+endobj
+4 0 obj
+<< /Length 352 /Filter /FlateDecode >>
+stream
+¥Æçÿ¥yÀ�L4Wˆ®{©ÈÒ
+뫉û\ÐòòÆӂ˼`ÁS¢Éñ8¦1¾4â€v×än˳?ÁD<¨ž¤#“Õ6îçJµÔ?à�šýÇÝ
3¥­Å g&eökûóA¹ŽŒpÉ@�…“¶QÂÅþÊîšÔÊ>ȃ'«úœˆ l•¡:ùšÛ*ÚáS@5xw^Úuž�…¦€ëv‰véúY\Bæ&Ť.õÖÕ¼ó©f(+ô£KˆN.ÁÎIÀ•èUÔæ�xu&²¡ÑTtàó¸®7©ÿÝÐ<c>þ"ˆþâóÈÀeêMq²U]æ¿—ø¥Ê¸õ=%dÉ®å(³y’†Mçƒíº7Üî‚¢øv_ݧã»çÿX�POç\'f_áYúof7}/tõaCÉàÿInhìôòÜ\�V®wRøæ[pq5X‘3;Ý�|šš sŒt·JÕ 
+endstream
+endobj
+5 0 obj
+<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Subtype /Type1 /Type /Font >>
+endobj
+6 0 obj
+<< /BaseFont /Times-Roman /Encoding /WinAnsiEncoding /Subtype /Type1 /Type /Font >>
+endobj
+7 0 obj
+<< /CF << /StdCF << /AuthEvent /DocOpen /CFM /AESV3 /Length 32 >> >> /Filter /Standard /Length 256 /O <eed09d7bae817b88ec68c4bba71e4688bad49a26f13d5f1b558fd8d6246663774b7e74b08734ad7a57473b33ec19e47d> /OE <33dc073711e1735085e2efd64928fa75af518dcf25ec7ff7d3ae22976ab8ded4> /P -3136 /Perms <b360a35c81e6b6380ef770702952140a> /R 6 /StmF /StdCF /StrF /StdCF /U <0d6d86441425cdaee0cda2eec22acd2391b314ff4c0bb60c90e9955907889824f87b528709af1246ab59dd722a89cb2a> /UE <2da7189a149f6fcbdf38688a565f280f8e6a4b727a3f528dbb9e014a88ebd88f> /V 5 >>
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000015 00000 n 
+0000000130 00000 n 
+0000000189 00000 n 
+0000000327 00000 n 
+0000000751 00000 n 
+0000000848 00000 n 
+0000000947 00000 n 
+trailer << /Root 1 0 R /Size 8 /ID [<f341436d4fd6835a35fb5f4313bdd156><f341436d4fd6835a35fb5f4313bdd156>] /Encrypt 7 0 R >>
+startxref
+1497
+%%EOF

corpus/edge-cases/text_form.pdf

@@ -0,0 +1,70 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj
+<<
+  /Type /Catalog
+  /Pages 2 0 R
+  /AcroForm << /Fields [ 4 0 R ] /DR 5 0 R >>
+>>
+endobj
+2 0 obj
+<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
+endobj
+3 0 obj
+<<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources 5 0 R
+  /MediaBox [ 0 0 300 300 ]
+  /Contents 8 0 R
+  /Annots [ 4 0 R ]
+>>
+endobj
+4 0 obj
+<<
+  /Type /Annot
+  /FT /Tx
+  /T (Text Box)
+  /DA (0 0 0 rg /F1 12 Tf)
+  /Rect [ 100 100 200 130 ]
+  /Subtype /Widget
+>>
+endobj
+5 0 obj
+<< /Font 6 0 R >>
+endobj
+6 0 obj
+<< /F1 7 0 R >>
+endobj
+7 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+8 0 obj
+<< /Length 51 >>
+stream
+BT
+0 0 0 rg
+/F1 12 Tf
+100 150 Td
+(Test Form) Tj
+ET
+endstream
+endobj
+xref
+0 9
+0000000000 65535 f 
+0000000015 00000 n 
+0000000114 00000 n 
+0000000173 00000 n 
+0000000309 00000 n 
+0000000445 00000 n 
+0000000478 00000 n 
+0000000509 00000 n 
+0000000585 00000 n 
+trailer<< /Root 1 0 R /Size 9 >>
+startxref
+685
+%%EOF

corpus/edge-cases/unsupported_feature.pdf

@@ -0,0 +1,37 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+  /Collection /Test
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /Count 3
+  /Kids [
+    3 0 R
+  ]
+>>
+endobj
+% Page number 0.
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <</F1 15 0 R>>
+  >>
+  /MediaBox [0 0 612 792]
+  /Tabs /R
+>>
+endobj
+xref
+0 4
+0000000000 65535 f 
+0000000015 00000 n 
+0000000088 00000 n 
+0000000176 00000 n 
+trailer<< /Root 1 0 R /Size 4 >>
+startxref
+310
+%%EOF

corpus/edge-cases/zero_length_stream.pdf

@@ -0,0 +1,71 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [ 0 0 200 200 ]
+  /Count 1
+  /Kids [ 3 0 R ]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 4 0 R
+      /F2 5 0 R
+    >>
+  >>
+  /Contents [6 0 R 7 0 R]
+>>
+endobj
+4 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Times-Roman
+>>
+endobj
+5 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Helvetica
+>>
+endobj
+6 0 obj <<
+  /Filter /FlateDecode
+  /Length 0
+>>
+stream
+endstream
+endobj
+7 0 obj <<
+>>
+stream
+BT
+20 50 Td
+/F1 12 Tf
+(Hello, world!) Tj
+0 50 Td
+/F2 16 Tf
+(Goodbye, world!) Tj
+ET
+endstream
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000015 00000 n 
+0000000061 00000 n 
+0000000154 00000 n 
+0000000304 00000 n 
+0000000382 00000 n 
+0000000458 00000 n 
+0000000531 00000 n 
+trailer<< /Root 1 0 R /Size 8 >>
+startxref
+652
+%%EOF

corpus/fonts/cns1_test.pdf

@@ -0,0 +1,90 @@
+%PDF-1.3
+%“Œ‹ž ReportLab Generated PDF document (opensource)
+1 0 obj
+<<
+/F1 2 0 R /F2 3 0 R
+>>
+endobj
+2 0 obj
+<<
+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
+>>
+endobj
+3 0 obj
+<<
+/BaseFont /MSung-Light /DescendantFonts [ <<
+/BaseFont /MSung-Light /CIDSystemInfo <<
+/Ordering (CNS1) /Registry (Adobe) /Supplement 1
+>> /DW 1000 /FontDescriptor <<
+/Ascent 752 /CapHeight 737 /Descent -271 /Flags 6 /FontBBox [ -160 -249 1015 888 ] /FontName /MSung-Light 
+  /ItalicAngle 0 /Leading 148 /MaxWidth 1000 /MissingWidth 500 /StemH 45 /StemV 58 
+  /Type /FontDescriptor /XHeight 553
+>> /Subtype /CIDFontType0 /Type /Font 
+  /W [ 1 2 250 3 [ 408 668 490 875 698 250 240 ] 10 [ 240 417 667 250 313 250 520 500 ] 18 26 500 
+  27 28 250 29 31 667 32 [ 396 921 677 615 719 760 625 552 771 802 
+  354 ] 43 [ 354 781 604 927 750 823 563 823 729 542 
+  698 771 729 948 771 677 635 344 520 344 
+  469 500 250 469 521 427 521 438 271 469 
+  531 250 ] 
+  75 [ 250 458 240 802 531 500 521 ] 82 [ 521 365 333 292 521 458 677 479 458 427 
+  480 496 480 667 ] ]
+>> ] /Encoding /UniGB-UCS2-H /Name /F2 /Subtype /Type0 /Type /Font
+>>
+endobj
+4 0 obj
+<<
+/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>> 
+  /Type /Page
+>>
+endobj
+5 0 obj
+<<
+/PageMode /UseNone /Pages 7 0 R /Type /Catalog
+>>
+endobj
+6 0 obj
+<<
+/Author (anonymous) /CreationDate (D:20260602153802+05'00') /Creator (anonymous) /Keywords () /ModDate (D:20260602153802+05'00') /Producer (ReportLab PDF Library - \(opensource\)) 
+  /Subject (unspecified) /Title (untitled) /Trapped /False
+>>
+endobj
+7 0 obj
+<<
+/Count 1 /Kids [ 4 0 R ] /Type /Pages
+>>
+endobj
+8 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 107
+>>
+stream
+GapQh0E=F,0U\H3T\pNYT^QKk?tc>IP,;W#U1^23ihPEM_?C]6_CBF/28[_U!/s9cYpe/lM_Qn>nC.&g0fCf=<!^TD#gi_<=5X,[c-mU(~>endstream
+endobj
+xref
+0 9
+0000000000 65535 f 
+0000000061 00000 n 
+0000000102 00000 n 
+0000000209 00000 n 
+0000001155 00000 n 
+0000001358 00000 n 
+0000001426 00000 n 
+0000001687 00000 n 
+0000001746 00000 n 
+trailer
+<<
+/ID 
+[<01c8dab3d2c3e771bf716fccf2b52ce4><01c8dab3d2c3e771bf716fccf2b52ce4>]
+% ReportLab generated PDF document -- digest (opensource)
+
+/Info 6 0 R
+/Root 5 0 R
+/Size 9
+>>
+startxref
+1943
+%%EOF

corpus/fonts/custom_encoding.pdf

@@ -0,0 +1,62 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+
+3 0 obj
+<< /Type /Page /Parent 2 0 R
+   /MediaBox [0 0 612 792]
+   /Contents 4 0 R
+   /Resources <<
+     /Font <<
+       /F1 5 0 R
+       /F2 6 0 R
+     >>
+   >>
+>>
+endobj
+
+4 0 obj
+<< /Length 213 >>
+stream
+BT
+/F1 12 Tf
+72 720 Td
+(Custom Encoding Test Document) Tj
+-72 -720 Td
+72 700 Td
+(WinAnsi encoding verification text.) Tj
+-72 -700 Td
+72 680 Td
+(All standard ASCII chars should decode correctly.) Tj
+-72 -680 Td
+ET
+endstream
+endobj
+
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
+endobj
+
+6 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
+endobj
+
+xref
+0 7
+0000000000 65535 f 
+0000000009 00000 n 
+0000000059 00000 n 
+0000000117 00000 n 
+0000000290 00000 n 
+0000000554 00000 n 
+0000000652 00000 n 
+trailer
+<< /Size 7 /Root 1 0 R >>
+startxref
+752
+%%EOF

corpus/fonts/embedded_cid_font.pdf

@@ -0,0 +1,51 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+
+3 0 obj
+<< /Type /Page /Parent 2 0 R
+   /MediaBox [0 0 612 792]
+   /Contents 4 0 R
+   /Resources << /Font << /F1 5 0 R >> >>
+>>
+endobj
+
+4 0 obj
+<< /Length 58 >>
+stream
+BT
+/F1 12 Tf
+72 720 Td
+(CID Text) Tj
+-72 -720 Td
+ET
+endstream
+endobj
+
+5 0 obj
+<< /Type /Font /Subtype /Type0 /BaseFont /NotoSansCJKjp-Regular /Encoding /Identity-H /DescendantFonts [6 0 R] >>
+endobj
+
+6 0 obj
+<< /Type /Font /Subtype /CIDFontType2 /BaseFont /NotoSansCJKjp-Regular /CIDSystemInfo << /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >> >>
+endobj
+
+xref
+0 7
+0000000000 65535 f 
+0000000010 00000 n 
+0000000064 00000 n 
+0000000126 00000 n 
+0000000270 00000 n 
+0000000384 00000 n 
+0000000518 00000 n 
+trailer
+<< /Size 7 /Root 1 0 R >>
+startxref
+684
+%%EOF

corpus/fonts/embedded_truetype.pdf

@@ -0,0 +1,63 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+
+3 0 obj
+<< /Type /Page /Parent 2 0 R
+   /MediaBox [0 0 612 792]
+   /Contents 4 0 R
+   /Resources <<
+     /Font <<
+       /F1 5 0 R
+       /F2 6 0 R
+     >>
+   >>
+>>
+endobj
+
+4 0 obj
+<< /Length 225 >>
+stream
+BT
+/F1 14 Tf
+72 720 Td
+(Embedded TrueType Font Test Document) Tj
+-72 -720 Td
+/F1 12 Tf
+72 700 Td
+(This PDF uses a referenced TrueType font.) Tj
+-72 -700 Td
+72 680 Td
+(Text extraction should work correctly.) Tj
+-72 -680 Td
+ET
+endstream
+endobj
+
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
+endobj
+
+6 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
+endobj
+
+xref
+0 7
+0000000000 65535 f 
+0000000009 00000 n 
+0000000059 00000 n 
+0000000117 00000 n 
+0000000290 00000 n 
+0000000566 00000 n 
+0000000664 00000 n 
+trailer
+<< /Size 7 /Root 1 0 R >>
+startxref
+764
+%%EOF

corpus/fonts/hebrew_mirrored.pdf

@@ -0,0 +1,175 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /Count 1
+  /Kids [3 0 R]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Contents 4 0 R
+  /MediaBox [0 0 200 200]
+  /Resources <<
+    /ProcSet [/PDF /Text]
+    /Font <<
+      /F1 5 0 R
+    >>
+  >>
+>>
+endobj
+4 0 obj <<
+  /Length 110
+>>
+stream
+BT
+/F1 12 Tf
+150 160 Td
+[<01>2<02>2<03>-4<02>2<04>5<05>]TJ
+ET
+
+BT
+0 100 Td
+-1 0 0 1 50 40 Tm
+[<01>5<05>]TJ
+ET
+endstream
+endobj
+5 0 obj <<
+  /Type /Font
+  /Subtype /TrueType
+  /BaseFont /BAAAAA+NotoSansHebrew-Regular
+  /FirstChar 0
+  /FontDescriptor 6 0 R
+  /LastChar 5
+  /ToUnicode 8 0 R
+  /Widths [600 294 235 645 397 542]
+>>
+endobj
+6 0 obj <<
+  /Type /FontDescriptor
+  /Ascent 1069
+  /CapHeight 869
+  /Descent -293
+  /Flags 4
+  /FontBBox [-210 -252 716 869]
+  /FontFile2 7 0 R
+  /FontName /BAAAAA+NotoSansHebrew-Regular
+  /ItalicAngle 0
+  /StemV 80
+>>
+endobj
+7 0 obj <<
+  /Filter [/ASCII85Decode /FlateDecode]
+  /Length 4240
+>>
+stream
+GhVOeCMXsCFZt#(%p<i9AU.uO6]Ye/ifKHXbLV!I1<gsgHY$Eg-tbPt@rk=Ap?l+FQ/u;^8JASXVQS>V
+nuG<j"nF$,Rii%*89oFrq6<34,1m)cJ0IF5(22*&"c<?^q>^jZEkc[.-qdl^dk1F&,fKT9F2)eaH/1QX
+<gN4)W0n]g:798;]<j42dYR+b7&$II`er0Do2PVi55OH.DU&3oF7^D>=p%W;pT6OOkC.gL0;aQZ%EQ]$
+CMpO/ouiOe*=oqMFW5p^ge<JWIt[XVg[cqr8aR%^g[TKTpYK&S\F4)WJbd&hhE(,./q'ef/\k-FOKm`h
+f=q)o6_0Sd-Ap"u/`'bn6!o+8Zu;5]Q^2D[J,6jbf.d4Ck't7.im"Z=^^/7D4VXdnJ*hX.ia&@S<Zl_D
+Xq`#Xlf3.[Q6GL-.IS+nj<JEQ1+=HefoeH8Qi0[)c<.\^P$4LTD?DamV./;?-PDR^8[b$k8'^LB2C4&'
+NUg=HC6k8A'.Knb7Wh)elDCX74)L`tc?`Z\Sb;i$>AElUWZdj+!8FWu:gnKPdkmGSM;%q39@mC[7<ZEH
+BP,jX746Y[8VL'e=mlrkg+[&/4Xn#1'(72pCc`cXNN+/Z0?DM&g%BgA@co7e)9J3X;X\\(5@"=N^/d""
+T`@poCcT@G`,Q43B>^KnLk;$<#C1`ZRI:qa#-tMr)R?VE.RR\7=sKe@N)#YuA]JfMaHgYN/!?c&OsZTe
+^cXe0%l-1*krrj[$8W`Wl)*K?>t9dJirfrR7MYH\=XPN5Q2qL_I4eudg4b]KRT\<OAl/T-PZ'?3dZ$p;
+<A/YSC[`O')u&1O:[moi)'2=Vd5KR72=0E6LAR(+/#@80mS5hidn)NRbS3]PMfW2+=&LegaZXffSM\+*
+%@ESe'CB7@?)cYMPG3X<A98q?pK.WHgPW4%FOOV*1/i@`;U-_0)m/crX^pV$",X/"BsMn?;R::1,iO?7
+d*"n-c#gLp.bPO!Jk.*r-gP*5[&0>a[qOG`>on_9`Lhgl?)c@P!H#2#PFffIEAQP^TS_DS:Xem?lK2!M
+h#c+\A=NN1)ZW-WA38,70l'k`MSW-$8rA0><3W]ffk"ZqCrdIAQBE9>j@mZ'<HHGofhsV-;mOl@]Mb1L
+qSIIYnkLM"Z97EVYElh]%[>>bb#jgf**OlY[IHaK5,@C./bHbDalE":hWu.1N,@;S&gl,n3:<uV$;N?L
+*(2O_s!JrF=fbn$ki"!LOY9qefX_V\<NFjuO0ngY8TpJ(,;39KIP'3^N)o:sU^S>oeCuK%7\,dgAiUN1
+E!0r_=$ol73>k&7]_*E"mUSrX[62980+uUk50ME#I*V:L4N8s+bi=/qMstu,ntJ5f[(HSa5@JhaGYD:Q
+]auqs7`nFM/]DAtfVG=ND6[_hnTB=9[;GHTf&F!MBNQP7P.LnAZ2VP"Ploh`?NTa8nEUYFj3^8B4<?t>
+4SBp^*fhF-*fhL/*r?NkO.Sgf4:WJt*n-E0W]KmAKl)1[L<V@'WjXmHBl;cn9--aK@lTjrE@*#k?-6on
+,:;=;+3B=[oI&q3p0&\dDEW8m^^H(gHjR8,7ldq.4hG]/>/mf;Hl1F^:B&.Xiqu!h(RA"GpXOmI2eJ(p
+-Ufu7H4'I5!71?\VoB`KajX::mq>=Rr&Q;g]E7s$-kZ.E)/3T4mfoT]_V3^s3J#ALKI7nF?TacHK]qhj
+L<ncjl]HN[$fPFZ;`(\GZEhSIf%/9H@>qA*jpO7b7bY=!,<-,5gQ6!\>V]AQ)^-GB`&C"Q_Tt5F^2b04
+H^dXF*&#KkrUdWPo<maGSjIV2C<A6%Y#l1eDB-11=5MCe/mQ;H^`5k(:$aRZO01rVm/hX\fYmD8f*Dj@
+M@i=qKg0)_(>4O.G!a5^E2>bO$]QiDc\2C6pdgJSEti>7EGJ-(WqC@QJiDd@NbtVSKCL`LBr7+Q7e(#%
+mfT9"IKcR0OZpuAD#Y..(\(Ya*`U>DQ9W"+'>%SR14Rbu3okUD1UoZtSAfKYf.IftGdnZe:>-X$>r]=V
+G-"HZ$ThCqcd_nA%GR%YOrq)?cB9h+fAIVB(kUJ,MLq_e+>iX=AF@bgIcap`;p5$2F+]o@5&$NLhu:.I
+2<(<.=8A484n4$P`>fQKA"tBRWbq]tDur6ODulS&f!BB<ZPSc21WW3#buP0(+9gU#Z,bH/SFFh24G?`5
+dOt)^QkNbrnhfi)90SEq&n`q[F;s7b!nXc$g#@aPQN3qbFeWl+!e=Zt`EF'M1(Bm%7HR8DbV4gg<TZ"1
+'QY,(Ci/)_["1h766XD'&ADrraHq=DH%OF$jsfp?jsbDCGm`Bc4nAYbhNNN6HSW+QHSVNRk<)"p3=KZH
+@5,_A#.Kh(3M>-L01N,n^DOYar*o;enr(m9jVO\;jVOZqjVO[\jEPdIJ4jW(1Ou\+n/U@A-XG!&R,,`f
+2W1\3O?*%L7g,>@T`!f8eMl#PBrq/m2=C@'l+[6\!TC@KAA[MWaFrFRGAPIZkLE8R^YhJV5O]iFIg_\G
+iO=6VK\u;C'>=b?fe%PhWL9K1abNRi>otj#+Z';<n+i%OCHc:GQ`]p-obCOtip.a^cf<UjY!FG)PRTLL
++"jYG+4_li`+jB7`VE<s"4AVj".>pGko/fq8/"=:R5@Xn#Q]c2U1bel!_n`0,C>VAYsbZH&PH@FJ$f#/
+0td=Kg'>AR25F7&YOo!<mI]$*rRjT2aR!Q]%Fi)aOtJSn?+UM2'/_)[:p`0l2g4bJBatZEkin+Um_FDG
+q;M6)[I9OhMIS=<lFaD[8\^p:_H)J0l0'R9&ZuGO_>dFFCb.uJQlb(Q4<5Au742_H^C[$nr;0XJ4b3Tg
+[;48b%t<.Gdkk?8S<Vi`;Poe&S_S0PK8iOs+@Vd0N*gN+=Plp50G^#i&Kg"CSmrqAknTi"D:*>YSUrma
+g=r&8<Pj&ND`$13'HCOjG_kBC'T)*ekL$!:6XuRpIX?)+^V5+S2cXNJB2ec80tlUL?G#q-m,hbS%$"V@
+p"JQ?igh1:m'G\$*s_?+,4a,JI@3GnX*+U"_Zr.(+*>1jkFAnU9>ZXAU\g3ArglE9&,uBU"0[/n3+!B'
+W[)MC"?*YACpqa=bi'p-F[R+Y&Jh':>DS9I;-PEL7\\5FC1<[_U$dQmm+CP5IQgiD>SQGm;oRG4k5.U/
+0D?:mI/C=6)f;AM:/Ut^\F':\li2"qaZh<t\"cP;!(<3T?G(>oo5D,:=$o+C?]KqCU?geEFjWG&V"PRp
+&9$#`m3'G9"&tosqAZg/@%\C2I=Z6p1:0Yl"+@)T)so1U@.,fWm/arHR8d:MJXMQVC8K5]=Jm_KIXjdB
+83[K$Qla[fJ0ST]0*O=-3O'X`"r\'Q&`>h%Y8I@YUo\`'^eHY_XTc5&7^\ik\4-8s4VZgTU@:6TdipBk
+I9NLLkSE!O'mP*-*Z_.eTF%\P6@O])QsDsH.[1,u#K\!Fdp$""eZ=M1#$Z$(ZO5i+BWp@?TH^qnB%D7/
+h#sWFD^p`NF:Yfb85;E[:q^5Oc;Ss`!7br9/Ss$74_<U(m:SQ#s-3*J/tuGNgXsJtKtqF1bXL_J,7?@i
+'BEj1Ilon*aW+o61dn83AfE7FcXE_u9)ggC=\p"(7u!%6Y7^]'TMPP?@sg_akMgC`EYfUsdCeJO#2(2D
+hQ;Br<4m&15-"(TYi[P!iN8/Ok#@VVm[UO"bZs@LQZDI&pjXBX+4dX`csI+Oe#J=,^c=d&n:"<LdpP?/
+57NYh_**Nu%GYGtPrZ:2pA@k$5I#/DZ?u(p.mKdY_mpagr"[aHSA,e@EE+%EftN*$X1t;gS`@?f>ZH3j
+dG.dX,4]pij-88?@>Es&4"OKq4Pb_H)9:8fq`(0Ie[V>KnkIYS^43jk=+).,aXsOX(4cqM6KaT:7HMS5
+5VqW;.CtD!kNC3=e+gS7q$"nkh'kl'g`'1Rk*3jhL&JE!FZWhT#NI(1R)?@JnB0)&ZC7m>1Wud3mOB<F
+<?1UlEeaPIK4D$_#X<cE%HM2B7nrD$SI/J?MJ9$\[%L0[X#c.:>Z)WgX'F4+E];'H25pPfMT*QlPBH<B
+2jEaIeeu%@g_SZi^/+`r>koI@22thoJ$T#-Ig:8=C[r)9A@gh.nJ+P$rtk!a7oB995m%NRR_tlI7kn;C
+L?s85#0T`VHa<+&O5K'$Eo(6pn-XUq4Y6-9Re-B$rhtBq!Tfi]GEbPb2*A\G,P/,p;d\J77]S.F#9J`B
+DopA\Sjp8ME&Btm*&[^S4k/0Z~>
+endstream
+endobj
+8 0 obj <<
+  /Length 377
+>>
+stream
+/CIDInit/ProcSet findresource begin
+12 dict begin
+begincmap
+/CIDSystemInfo<<
+/Registry (Adobe)
+/Ordering (UCS)
+/Supplement 0
+>> def
+/CMapName/Adobe-Identity-UCS def
+/CMapType 2 def
+1 begincodespacerange
+<00> <FF>
+endcodespacerange
+5 beginbfchar
+<01> <05DF>
+<02> <05D9>
+<03> <05DE>
+<04> <05E0>
+<05> <05D1>
+endbfchar
+endcmap
+CMapName currentdict /CMap defineresource pop
+end
+end
+endstream
+endobj
+xref
+0 9
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000131 00000 n 
+0000000309 00000 n 
+0000000471 00000 n 
+0000000678 00000 n 
+0000000905 00000 n 
+0000005238 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 9
+>>
+startxref
+5667
+%%EOF

corpus/fonts/malformed_font.pdf

@@ -0,0 +1,46 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+
+3 0 obj
+<< /Type /Page /Parent 2 0 R
+   /MediaBox [0 0 612 792]
+   /Contents 4 0 R
+   /Resources << /Font << /F1 5 0 R >> >>
+>>
+endobj
+
+4 0 obj
+<< /Length 64 >>
+stream
+BT
+/F1 12 Tf
+72 720 Td
+(Malformed Font) Tj
+-72 -720 Td
+ET
+endstream
+endobj
+
+5 0 obj
+<< /Type /Font /Subtype /UnknownType /BaseFont /BrokenFont >>
+endobj
+
+xref
+0 6
+0000000000 65535 f 
+0000000010 00000 n 
+0000000064 00000 n 
+0000000126 00000 n 
+0000000270 00000 n 
+0000000390 00000 n 
+trailer
+<< /Size 6 /Root 1 0 R >>
+startxref
+472
+%%EOF

corpus/fonts/multipage_font_mix.pdf

@@ -0,0 +1,46 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+
+3 0 obj
+<< /Type /Page /Parent 2 0 R
+   /MediaBox [0 0 612 792]
+   /Contents 4 0 R
+   /Resources << /Font << /F1 5 0 R >> >>
+>>
+endobj
+
+4 0 obj
+<< /Length 60 >>
+stream
+BT
+/F1 12 Tf
+72 720 Td
+(Page 1 Mix) Tj
+-72 -720 Td
+ET
+endstream
+endobj
+
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
+endobj
+
+xref
+0 6
+0000000000 65535 f 
+0000000010 00000 n 
+0000000064 00000 n 
+0000000126 00000 n 
+0000000270 00000 n 
+0000000386 00000 n 
+trailer
+<< /Size 6 /Root 1 0 R >>
+startxref
+488
+%%EOF

corpus/fonts/no_tounicode.pdf

@@ -0,0 +1,46 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+
+3 0 obj
+<< /Type /Page /Parent 2 0 R
+   /MediaBox [0 0 612 792]
+   /Contents 4 0 R
+   /Resources << /Font << /F1 5 0 R >> >>
+>>
+endobj
+
+4 0 obj
+<< /Length 66 >>
+stream
+BT
+/F1 12 Tf
+72 720 Td
+(No ToUnicode Map) Tj
+-72 -720 Td
+ET
+endstream
+endobj
+
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Symbol >>
+endobj
+
+xref
+0 6
+0000000000 65535 f 
+0000000010 00000 n 
+0000000064 00000 n 
+0000000126 00000 n 
+0000000270 00000 n 
+0000000392 00000 n 
+trailer
+<< /Size 6 /Root 1 0 R >>
+startxref
+464
+%%EOF

corpus/fonts/rotated_text.pdf

@@ -0,0 +1,71 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [ 0 0 200 200 ]
+  /Count 1
+  /Kids [ 3 0 R ]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 4 0 R
+    >>
+  >>
+  /Contents 5 0 R
+>>
+endobj
+4 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Times-Roman
+>>
+endobj
+5 0 obj <<
+  /Length 406
+>>
+stream
+BT
+0 0 Td
+/F1 12 Tf
+0.70710678118 -0.70710678118 0.70710678118 0.70710678118 100 100 Tm
+(Hello,) Tj
+0 0 Td
+/F1 12 Tf
+-0.70710678118 -0.70710678118 0.70710678118 -0.70710678118 100 100 Tm
+( world!\r
+) Tj
+0 0 Td
+/F1 12 Tf
+-0.70710678118 0.70710678118 -0.70710678118 -0.70710678118 100 100 Tm
+(Goodbye,) Tj
+0 0 Td
+/F1 12 Tf
+0.70710678118 0.70710678118 -0.70710678118 0.70710678118 100 100 Tm
+( world!) Tj
+ET
+endstream
+endobj
+xref
+0 6
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000161 00000 n 
+0000000287 00000 n 
+0000000365 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 6
+>>
+startxref
+823
+%%EOF

corpus/fonts/rotated_text_90.pdf

@@ -0,0 +1,71 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [0 0 200 200]
+  /Count 1
+  /Kids [3 0 R]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 4 0 R
+    >>
+  >>
+  /Contents 5 0 R
+>>
+endobj
+4 0 obj <<
+  /Type /Font
+  /Subtype /Type1
+  /BaseFont /Times-Roman
+>>
+endobj
+5 0 obj <<
+  /Length 210
+>>
+stream
+BT
+0 0 Td
+/F1 12 Tf
+1 0 0 1 100 100 Tm
+(Hello,) Tj
+0 0 Td
+/F1 12 Tf
+0 1 -1 0 100 100 Tm
+( world!\r
+) Tj
+0 0 Td
+/F1 12 Tf
+-1 0 0 -1 100 100 Tm
+(Goodbye,) Tj
+0 0 Td
+/F1 12 Tf
+0 -1 1 0 100 100 Tm
+( world!) Tj
+ET
+endstream
+endobj
+xref
+0 6
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000157 00000 n 
+0000000283 00000 n 
+0000000361 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 6
+>>
+startxref
+623
+%%EOF

corpus/fonts/subset_font.pdf

@@ -0,0 +1,46 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+
+3 0 obj
+<< /Type /Page /Parent 2 0 R
+   /MediaBox [0 0 612 792]
+   /Contents 4 0 R
+   /Resources << /Font << /F1 5 0 R >> >>
+>>
+endobj
+
+4 0 obj
+<< /Length 61 >>
+stream
+BT
+/F1 12 Tf
+72 720 Td
+(Subset Text) Tj
+-72 -720 Td
+ET
+endstream
+endobj
+
+5 0 obj
+<< /Type /Font /Subtype /TrueType /BaseFont /ABCDEF+Arial /FirstChar 32 /LastChar 126 >>
+endobj
+
+xref
+0 6
+0000000000 65535 f 
+0000000010 00000 n 
+0000000064 00000 n 
+0000000126 00000 n 
+0000000270 00000 n 
+0000000387 00000 n 
+trailer
+<< /Size 6 /Root 1 0 R >>
+startxref
+496
+%%EOF

corpus/fonts/utf-8.pdf

@@ -0,0 +1,76 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+
+3 0 obj
+<< /Type /Page /Parent 2 0 R
+   /MediaBox [0 0 612 792]
+   /Contents 4 0 R
+   /Resources <<
+     /Font <<
+       /F1 5 0 R
+       /F2 6 0 R
+     >>
+   >>
+>>
+endobj
+
+4 0 obj
+<< /Length 532 >>
+stream
+BT
+/F1 12 Tf
+72 720 Td
+(Hello World - UTF-8 Test Document) Tj
+-72 -720 Td
+72 700 Td
+(Standard Latin Text for Encoding Verification) Tj
+-72 -700 Td
+72 680 Td
+(Font Size Detection Sample: Small Text 12pt) Tj
+-72 -680 Td
+/F2 18 Tf
+72 650 Td
+(LARGE TEXT FOR SIZE 18PT DETECTION) Tj
+-72 -650 Td
+72 620 Td
+(More 18pt content: ABCDEFGHabcdefgh 0123456789) Tj
+-72 -620 Td
+/F1 12 Tf
+72 590 Td
+(Back to 12pt: The quick brown fox jumps over the lazy dog) Tj
+-72 -590 Td
+72 570 Td
+(Special chars: copyright section paragraph) Tj
+-72 -570 Td
+ET
+endstream
+endobj
+
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
+endobj
+
+6 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
+endobj
+
+xref
+0 7
+0000000000 65535 f 
+0000000009 00000 n 
+0000000059 00000 n 
+0000000117 00000 n 
+0000000290 00000 n 
+0000000873 00000 n 
+0000000971 00000 n 
+trailer
+<< /Size 7 /Root 1 0 R >>
+startxref
+1071
+%%EOF

corpus/fonts/vertical_identity_v.pdf

@@ -0,0 +1,49 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+
+3 0 obj
+<< /Type /Page /Parent 2 0 R
+   /MediaBox [0 0 612 792]
+   /Contents 4 0 R
+   /Resources <<
+     /Font <<
+       /F1 5 0 R
+     >>
+   >>
+>>
+endobj
+
+4 0 obj
+<< /Length 50 >>
+stream
+BT
+/F1 12 Tf
+72 720 Td
+(Vertical Text Test) Tj
+ET
+endstream
+endobj
+
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Identity-V /Encoding /WinAnsiEncoding >>
+endobj
+
+xref
+0 6
+0000000000 65535 f 
+0000000009 00000 n 
+0000000059 00000 n 
+0000000117 00000 n 
+0000000273 00000 n 
+0000000373 00000 n 
+trailer
+<< /Size 6 /Root 1 0 R >>
+startxref
+482
+%%EOF

corpus/fonts/vertical_text.pdf

@@ -0,0 +1,146 @@
+%PDF-1.7
+% ò¤ô
+1 0 obj <<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj <<
+  /Type /Pages
+  /MediaBox [0 0 200 200]
+  /Count 1
+  /Kids [3 0 R]
+>>
+endobj
+3 0 obj <<
+  /Type /Page
+  /Parent 2 0 R
+  /Resources <<
+    /Font <<
+      /F1 4 0 R
+    >>
+  >>
+  /Contents 8 0 R
+>>
+endobj
+4 0 obj <<
+  /Type /Font
+  /Subtype /Type0
+  /Encoding /UniGB-UTF16-V
+  /BaseFont /Test
+  /DescendantFonts [5 0 R]
+>>
+endobj
+5 0 obj <<
+  /Type /Font
+  /Subtype /CIDFontType2
+  /BaseFont /Test
+  /CIDSystemInfo <<
+    /Registry (Adobe)
+    /Ordering (GB1)
+    /Supplement 4
+  >>
+  /FontDescriptor 6 0 R
+  /DW 1000
+  /W [
+    1 [278]  %space
+    2 [278]  %!
+    41 [722] %H
+    56 [944] %W
+    69 [556] %d
+    70 [556] %e
+    77 [222] %l
+    80 [556] %o
+    83 [333] %r
+  ]
+  /DW2 [0 -1000]
+  /W2 [
+    1 [-723 139 623]    %space
+    2 [-918 139 818]    %!
+    41 [-918 361 818]   %H
+    56 [-918 472 818]   %W
+    69 [-918 278 818]   %d
+    70 [-723 278 623]   %e
+    77 [-918 111 818]   %l
+    80 [-723 278 623]   %o
+    83 [-723 166.5 623] %r
+  ]
+>>
+endobj
+6 0 obj <<
+  /Type /FontDescriptor
+  /Ascent 718
+  /CapHeight 500
+  /Descent -207
+  /Flags 32
+  /FontBBox [-166 -225 1000 931]
+  /FontFile2 7 0 R
+  /FontName /Test
+  /ItalicAngle 0
+  /StemV 80
+>>
+endobj
+7 0 obj <<
+  /Filter [/ASCII85Decode /FlateDecode]
+  /Length1 2456
+  /Length 1684
+>>
+stream
+GhU\K?YiY@)#qml\*a1%h:c`s)m$[;?!@Nf?/*!!0XbkOrTQd[?,p@=KMg=Kc?4![MV1<a'7u=>As+]B
+'495o@Y1$[N=Y"V^.<W,L*pI>$-"n:U;RjO^N4a)+9@>=#<aH4hsYsm^[pn0#QX)I2;k-WI'UY:m#)O\
+!g"OF:X1W4EpNXp_Z9<>jGhF!S/7idXo\$E.-Hf!Q!CSMQ;G;Ho<MnQm<%mSRl=19)qpfU-":lYlDY":
+q[SB7Kad?0hi#3nS@>;[nuK#ZMO7Yf&Pd!%@tr;<SS6@/J%QJ"c+uL"7+k>*$^oXFinbsi#J)/DIn)-'
+OtV7j?AtLdEaZ<5\9I,6ls*Q:mnh.)MB'-QHi$N,Ikc"ackl.lS#dYZ!t(lZ2Y9/.d?6]&WH`a%*Dndd
+5ZDU!@`<TjWWl6%Z2kjXXjaImr^Ma-T><lC1XMA)H$dDTq".B&f:Ti(-PZ;?<68ZDLN]"6A!sT%,6eE'
+oKP>!?&8GX>ODU?s6Y&&nC(/%5?!?Se<]Oma7S5D.CRLdfV,J@+O5cukCK0_/Q.0>kCcDhj_RFjOj_;E
+O(`NP]LJqP/$Go$)iReI-F9PG[a)Id^1$o!7U))'C_p:q:)>s-h6jks?0ZBU]Q.YOcbF+2pV*6U5[XQ,
+rW@d?_^!"FZ%J`;LW@:GO\,=t2<@P/RdFJL^;'$FBD.DEp(RB7lU7pu[rQhJ`V]-6]nJOS"lL1WRHsLe
+hR_DZX_;p-\@6oAhpP?iE7?hRHqVT![)6KsZj9"_?H1@.H*j]l_F7D08MnQa/(=+;_Nef!A>8-]iB"YP
+pmt$6^7N"dW8GP=MTAB=AL';>ZGf;7!TLS)!H.MpfumsW'pMgZ(M@ZVaQcJ<9o\,enaja=NQ/'Gcs%$/
+jG1bCp2\'8<9V+?+mGrSog0VnZuB-9'ck=UaE:OU^\E>JnWg-Y4+P?=lr)!4aqEqbZ:T.p7cO"8H:b#5
+ZtjGmlA-b#\eYg8f5$4THGmU;$[\b*p<S\VN)X%g?[l!h2TSn-+-m)tma:&BWoK!jc(07P;&hpi@)^q'
+ijFn<dVL/6fB9=p'%&=oY0I$Pg9nM(M3+$sq6%ThF=&Q"mqN=/6Y?Vs/IqN7lA19"W4chXZtjj1@Qn<\
+"jYCeT,S26":(iDrRbDhL:,rFQ#Wq09i&+Y]+j"RYmjNjZ.[F-E*0#tBPQfdLG#DalFOJ\P3maIAMi.C
+=?IFg#_aIb0k.Y3A`/iA$QB1\BBHMQ\[cf#9!QHAkgRDKWO(k8/AaqmfcFAZ!$UB&U7p#^A8D_CSdnfj
+%2!@<\-\16TC<T`cu#]7eB`[X51t6.)PK[?"Nhm_@L>0+UgafRn=tkd-ms-eej'u7=0WP:EV;`hloWXt
+=><Lb$rIa<N4Z*!%`VmD`B9OX/M(<>fSH20/b6eu!4$i^emK@2+:0L<j7d.5bX3mY0=Z#sTFGlH&uZs]
+ZkVAJ%-k]\Ci4*8+dpI\46KfOYM8AY'1.NZ+e":":csG8H*k[\)K"Nm5uo>IOA@7`#U@D`_ASTW0K)<3
+aYW/uJmllYfeL's)@7]##r<0tDW*$N97.7b["F=t;:[Fkc/aYdbK5FqQ/"^8YSA1`a5&AWZ]q)Kk>C:l
+_[=Oh?39tHUP>R@HqfVJQ<+j-)8Y_688bp6H/?'ZTdYB8rjP"t&b_Lc'A9H]a8~>
+endstream
+endobj
+8 0 obj <<
+  /Length 179
+>>
+stream
+BT
+/F1 12 Tf
+10 190 Td
+(\000H\000e\000l\000l\000o\000 ) Tj
+(\000W\000o\000r\000l\000d\000!) Tj
+ET
+BT
+/F1 12 Tf
+110 190 Td
+[(\000H) 100 (\000e) -100 (\000l\000l) 200 (\000o)] TJ
+ET
+endstream
+endobj
+xref
+0 9
+0000000000 65535 f 
+0000000015 00000 n 
+0000000068 00000 n 
+0000000157 00000 n 
+0000000283 00000 n 
+0000000408 00000 n 
+0000001041 00000 n 
+0000001244 00000 n 
+0000003038 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 9
+>>
+startxref
+3270
+%%EOF

corpus/fonts/with_tounicode.pdf

@@ -0,0 +1,62 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+
+3 0 obj
+<< /Type /Page /Parent 2 0 R
+   /MediaBox [0 0 612 792]
+   /Contents 4 0 R
+   /Resources <<
+     /Font <<
+       /F1 5 0 R
+       /F2 6 0 R
+     >>
+   >>
+>>
+endobj
+
+4 0 obj
+<< /Length 233 >>
+stream
+BT
+/F1 12 Tf
+72 720 Td
+(ToUnicode CMap Test Document) Tj
+-72 -720 Td
+72 700 Td
+(This PDF has a ToUnicode mapping for correct extraction.) Tj
+-72 -700 Td
+72 680 Td
+(Unicode text should be extractable from this PDF.) Tj
+-72 -680 Td
+ET
+endstream
+endobj
+
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
+endobj
+
+6 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Times-Roman /Encoding /WinAnsiEncoding >>
+endobj
+
+xref
+0 7
+0000000000 65535 f 
+0000000009 00000 n 
+0000000059 00000 n 
+0000000117 00000 n 
+0000000290 00000 n 
+0000000574 00000 n 
+0000000672 00000 n 
+trailer
+<< /Size 7 /Root 1 0 R >>
+startxref
+772
+%%EOF

docker-compose.yml

@@ -0,0 +1,52 @@
+services:
+  gateway:
+    build:
+      context: .
+      dockerfile: gateway/Dockerfile
+      network: host
+    image: pdf-engine-gateway:dev
+    container_name: pdf-engine-gateway
+    environment:
+      PDFENGINE_ENVIRONMENT: dev
+      PDFENGINE_ENGINE_AVAILABLE: "true"
+      # Keep browser access explicit; production deployments should replace
+      # this with the exact frontend origin(s), never a wildcard.
+      PDFENGINE_CORS_ALLOWED_ORIGINS: http://localhost:5173,http://127.0.0.1:5173,https://pdf-dev.maskantech.in
+      PDFENGINE_CORS_ALLOW_CREDENTIALS: "false"
+      PORT: 8765
+    ports:
+      - "8765:8765"
+    volumes:
+      - ./gateway:/home/app
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8765/health').read()" ]
+      interval: 20s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+    restart: unless-stopped
+
+  frontend:
+    build:
+      context: ./frontend
+      dockerfile: Dockerfile
+      target: development
+    image: pdf-engine-frontend:dev
+    container_name: pdf-engine-frontend
+    environment:
+      VITE_GATEWAY_URL: https://pdfapi-dev.maskantech.in
+    ports:
+      - "5173:5173"
+    volumes:
+      - ./frontend:/app
+      - /app/node_modules
+    depends_on:
+      - gateway
+    restart: unless-stopped
+
+  gotenberg:
+    image: gotenberg/gotenberg:8
+    container_name: pdf-engine-gotenberg
+    ports:
+      - "3000:3000"
+    restart: unless-stopped

docs/PDF_SECURITY_FEATURE_AUDIT.md

@@ -0,0 +1,298 @@
+# PDF Security / Unlock / Protect — Feature Audit Report
+
+**Date of Audit:** August 13, 2026  
+**Audited Subsystems:** Frontend (`pdf/frontend`), Gateway API (`pdf/gateway`), C++ PDF Engine (`pdf/engine`), Pybind11 Bindings (`pdf/bindings`), Security Tests (`pdf/tests/security`)  
+**Audit Purpose:** Evaluate the exact current state of PDF security, password authentication, permission enforcement, encryption detection, password removal, and PDF protection capabilities in the existing codebase.
+
+---
+
+## 1. Executive Summary
+
+The existing PDF editor codebase possesses a **robust reading, authenticating, and permission-enforcing pipeline** for password-protected PDFs, but **lacks all writing/creation capabilities for PDF encryption and protection**.
+
+Key findings:
+* **Reading & Authenticating Encrypted PDFs:** **FULLY IMPLEMENTED**. The system detects encrypted PDFs, prompts the user via a React modal, passes the password to PDFium in C++, validates credentials, returns helpful error messages on invalid passwords, and renders the document upon success.
+* **Granular Permission Surfacing & Enforcement:** **FULLY IMPLEMENTED**. PDFium extracts security revision numbers, encryption types (RC4, AES-128, AES-256), and permission flags. The Python FastAPI Gateway exposes these flags in `PermissionsResponse` and enforces HTTP `403 Forbidden` errors if a user attempts forbidden edits (annotations, text replacements, page rotations) or unauthorized exports.
+* **Password Removal / Unlocking:** **PARTIALLY IMPLEMENTED (Implicit)**. Opening a protected PDF with a valid password loads the decrypted document into memory. Exporting the document via `/documents/{id}/export` writes out an unencrypted PDF file. However, there is no explicit UI button or API endpoint dedicated to "Remove Password".
+* **Protecting / Encrypting PDFs:** **NOT IMPLEMENTED / MISSING**. There is no functionality in the C++ engine (PDFium/QPDF), Pybind11 bindings, Gateway API, or Frontend UI to password-protect an unencrypted PDF, set user/owner passwords, or configure output permissions.
+
+---
+
+## 2. User-Facing Capability Summary
+
+### CURRENTLY AVAILABLE
+* **✓ Open Password-Protected PDFs:** Prompts for credentials when an encrypted PDF is uploaded.
+* **✓ Password Validation & Error Feedback:** Rejects incorrect passwords with clear inline UI feedback and allows unlimited retries.
+* **✓ Post-Authentication Rendering & Extraction:** Full page rendering, OCR, text extraction, font listing, layout analysis, and display list extraction work seamlessly after authentication.
+* **✓ Encryption & Security Inspection:** Detects and displays encryption standards (RC4-40, RC4-128, AES-128, AES-256) and security revision level (2 through 6) in the Inspector panel.
+* **✓ Permission Enforcement:** Gateway blocks unauthorized edits, annotations, page reordering, and exports with HTTP `403 Forbidden` responses if disallowed by the PDF's security settings.
+* **✓ Unprotected Export:** Exporting an authenticated PDF generates an unencrypted PDF that can subsequently be opened without a password.
+
+### NOT CURRENTLY AVAILABLE
+* **✗ Explicit Password Removal UI/API:** No button or endpoint explicitly labeled "Unlock PDF" or "Remove Security".
+* **✗ Password-Protect PDF / Lock PDF:** Cannot apply passwords to an unencrypted PDF.
+* **✗ Configure Output Permissions:** Cannot set or modify permission flags for printing, copying, editing, or annotating.
+* **✗ Separate Owner Password Prompting:** Prompts only with a generic "Document password" input; does not request owner password specifically when attempting restricted operations.
+* **✗ Re-encrypting Edited PDFs:** Saved/exported PDFs are saved without encryption.
+* **✗ Attempt Rate Limiting:** No rate limiting on password validation attempts at the API level.
+
+---
+
+## 3. Protected PDF Open Flow
+
+The upload and document initialization flow is traced across the full stack:
+
+```
+User Selects Encrypted PDF
+           │
+           ▼
+[Frontend] gatewayService.uploadDocument(file, password="")
+           │
+           ▼ (POST /documents)
+[Gateway API] upload_document() in crud.py
+           │
+           ▼
+[Pybind11] PdfDocument.load_from_memory(bytes_data, "")
+           │
+           ▼
+[C++ Engine] PdfDocument::loadFromMemory() -> FPDF_LoadMemDocument()
+           │
+           ▼ (PDFium returns FPDF_ERR_PASSWORD)
+[C++ Engine] mapPdfiumError() returns EngineError::PasswordRequired
+           │
+           ▼
+[Pybind11] Throws ValueError("Password required to open this PDF")
+           │
+           ▼
+[Gateway API] Catches ValueError -> Raises HTTP 401 ("Password required")
+           │
+           ▼
+[Frontend] gatewayService catches 401 -> Throws PasswordError
+           │
+           ▼
+[Frontend] App.tsx sets passwordPrompt state -> PasswordModal renders
+```
+
+### Exact Code Implementation Points
+* **Encryption Detection & Password Loading (C++ Engine):**
+  * File: [`pdf/engine/src/parser/pdfium_document.cpp`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_document.cpp#L42-L48)
+  * Function: `pdfengine::PdfDocument::loadFromMemory(const std::vector<uint8_t>& data, const std::string& password)`
+  * C++ API: `FPDF_LoadMemDocument(buffer_copy.data(), static_cast<int>(buffer_copy.size()), password.empty() ? nullptr : password.c_str())`
+* **Error Mapping (C++ Engine):**
+  * File: [`pdf/engine/src/parser/pdfium_internal.cpp`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_internal.cpp#L271-L285)
+  * Function: `pdfengine::parser::mapPdfiumError(unsigned long err, bool passwordProvided)`
+  * Logic: Maps `FPDF_ERR_PASSWORD` to `EngineError::PasswordRequired` (if `password` is empty) or `EngineError::InvalidPassword` (if `password` was provided).
+* **Pybind11 Translation:**
+  * File: [`pdf/bindings/python/pdfengine_py.cpp`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/bindings/python/pdfengine_py.cpp#L18-L21)
+  * Function: `throw_on_error(pdfengine::EngineError err)`
+  * Logic: Maps `PasswordRequired` -> `PyExc_ValueError("Password required to open this PDF")` and `InvalidPassword` -> `PyExc_ValueError("Invalid password provided for this PDF")`.
+* **Gateway Endpoint:**
+  * File: [`pdf/gateway/app/routers/documents/crud.py`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/crud.py#L142-L189)
+  * Endpoint: `POST /documents?password={password}` (`upload_document`)
+  * Logic: Catches `ValueError` from pybind11 and raises `HTTPException(status_code=401, detail="Password required")` or `HTTPException(status_code=401, detail="Invalid password")`.
+* **Frontend Password Dialog & Resubmission:**
+  * Files: [`pdf/frontend/src/lib/gatewayService.ts`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/frontend/src/lib/gatewayService.ts#L471-L486), [`pdf/frontend/src/App.tsx`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/frontend/src/App.tsx#L638-L652), [`pdf/frontend/src/components/PasswordModal.tsx`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/frontend/src/components/PasswordModal.tsx#L15-L95)
+  * Logic: `gatewayService` throws `PasswordError`. `App.tsx` catches `PasswordError` and opens `PasswordModal`. User enters password, triggering resubmission to `uploadDocument(file, password)`.
+
+---
+
+## 4. Password Authentication & Validation Flow
+
+| Stage | Implementation Status | Evidence / Location |
+|---|---|---|
+| **Password Reaches Backend** | **IMPLEMENTED** | `uploadDocument(file, password)` in `gatewayService.ts:L476` sends `POST /documents?password=...`. |
+| **Backend Passes Password to C++** | **IMPLEMENTED** | `crud.py:L173` calls `doc = pdfengine.PdfDocument.load_from_memory(bytes_data, password)`. |
+| **C++ Engine Validation** | **IMPLEMENTED** | `pdfium_document.cpp:L42` executes `FPDF_LoadMemDocument(..., password.c_str())`. |
+| **Incorrect Password Rejected** | **IMPLEMENTED** | `pdfium_internal.cpp:L280` returns `EngineError::InvalidPassword` -> HTTP 401 `"Invalid password"` -> `App.tsx` shows red error message in modal. |
+| **Correct Password Accepted** | **IMPLEMENTED** | `FPDF_LoadMemDocument` returns document pointer -> Gateway stores document info and returns HTTP 201 response. |
+| **Password Retry Loop** | **IMPLEMENTED** | `App.tsx:L650` retains modal open on failure with updated error message, allowing infinite retry attempts. |
+| **Rate Limiting / Attempt Limit** | **MISSING** | Neither Gateway nor C++ Engine tracks failed attempts or implements delays/lockouts. |
+
+---
+
+## 5. Rendering & Feature Support After Authentication
+
+Once authenticated, all engine features operate on the unlocked in-memory PDF handle:
+
+| Feature | Status | Evidence / Implementation Location |
+|---|---|---|
+| **Page Count** | **IMPLEMENTED** | [`pdfium_document.cpp:L101`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_document.cpp#L101) (`FPDF_GetPageCount`) |
+| **Document Metadata** | **IMPLEMENTED** | [`pdfium_document.cpp:L109`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_document.cpp#L109) (`FPDF_GetMetaText`) |
+| **Font Inventory** | **IMPLEMENTED** | [`pdfium_document.cpp:L280`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_document.cpp#L280), [`fonts.py:L16`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/fonts.py#L16) |
+| **Page Text Extraction** | **IMPLEMENTED** | [`pdfium_page.cpp:L100`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_page.cpp#L100), [`render.py:L172`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/render.py#L172) |
+| **Page Image Rendering** | **IMPLEMENTED** | [`pdfium_page.cpp:L50`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_page.cpp#L50), [`render.py:L19`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/render.py#L19) |
+| **Display List Extraction** | **IMPLEMENTED** | [`pdfium_page.cpp:L300`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_page.cpp#L300), [`content.py:L133`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/content.py#L133) |
+| **OCR Support** | **IMPLEMENTED** | [`ocr.py:L63`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/ocr.py#L63) (Runs Tesseract on rendered page image) |
+| **Layout Model Extraction** | **IMPLEMENTED** | [`layout.py:L120`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/layout.py#L120) (`extract_document_model()`) |
+| **Editing Operations** | **PARTIAL** | [`edits.py:L395`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/edits.py#L395) checks `permissions`. Allowed edits modify stream in-memory. |
+| **Exporting Document** | **IMPLEMENTED** | [`export.py:L17`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/export.py#L17) checks `canCopy` and calls `save_full_for_export()`. |
+
+---
+
+## 6. Unlock / Decrypt Capability
+
+| Action | Status | Description |
+|---|---|---|
+| **Open Protected PDF** | **IMPLEMENTED** | Via `upload_document` with `password`. |
+| **Authenticate** | **IMPLEMENTED** | Validated via `FPDF_LoadMemDocument`. |
+| **In-Memory Decryption** | **IMPLEMENTED** | PDFium decrypts document structure in RAM for standard operations. |
+| **Save/Export Unprotected Copy** | **IMPLEMENTED (Implicit)** | `doc.save_full_for_export()` calls PDFium's `FPDF_SaveWithVersion(doc_, &writer, 0, 14)`. Because PDFium does not attach an encryption handler during save, the output PDF is **unencrypted**. |
+| **Reopen Exported Copy Without Password** | **IMPLEMENTED** | The exported PDF contains no `/Encrypt` dictionary; reopening requires no password. |
+| **Explicit "Remove Password" Endpoint / UI** | **MISSING** | No dedicated route (e.g. `POST /documents/{id}/unlock`) or UI action exists. |
+
+---
+
+## 7. Protect / Encrypt Capability
+
+A comprehensive search across C++ Engine (`pdf/engine`), Pybind11 (`pdf/bindings`), Gateway (`pdf/gateway`), and Frontend (`pdf/frontend`) reveals **NO code for creating encrypted PDFs**:
+
+* **QPDF Encryption Writer:** Not implemented. [`qpdf_writer.cpp`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/qpdf/qpdf_writer.cpp) contains stream replacement and appearance helpers, but no `QPDFWriter::setEncryption` calls.
+* **PDFium Encryption Output:** PDFium's public writing API lacks native PDF encryption creation functions.
+* **Python Encryption Libraries:** `pikepdf` is imported **only** in [`pdf/tests/security/test_permissions.py`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/tests/security/test_permissions.py#L30) to generate test fixtures. It is not present in Gateway production services.
+* **Frontend Encryption Controls:** No modal, form, or state exists for password-protecting documents.
+
+---
+
+## 8. Password Types & Permission Management
+
+### Password Types
+* **User Password:** **IMPLEMENTED**. Used for opening documents.
+* **Owner Password:** **PARTIALLY IMPLEMENTED**. When opened with an Owner password, PDFium elevates document permissions. The C++ engine detects this by comparing user vs doc permissions (`perms.ownerUnlocked = (p != up)` in [`pdfium_document.cpp:L168`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_document.cpp#L168)) and displays an `"Owner"` badge in `InspectorPanel.tsx:L569`. However, there is no UI workflow to enter an owner password separately to unlock restricted actions.
+
+### PDF Permissions Matrix
+
+| Permission | Existing Support | Where Implemented |
+|---|---|---|
+| **Print** (`canPrint`) | Surfaced & Displayed | `pdfium_document.cpp:L171`, `store.py:L18`, `InspectorPanel.tsx:L573` |
+| **Modify** (`canModify`) | Surfaced & Enforced | `pdfium_document.cpp:L172`, `edits.py:L375`, `edits.py:L395` (HTTP 403) |
+| **Copy** (`canCopy`) | Surfaced & Enforced | `pdfium_document.cpp:L173`, `export.py:L30`, `export.py:L65` (HTTP 403 on Export) |
+| **Extract** (`canCopy`) | Surfaced & Enforced | Same bit as Copy (`0x10`) in PDFium spec |
+| **Annotate** (`canAnnotate`) | Surfaced & Enforced | `pdfium_document.cpp:L174`, `edits.py:L370-374`, `edits.py:L395` (HTTP 403) |
+| **Fill Forms** (`canFillForms`) | Surfaced & Enforced | `pdfium_document.cpp:L175`, `edits.py:L376`, `edits.py:L395` |
+| **Accessibility** (`canExtractForAccessibility`) | Surfaced & Displayed | `pdfium_document.cpp:L176`, `store.py:L24` |
+| **Document Assembly** (`canAssemble`) | Surfaced & Enforced | `pdfium_document.cpp:L177`, `edits.py:L377` (HTTP 403 on rotation/deletion) |
+| **High-Quality Print** (`canPrintHighRes`) | Surfaced & Displayed | `pdfium_document.cpp:L178`, `store.py:L19` |
+| **Permission Configuration (Writing)** | **MISSING** | No engine or gateway code exists to modify permission flags. |
+
+---
+
+## 9. Encryption Algorithm Surfacing
+
+The C++ engine inspects the PDF security handler revision via `FPDF_GetSecurityHandlerRevision(doc_)`:
+
+```cpp
+// pdfium_document.cpp (lines 147-164)
+switch (rev) {
+    case 2: perms.encryption = "RC4-40"; break;
+    case 3: perms.encryption = "RC4-128"; break;
+    case 4: perms.encryption = "AES-128"; break;
+    case 5:
+    case 6: perms.encryption = "AES-256"; break;
+    default: perms.encryption = "Unknown"; break;
+}
+```
+
+* **Surfacing:** Mapped to `PermissionsResponse.encryption` ([`document.py:L12`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/schemas/document.py#L12)) and rendered as a badge in the Inspector panel ([`InspectorPanel.tsx:L567`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/frontend/src/components/InspectorPanel.tsx#L567)).
+* **Creation:** Encryption creation was not found in the codebase.
+
+---
+
+## 10. Frontend UI State
+
+| Capability | Status | Front-End Evidence |
+|---|---|---|
+| **A. Unlock existing protected PDF** | **IMPLEMENTED** | `PasswordModal.tsx` renders when `passwordPrompt` state is non-null. |
+| **B. Remove password / security** | **MISSING** | No UI button or option. |
+| **C. Protect an unprotected PDF** | **MISSING** | No UI button or option. |
+| **D. Set a password** | **MISSING** | No input fields for protecting PDFs. |
+| **E. Configure permissions** | **MISSING** | No permissions toggle matrix in settings or export dialog. |
+| **F. Export document** | **IMPLEMENTED** | TopBar export button triggers file download. |
+
+---
+
+## 11. Gateway / API Endpoints Audit
+
+| Endpoint | Method | Purpose | Implemented Behavior | C++ Call |
+|---|---|---|---|---|
+| `/documents` | `POST` | Upload & open PDF | Accepts `password` query param. Passes password to engine. Returns HTTP 401 on missing/wrong password, HTTP 201 with permissions on success. | `PdfDocument::loadFromMemory` |
+| `/documents/{id}/export` | `GET` | Export PDF | Verifies `permissions.canCopy`. Returns HTTP 403 if forbidden. Calls `save_full_for_export()`. | `PdfiumDocument::saveFullForExport` |
+| `/documents/{id}/export-remote` | `POST` | Export to remote URL | Verifies `permissions.canCopy`. Streams file to target URL. | `PdfiumDocument::saveFullForExport` |
+| `/edits` | `POST` | Apply PDF edits | Maps operation types to permissions (`canAnnotate`, `canModify`, `canFillForms`, `canAssemble`). Returns HTTP 403 if restricted. | Engine edit APIs |
+| `/documents/{id}/unlock` | N/A | Dedicated unlock | **MISSING** | N/A |
+| `/documents/{id}/protect` | N/A | Protect document | **MISSING** | N/A |
+
+---
+
+## 12. End-to-End Export & Reopen Verification Scenarios
+
+### SCENARIO A: Unprotected PDF -> Protect with Password -> Export -> Reopen -> Prompted for Password
+* **Status:** **NOT WORKING / IMPOSSIBLE TODAY**
+* **Reason:** "Protect with password" is not implemented anywhere in the backend or engine.
+
+### SCENARIO B: Protected PDF -> Enter Password -> Opened -> Export -> Reopen Exported PDF -> No Password Required
+* **Status:** **FULLY WORKING TODAY (Implicitly)**
+* **Reason:** PDFium loads the decrypted PDF structure into memory. Exporting via `GET /documents/{id}/export` writes the file without encryption. Reopening the exported file requires no password.
+
+### SCENARIO C: Protected PDF -> Enter Wrong Password -> Rejected -> Enter Correct Password -> Opened
+* **Status:** **FULLY WORKING TODAY**
+* **Reason:** Invalid password returns HTTP 401 with `"Invalid password"`. The frontend displays `"Incorrect password — please try again."` and keeps the modal open. Re-submitting with the correct password opens the document cleanly.
+
+---
+
+## 13. Security Observations & Risks
+
+* **CONFIRMED FROM CODE — Password Passed in Query String:**
+  In [`gatewayService.ts:L476`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/frontend/src/lib/gatewayService.ts#L476), the upload URL is constructed as `${this.baseUrl}/documents?password=${encodeURIComponent(password)}`. Transmitting passwords in GET/POST URL query parameters poses a security risk because query parameters may be recorded in server access logs or proxy logs.
+* **CONFIRMED FROM CODE — Absence of API Rate Limiting:**
+  In [`crud.py:L142-L190`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/crud.py#L142-L189), there is no rate-limiting or lock-out mechanism for password validation requests, allowing automated brute-force attempts.
+* **CONFIRMED FROM CODE — Implicit Decryption on Export:**
+  In [`export.py:L38`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/export.py#L38), exported PDFs are saved unencrypted. Users who upload a password-protected PDF and subsequently export it will receive an unencrypted file without explicit warning that password protection has been stripped.
+* **CONFIRMED FROM CODE — Plaintext Password In-Memory Only:**
+  Passwords are passed directly to `load_from_memory` and are **not** persisted in `document_store` or written to disk.
+
+---
+
+## 14. Complete Feature Matrix
+
+| Feature | Status | Existing Location | Evidence |
+|---|---|---|---|
+| **Detect encrypted PDF** | `IMPLEMENTED` | `pdfium_document.cpp:L142` | `perms.isEncrypted = (rev != -1)` |
+| **Password popup** | `IMPLEMENTED` | `PasswordModal.tsx:L15` | `<PasswordModal state={passwordPrompt} ... />` |
+| **Validate password** | `IMPLEMENTED` | `pdfium_document.cpp:L42`, `crud.py:L173` | `load_from_memory(bytes_data, password)` |
+| **Wrong password handling** | `IMPLEMENTED` | `crud.py:L183`, `App.tsx:L650` | HTTP 401 `"Invalid password"` -> UI error |
+| **Correct password handling** | `IMPLEMENTED` | `crud.py:L174`, `App.tsx:L646` | Returns `DocumentInfoResponse` -> Document opens |
+| **Render protected PDF** | `IMPLEMENTED` | `pdfium_page.cpp:L50`, `render.py:L19` | Renders tiles/pages post-authentication |
+| **OCR protected PDF** | `IMPLEMENTED` | `ocr.py:L63` | Executes Tesseract on authenticated doc pages |
+| **Edit protected PDF** | `PARTIAL` | `edits.py:L395` | Enforces permissions, but doesn't re-encrypt |
+| **Export protected PDF** | `PARTIAL` | `export.py:L17` | Enforces `canCopy`, but exports UNENCRYPTED |
+| **Remove password** | `PARTIAL` | `export.py:L38` | Exporting strips password (implicit, no explicit API) |
+| **Export unprotected PDF** | `IMPLEMENTED` | `export.py:L38` | `save_full_for_export()` outputs unencrypted PDF |
+| **Reopen unprotected PDF** | `IMPLEMENTED` | `crud.py:L141` | Exported file reopens without password |
+| **Protect PDF** | `MISSING` | N/A | No code exists to protect/encrypt PDF |
+| **Set user password** | `MISSING` | N/A | No functionality to set user password |
+| **Set owner password** | `MISSING` | N/A | No functionality to set owner password |
+| **AES encryption (detection)** | `IMPLEMENTED` | `pdfium_document.cpp:L155-160` | Revision 4/5/6 mapped to `"AES-128"` / `"AES-256"` |
+| **AES-256 (detection)** | `IMPLEMENTED` | `pdfium_document.cpp:L158` | Revision 5/6 mapped to `"AES-256"` |
+| **RC4 (detection)** | `IMPLEMENTED` | `pdfium_document.cpp:L148-153` | Revision 2/3 mapped to `"RC4-40"` / `"RC4-128"` |
+| **Print permission** | `IMPLEMENTED` | `pdfium_document.cpp:L171`, `store.py:L18` | Surfaced in permissions API |
+| **Copy permission** | `IMPLEMENTED` | `pdfium_document.cpp:L173`, `export.py:L30` | Enforced on Export (returns HTTP 403) |
+| **Modify permission** | `IMPLEMENTED` | `pdfium_document.cpp:L172`, `edits.py:L375` | Enforced on edits (returns HTTP 403) |
+| **Annotation permission** | `IMPLEMENTED` | `pdfium_document.cpp:L174`, `edits.py:L370` | Enforced on annotations (returns HTTP 403) |
+| **Form permission** | `IMPLEMENTED` | `pdfium_document.cpp:L175`, `edits.py:L376` | Enforced on form fills (allows if permitted) |
+| **Extraction permission** | `IMPLEMENTED` | `pdfium_document.cpp:L173`, `store.py:L21` | Surfaced as `canCopy` |
+| **Document assembly** | `IMPLEMENTED` | `pdfium_document.cpp:L177`, `edits.py:L377` | Enforced on page rotate/delete ops |
+| **Accessibility permission** | `IMPLEMENTED` | `pdfium_document.cpp:L176`, `store.py:L24` | Surfaced as `canExtractForAccessibility` |
+| **High-quality printing** | `IMPLEMENTED` | `pdfium_document.cpp:L178`, `store.py:L19` | Surfaced as `canPrintHighRes` |
+| **Security UI** | `PARTIAL` | `PasswordModal.tsx`, `InspectorPanel.tsx` | Password prompt modal + Inspector security badge exist |
+| **Security API** | `PARTIAL` | `crud.py`, `export.py`, `edits.py` | Upload & export handle passwords & perms |
+| **C++ security implementation** | `PARTIAL` | `pdfium_document.cpp` | Document load & permission inspection implemented |
+| **Pybind security bindings** | `PARTIAL` | `pdfengine_py.cpp:L252` | `DocumentPermissions` & `load_from_memory` bound |
+
+---
+
+## 15. Production Readiness Summary
+
+* **Opening & Viewing Protected PDFs:** **PRODUCTION READY**. Robust, fully tested with unit tests ([`pdf/engine/tests/document_load_test.cpp`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/tests/document_load_test.cpp#L27-L70)) and security integration tests ([`pdf/tests/security/test_permissions.py`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/tests/security/test_permissions.py)).
+* **Permission Enforcement:** **PRODUCTION READY**. Gateway correctly returns HTTP 403 Forbidden for restricted edits and exports.
+* **Password Removal / Unlocking:** **NEEDS FEATURIZATION**. Works implicitly when exporting, but lacks dedicated API routes and UI buttons for explicit unlock workflows.
+* **Protecting / Encrypting PDFs:** **NOT PRODUCTION READY (0% IMPLEMENTED)**. Creation of password-protected PDFs or custom permission dictionaries requires adding QPDF or pikepdf encryption writers to the engine/gateway layer.

docs/phase0.md

@@ -1,348 +0,0 @@
-# Phase 0 — Infrastructure
-
-> Goal of Phase 0: **make the entire stack compile and run on Linux, macOS, and
-> Windows.** No editing, no rendering features. Success = the engine library
-> builds clean on all three platforms and the smoke test passes. Phase 1 does
-> not begin until Gate G0 and G0b are reached.
-
-## Phase 0 task board
-
-| # | Task | Owner | This session |
-|---|------|-------|--------------|
-| 1 | CMake root + vcpkg + CI/CD pipeline       | Dev 1 | **Done — scaffolded** |
-| 2 | PDFium build (depot_tools + GN + Ninja)   | Dev 1 | Build scripts staged; not yet run |
-| 3 | Skia build integration                    | Dev 2 | Not started |
-| 4 | FreeType + HarfBuzz vcpkg integration      | Dev 3 | In manifest; wrappers not started |
-| 5 | FastAPI service scaffolding                | Dev 1 | Placeholder dir only |
-| 6 | React + TypeScript frontend scaffolding    | Dev 2 | Placeholder dir only |
-| 7 | WASM hello-world build (Emscripten)        | Dev 1 | Toolchain hook + preset stubbed |
-| 8 | Frozen interface contracts (Gate G0b)      | All   | Placeholder header; **not designed** |
-
-This session delivered **Task 1 in full** plus the repository structure for
-everything else. Scope was deliberately limited to the build pipeline — see
-"What is intentionally not done" below.
-
-## What was built this session
-
-```
-Code/
-├── CMakeLists.txt              root build; refuses to configure without a pinned baseline
-├── CMakePresets.json           debug/release/asan per platform + a wasm stub
-├── vcpkg.json                  dependency manifest (freetype, harfbuzz, spdlog, gtest)
-├── .clang-format .clang-tidy    style + naming rules from blueprint §16.1
-├── .gitignore .gitattributes .editorconfig
-├── cmake/
-│   ├── pdfium.cmake            turns the PDFium install tree into pdfium::pdfium
-│   ├── CompilerWarnings.cmake  high warning levels per compiler
-│   ├── Sanitizers.cmake        ASan/UBSan wiring
-│   └── toolchains/wasm.cmake   Emscripten hook (stub)
-├── engine/
-│   ├── CMakeLists.txt
-│   ├── include/pdfengine/      public headers (version, umbrella, pdf_document placeholder)
-│   ├── src/core/               engine_info.cpp — version/build introspection
-│   ├── src/parser/             pdfium_loader — the ONLY FPDF_-allowed dir (Rule R2)
-│   └── tests/                  gtest smoke test backing Gate G0
-├── third_party/pdfium/         from-source build scripts + pinned-ref file + args.gn
-├── scripts/
-│   ├── bootstrap.{sh,ps1}      installs vcpkg, pins the dependency baseline
-│   └── check_pdfium_boundary.{sh,ps1}   Rule R2 enforcement
-├── .github/workflows/ci.yml    Linux/macOS/Windows build matrix + lint jobs
-├── bindings/ gateway/ frontend/ wasm/ corpus/   placeholder dirs with READMEs
-└── docs/phase0.md              this file
-```
-
-## How to build (developer onboarding)
-
-### Prerequisites
-
-| Tool | Version | Notes |
-|------|---------|-------|
-| CMake | >= 3.25 | presets v6 |
-| Ninja | any recent | the only generator used |
-| C++ compiler | MSVC 19.36+ / GCC 13+ / Clang 16+ | needs C++23 |
-| vcpkg | — | `scripts/bootstrap` installs it if `VCPKG_ROOT` is unset |
-| Git | any recent | |
-| clang-format | 22.1.5 | not natively packaged on Windows — `pip install clang-format==22.1.5` (CI is pinned to this exact version) |
-
-> **Windows:** either Visual Studio 2022/2026 (with the *"Desktop development
-> with C++"* workload) or **Build Tools 2026** (no IDE — installer product
-> `Microsoft.VisualStudio.Product.BuildTools`) is supported. `cl.exe` is not on
-> `PATH` by default — run builds from a *Developer PowerShell* or import
-> `VC\Auxiliary\Build\vcvars64.bat` first. Build Tools is not a default
-> `vswhere` product, so detection scripts need `vswhere -products *` (the
-> PDFium build script already does this). CI uses `ilammy/msvc-dev-cmd`.
->
-> `VCPKG_ROOT` set via `setx` (or the bootstrap script's persistent install)
-> does **not** propagate into already-open shells — set `$env:VCPKG_ROOT`
-> explicitly in that shell, or open a new terminal. PowerShell 5.1 is fine;
-> `pwsh` (7+) is not required by anything in this repo.
-
-### Steps
-
-```sh
-# 1. One-time setup — checks tools, installs vcpkg, pins the dependency baseline.
-pwsh scripts/bootstrap.ps1        # Windows
-./scripts/bootstrap.sh            # Linux / macOS
-
-# 2. Configure + build + test.
-cmake --preset windows-debug      # linux-debug | macos-debug
-cmake --build --preset windows-debug
-ctest --preset windows-debug
-```
-
-The first configure compiles the vcpkg dependencies (freetype, harfbuzz,
-spdlog, gtest) — slow once, cached after.
-
-> **Verifying everything at once:** use `scripts/test_phase0.{ps1,sh}` to run
-> the R2 boundary check, engine, gateway, and WASM smoke test as a single
-> command and get one pass/fail summary. See
-> ["Verifying your setup"](#verifying-your-setup--scriptstest_phase0ps1sh) below.
-
-### Building with PDFium
-
-PDFium is built separately from source (Task 2):
-
-```sh
-# Pin the revision first — edit third_party/pdfium/pdfium.pinned (see its README).
-pwsh third_party/pdfium/build_pdfium.ps1     # or .sh
-```
-
-Until then the engine builds with PDFium code paths `#ifdef`-ed out, which is
-the correct Phase 0 default — it keeps the pipeline green while Task 2 runs.
-
-On Linux/macOS, enabling PDFium is a single flag added to a debug build:
-
-```sh
-cmake --preset linux-debug -DPDFENGINE_WITH_PDFIUM=ON
-```
-
-On Windows there is more to it — see the next section.
-
-### Windows + PDFium
-
-PDFium's static-lib GN build forces the static CRT (`/MT`, `is_debug=false`)
-and offers no knob for "static lib + dynamic CRT". The engine, vcpkg deps,
-and PDFium must therefore all use the same static CRT, or the link dies with
-`LNK2038: 'RuntimeLibrary' mismatch`. The repo is wired for this ("Option A,
-all static CRT, release-flavored"):
-
-- The hidden `windows-base` preset in [CMakePresets.json](../CMakePresets.json)
-  sets `VCPKG_TARGET_TRIPLET=x64-windows-static` (vcpkg deps as static lib +
-  static CRT — first configure rebuilds them, ~7 min one-time) and
-  `CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded$<$<CONFIG:Debug>:Debug>`.
-- PDFium's own [args.gn](../third_party/pdfium/args.gn) keeps `is_debug=false`
-  (i.e. `/MT`).
-- The PDFium-linked engine build must be **RelWithDebInfo, not Debug** — a
-  Debug engine is `/MTd` and still mismatches PDFium's `/MT`. The plain
-  `windows-debug -DPDFENGINE_WITH_PDFIUM=ON` recipe will not link; use the
-  `win-local-pdfium` user preset below.
-
-#### `win-local-pdfium` user preset
-
-`CMakeUserPresets.json` is git-ignored (build dirs are per-developer and live
-outside OneDrive). Drop this preset in at `Code/CMakeUserPresets.json`, with
-your own username in `binaryDir`:
-
-```json
-{
-  "version": 6,
-  "configurePresets": [
-    {
-      "name": "win-local-pdfium",
-      "inherits": "windows-release",
-      "binaryDir": "C:/Users/<you>/pdfeng-build/win-local-pdfium",
-      "cacheVariables": { "PDFENGINE_WITH_PDFIUM": "ON" }
-    }
-  ],
-  "buildPresets": [
-    { "name": "win-local-pdfium", "configurePreset": "win-local-pdfium" }
-  ],
-  "testPresets": [
-    { "name": "win-local-pdfium", "inherits": "common", "configurePreset": "win-local-pdfium" }
-  ]
-}
-```
-
-Then, from a shell with `vcvars64.bat` imported:
-
-```powershell
-cmake --preset win-local-pdfium
-cmake --build --preset win-local-pdfium
-ctest --preset win-local-pdfium
-```
-
-Success looks like `pdfengine_smoke.exe` linking cleanly and logging
-`pdfium=on`. The build dir is intentionally outside OneDrive and on a
-space-free path — see the OneDrive section below for why.
-
-#### Gotcha: depot_tools shadows `ninja`
-
-The PDFium build adds `depot_tools` to `PATH` and the depot_tools `ninja` /
-`ninja.bat` are not real Ninja — they fail with:
-
-```
-Running ninja --version failed with unknown error
-... CMAKE_CXX_COMPILER not set, after EnableLanguage
-```
-
-If `depot_tools` ended up on your persistent `PATH`, CMake will pick its
-broken `ninja` for the engine build. Two fixes:
-
-- **Short-term:** point CMake at the real Ninja explicitly, e.g.
-  `cmake --preset win-local-pdfium -D CMAKE_MAKE_PROGRAM=C:/path/to/real/ninja.exe`.
-  This caches, so only the first configure needs the flag.
-- **Long-term (recommended):** keep `depot_tools` **off** the persistent
-  `PATH`. `third_party/pdfium/build_pdfium.ps1` already prepends it
-  per-run, so the PDFium build still works.
-
-## Verifying your setup — `scripts/test_phase0.{ps1,sh}`
-
-After bootstrap, this aggregator runs every Phase 0 check in sequence and
-prints one pass/fail summary. It is the local mirror of
-[.github/workflows/ci.yml](../.github/workflows/ci.yml) — new teammates should
-use it as the "did I set everything up right?" one-liner.
-
-Pieces it runs, in order:
-
-1. **Rule R2 boundary** — `scripts/check_pdfium_boundary.*`
-2. **Engine** — `cmake --preset … --build --preset … ctest --preset …`
-3. **Gateway** — `pip install -e .[dev]` + ruff (lint + format check) + pytest
-4. **WASM hello-world** — `cmake --preset wasm` + `node wasm/hello.test.mjs`
-
-A piece **skips** (not fails) when its toolchain is absent — no Emscripten on
-`PATH` skips WASM, no `python` skips the gateway, etc. Each piece is
-independent: one failure does not abort later pieces. Exit code is non-zero
-only if a piece genuinely **failed**, so you can pipe it into CI.
-
-### Windows
-
-`vcvars64.bat` must be active in the shell (or use *Developer PowerShell for
-VS*) and `VCPKG_ROOT` must be set:
-
-```powershell
-./scripts/test_phase0.ps1 -Preset win-local           # debug, no PDFium - fastest
-./scripts/test_phase0.ps1 -Preset win-local-pdfium    # release + PDFium - full
-```
-
-Both presets live in your local `CMakeUserPresets.json` (template in the
-"Windows + PDFium" section above), with `binaryDir` pointed outside OneDrive on
-a space-free path.
-
-Other useful flags: `-BinaryDir <path>` (override the preset's binaryDir),
-`-SkipEngine` / `-SkipGateway` / `-SkipWasm` (skip a piece explicitly).
-
-### Linux / macOS
-
-```sh
-./scripts/test_phase0.sh                              # auto-picks linux-debug or macos-debug
-./scripts/test_phase0.sh linux-asan                   # different preset
-SKIP_WASM=1 ./scripts/test_phase0.sh                  # if Emscripten not installed
-PHASE0_BINARY_DIR=/tmp/build ./scripts/test_phase0.sh # override binaryDir
-```
-
-Compatible with bash 3.2 (macOS default), so no `brew install bash` needed.
-
-### Gotcha: stale build dir after a triplet switch
-
-If a build dir was previously configured against the dynamic
-(`x64-windows`) vcpkg triplet and you re-run against the static
-(`x64-windows-static`) one — which is what the `windows-base` preset now pins
-for the static-CRT story — vcpkg correctly purges the old libs, but ninja's
-build graph still references them. You will see:
-
-```
-ninja: error: 'vcpkg_installed/x64-windows/debug/lib/harfbuzz.lib' ... missing
-```
-
-Fix by regenerating from scratch:
-
-```powershell
-cmake --preset win-local --fresh                      # or whichever preset
-./scripts/test_phase0.ps1 -Preset win-local
-```
-
-`--fresh` is the cleanest option (CMake ≥ 3.24); it preserves the build dir
-but invalidates the cache so all targets are re-resolved. Alternatively, delete
-the build dir and reconfigure.
-
-## Dependency pinning
-
-The blueprint rule is *"pin all dependency versions on Day 1, never track
-rolling HEAD."* Two mechanisms:
-
-- **vcpkg deps** — `scripts/bootstrap` runs `vcpkg x-update-baseline
-  --add-initial-baseline`, which writes a `builtin-baseline` commit into
-  `vcpkg.json`. That pins the entire dependency registry to one commit. The
-  root `CMakeLists.txt` **refuses to configure** until this is present.
-  → **The first commit to the repo must include the bootstrapped `vcpkg.json`**,
-  otherwise CI fails at the configure step (by design).
-- **PDFium** — `third_party/pdfium/pdfium.pinned` holds an exact commit SHA.
-  The build script refuses to run while it is the placeholder. Rebases are a
-  deliberate, scheduled (quarterly) action.
-
-## Engineering conventions
-
-- **Naming** (`.clang-tidy`): `CamelCase` types, `camelBack` functions,
-  `snake_case` file names.
-- **Errors**: `std::expected<T, E>` internally; `int error_code` across the C ABI.
-- **Branches**: `main`, `develop`, `feature/*`, `release/*`.
-- **Rule R2**: only `engine/src/parser/` may use raw `FPDF_*` APIs —
-  enforced by `scripts/check_pdfium_boundary.*` locally and in CI.
-
-## What is intentionally NOT done this session
-
-- **PDFium is not actually built** — scripts are staged; the revision needs to
-  be pinned and the (long) build run as the second half of Task 2.
-- **No interface contracts** — `engine/include/pdfengine/pdf_document.hpp` is a
-  placeholder. `PdfDocument` / `PdfPage` are designed and frozen at **Gate G0b**
-  in an all-devs session; nothing proceeds until it is signed off.
-- **Skia / FreeType / HarfBuzz wrappers** — FreeType + HarfBuzz are in the vcpkg
-  manifest and link-tested, but the actual wrappers are Dev 3's Phase 0/1 work.
-  Skia is Dev 2's task.
-- **FastAPI / React / WASM** — placeholder directories only. WASM has a toolchain
-  hook and preset stub so the integration point exists (Rule R5: WASM never
-  blocks shipping).
-
-## Gates ahead
-
-| Gate | Criterion | Unblocks |
-|------|-----------|----------|
-| **G0**  | All platforms build clean; PDFium + Skia + FreeType + HarfBuzz compile | Phase 1 |
-| **G0b** | `PdfDocument` / `PdfPage` contracts locked by all 3 devs | Coding begins |
-
-The CI `build` matrix is the automated half of G0. The smoke test
-(`engine/tests/smoke_test.cpp`) is what it runs.
-
-## OneDrive warning
-
-This checkout lives under `OneDrive\Work\Maskan\PDF Editor\Code`. The path is
-both OneDrive-synced **and** contains a space (`PDF Editor`). Both bite C++
-builds:
-
-1. **Sync churn** — build output is thousands of `.obj`/`.o` files. `out/` is
-   git-ignored, but OneDrive still tries to upload it.
-2. **File locks** — OneDrive can hold a handle on a file mid-sync, causing
-   intermittent "permission denied" errors during compile or link.
-3. **Spaces in build paths break tooling** — vcpkg/meson (harfbuzz) fail
-   with `LNK1181` when `vcpkg_installed` is under the spaced path, and
-   `depot_tools` / GN / Ninja `.bat` wrappers cannot handle a space in their
-   own path at all.
-
-**On Windows, building inside the repo path is not viable** — put the build
-dir outside OneDrive on a space-free path. The `win-local-pdfium` preset
-above already does this (`C:/Users/<you>/pdfeng-build/...`); do the same for
-any non-PDFium preset by overriding `binaryDir`:
-
-```powershell
-cmake --preset windows-release -B C:/Users/<you>/pdfeng-build/windows-release
-```
-
-The PDFium build is even stricter: `third_party/pdfium/build_pdfium.ps1`
-takes a `PDFIUM_BUILD_ROOT` env var and hard-errors if it contains a space.
-Use e.g. `C:\Users\<you>\pdfium-build`.
-
-On Linux/macOS the OneDrive path is still a sync nuisance but the toolchain
-itself is fine. Either point the build dir outside OneDrive, or exclude
-`out/` and `vcpkg/` from sync, or pause sync while building.
-
-Long term, the repository should live outside OneDrive on a real Git remote.

engine/CMakeLists.txt

@@ -1,19 +1,95 @@
-# pdfengine — the C++23 PDF SDK core.
-#
-
-# Generate the version header from the project version.
 configure_file(
     "${CMAKE_CURRENT_SOURCE_DIR}/include/pdfengine/version.hpp.in"
     "${CMAKE_CURRENT_BINARY_DIR}/generated/pdfengine/version.hpp"
     @ONLY)
 
-add_library(pdfengine STATIC
+find_package(PNG REQUIRED)
+
+add_library(pdfengine OBJECT
     src/core/engine_info.cpp
+    src/core/graphics_state.cpp
+    src/core/display_list.cpp
+    src/core/path_interpreter.cpp
+    src/core/skia_renderer.cpp
+    src/parser/content_stream_parser.cpp
+    src/parser/decoration_builder.cpp
+    src/document/document_normalizer.cpp
+    src/document/document_builder.cpp
+    src/document/document_validator.cpp
+    src/text/selection.cpp
+    src/text/text_layout_engine.cpp
+    src/fonts/face/font_face.cpp
+    src/fonts/face/free_type_manager.cpp
+    src/fonts/loader/font_resolver.cpp
+    src/fonts/shaping/hb_shaper.cpp
+    src/fonts/cache/glyph_bitmap.cpp
+    src/fonts/cache/glyph_cache.cpp
+    src/fonts/pdf_fonts/types/truetype_font.cpp
+    src/fonts/pdf_fonts/types/type1_font.cpp
+    src/fonts/pdf_fonts/types/cid_font.cpp
+    src/fonts/pdf_fonts/font_loader.cpp
+    src/fonts/pdf_fonts/font_descriptor.cpp
+    src/fonts/pdf_fonts/font_fallback.cpp
+    src/fonts/pdf_fonts/font_subset.cpp
+    src/fonts/pdf_fonts/font_cmap_builder.cpp
+    src/fonts/pdf_fonts/font_validator.cpp
+    src/fonts/pdf_fonts/font_extraction_service.cpp
+    src/fonts/pdf_fonts/embedded_font_reconstructor.cpp
+    src/fonts/pdf_fonts/encoding/encoding.cpp
+    src/fonts/pdf_fonts/encoding/tounicode_parser.cpp
+    src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp
     src/parser/pdfium_loader.cpp
-    src/fonts/font_face.cpp
-    src/fonts/hb_shaper.cpp
+    src/parser/pdfium_document.cpp
+    src/parser/pdfium_internal.cpp
+    src/parser/pdfium_reflow.cpp
+    src/parser/pdfium_page.cpp
+    src/parser/pdfium_page_model.cpp
+    src/parser/pdfium_fonts.cpp
+    src/parser/pdfium_edit_session.cpp
+    src/parser/pdfium_edit.cpp
+    src/parser/pdfium_edit_replace.cpp
+    src/parser/pdfium_edit_reflow.cpp
+    src/parser/pdfium_edit_annotations.cpp
+    src/parser/pdfium_edit_pages.cpp
+    src/parser/pdfium_edit_images.cpp
+    src/qpdf/qpdf_extractor.cpp
+    src/qpdf/qpdf_font_extractor.cpp
+    src/qpdf/qpdf_writer.cpp
+    src/qpdf/qpdf_resource_resolver.cpp
+    src/core/image_decoder.cpp
+    src/image/core/image_object.cpp
+    src/image/core/image_validator.cpp
+    src/image/core/image_builder.cpp
+    src/image/core/image_pipeline.cpp
+    src/image/core/image_manager.cpp
+    src/image/decoder/filter_decoder.cpp
+    src/image/decoder/sample_decoder.cpp
+    src/image/decoder/pixel_decoder.cpp
+    src/image/decoder/color_converter.cpp
+    src/image/decoder/mask_processor.cpp
+    src/image/decoder/image_decoder_factory.cpp
+    src/ocr/ocr_cache.cpp
+    src/ocr/image_cleaner.cpp
+    src/ocr/ocr_importer.cpp
+    src/ocr/ocr_coordinator.cpp
+    src/layout/layout_arena.cpp
+    src/layout/spatial_index.cpp
+    src/layout/multi_level_cache.cpp
+    src/layout/layout_session.cpp
+    src/layout/pass_registry.cpp
+    src/layout/layout_engine.cpp
+    src/layout/passes/line_detection_pass.cpp
+    src/layout/passes/paragraph_detection_pass.cpp
+    src/layout/passes/column_detection_pass.cpp
+    src/layout/passes/region_detection_pass.cpp
+    src/parser/lexer.cpp
+    src/parser/parser.cpp
+    src/parser/content_builder.cpp
+    src/serializer/content_serializer.cpp
+    src/serializer/ast_serializer.cpp
 )
 add_library(pdfengine::pdfengine ALIAS pdfengine)
+set_target_properties(pdfengine PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
 target_include_directories(pdfengine
     PUBLIC
@@ -29,11 +105,53 @@ target_link_libraries(pdfengine
     PRIVATE
         freetype
         harfbuzz::harfbuzz
+        harfbuzz::harfbuzz-subset
+        PNG::PNG
+        nlohmann_json::nlohmann_json
 )
 
 if(PDFENGINE_WITH_PDFIUM)
-    target_link_libraries(pdfengine PRIVATE pdfium::pdfium)
-    target_compile_definitions(pdfengine PRIVATE PDFENGINE_WITH_PDFIUM)
+    target_link_libraries(pdfengine PUBLIC pdfium::pdfium)
+    target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_PDFIUM)
+    # PDFium statically bundles its own libjpeg, zlib, etc. which conflict with vcpkg.
+    # MSVC ignores -Wl,--allow-multiple-definition (that is a GNU ld flag).
+    if(MSVC)
+        target_link_options(pdfengine PUBLIC "/FORCE:MULTIPLE")
+    else()
+        target_link_options(pdfengine PUBLIC "-Wl,--allow-multiple-definition")
+    endif()
+endif()
+
+if(EMSCRIPTEN)
+    target_compile_definitions(pdfengine PRIVATE PDFENGINE_FONT_DIR="/fonts")
+else()
+    target_compile_definitions(pdfengine PRIVATE PDFENGINE_FONT_DIR="${CMAKE_SOURCE_DIR}/engine/assets/fonts")
+endif()
+
+if(PDFENGINE_WITH_SKIA)
+    target_link_libraries(pdfengine PRIVATE skia::skia)
+    target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_SKIA)
+endif()
+
+# qpdf / libjpeg are used by sources that compile even when PDFENGINE_WITH_QPDF is
+# off (resource_resolver.hpp, qpdf_resource_resolver.cpp, image_decoder.cpp,
+# filter_decoder.cpp). vcpkg puts the headers on the include path automatically,
+# so the objects compile — but MSVC then fails with LNK2019 unless we link here.
+# pdfengine is an OBJECT library: PUBLIC is required so consumers (the Python
+# module and smoke test) actually get these libs on the final link line.
+find_package(ZLIB REQUIRED)
+find_package(JPEG REQUIRED)
+find_package(qpdf CONFIG REQUIRED)
+if(NOT TARGET zs)
+    add_library(zs ALIAS ZLIB::ZLIB)
+endif()
+if(NOT TARGET jpeg)
+    add_library(jpeg ALIAS JPEG::JPEG)
+endif()
+target_link_libraries(pdfengine PUBLIC qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
+
+if(PDFENGINE_WITH_QPDF)
+    target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_QPDF)
 endif()
 
 pdfengine_set_warnings(pdfengine)
@@ -42,3 +160,30 @@ pdfengine_enable_sanitizers(pdfengine)
 if(PDFENGINE_BUILD_TESTS)
     add_subdirectory(tests)
 endif()
+
+if(PDFENGINE_FUZZING)
+    set(PDFENGINE_FUZZ_SANITIZERS "fuzzer,address,undefined" CACHE STRING
+        "Sanitizer set for fuzzing (e.g. 'fuzzer,address,undefined' or just 'fuzzer')")
+
+    string(REPLACE "," ";" _fuzz_sans "${PDFENGINE_FUZZ_SANITIZERS}")
+    set(_fuzz_runtime_sans "")
+    foreach(_s IN LISTS _fuzz_sans)
+        if(NOT _s STREQUAL "fuzzer")
+            list(APPEND _fuzz_runtime_sans "${_s}")
+        endif()
+    endforeach()
+    list(JOIN _fuzz_runtime_sans "," _fuzz_runtime_str)
+
+    target_compile_options(pdfengine PRIVATE -fsanitize=fuzzer-no-link -fno-omit-frame-pointer)
+    if(_fuzz_runtime_str)
+        target_compile_options(pdfengine PRIVATE -fsanitize=${_fuzz_runtime_str})
+        target_link_options(pdfengine PUBLIC -fsanitize=${_fuzz_runtime_str})
+    endif()
+
+    add_executable(pdfengine_fuzz fuzz/fuzz_load.cpp)
+    target_link_libraries(pdfengine_fuzz PRIVATE pdfengine)
+    target_compile_options(pdfengine_fuzz PRIVATE
+        -fsanitize=${PDFENGINE_FUZZ_SANITIZERS} -fno-omit-frame-pointer)
+    target_link_options(pdfengine_fuzz PRIVATE
+        -fsanitize=${PDFENGINE_FUZZ_SANITIZERS})
+endif()

engine/fuzz/README.md

@@ -0,0 +1,65 @@
+# Fuzzing the PDF engine
+
+`fuzz_load.cpp` is a libFuzzer harness that drives the full
+**load → metadata → outline → render → text → annotations → hit-test → select**
+path with arbitrary bytes. Combined with AddressSanitizer it surfaces crashes,
+OOMs, and undefined behaviour in the parsing and rendering code.
+
+Resource ceilings from `pdfengine/hardened_limits.h` keep the fuzzer focused on
+logic bugs instead of trivial out-of-memory inputs (and those same ceilings now
+guard the production render path against integer-overflow / OOM).
+
+## Linux (primary)
+
+Clang + libFuzzer + ASan is best supported on Linux. PDFium must be built with
+the same Clang toolchain (so ASan is consistent across the static lib).
+
+```bash
+# Full ASan + coverage fuzzer
+cmake --preset fuzz-linux
+cmake --build --preset fuzz-linux
+
+# If your PDFium static lib is NOT ASan-instrumented, use coverage-only:
+cmake --preset fuzz-linux-nosan
+cmake --build --preset fuzz-linux-nosan
+
+# Run it against the downloaded corpus as a seed set
+python scripts/fetch_corpus.py                  # populates corpus/fuzz/ (gitignored)
+mkdir -p engine/fuzz/artifacts
+./out/build/fuzz-linux/bin/pdfengine_fuzz \
+    -artifact_prefix=engine/fuzz/artifacts/ \
+    corpus/fuzz/ corpus/
+```
+
+`corpus/fuzz/` and `corpus/` are passed as seed corpora; new coverage-expanding
+inputs are written back into the first directory. Crashes land in
+`engine/fuzz/artifacts/` (gitignored).
+
+## Windows (clang-cl)
+
+Native Windows fuzzing needs a Clang toolchain *and* a PDFium static lib built
+with the matching runtime. Configure with clang-cl and the existing
+`x64-windows-static` triplet, then enable fuzzing:
+
+```powershell
+cmake -S . -B C:/Users/<you>/pdfeng-build/fuzz-win -G Ninja `
+  -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl `
+  -DVCPKG_TARGET_TRIPLET=x64-windows-static `
+  -DPDFENGINE_FUZZING=ON -DPDFENGINE_WITH_PDFIUM=ON `
+  -DPDFENGINE_FUZZ_SANITIZERS=fuzzer `
+  --toolchain "$env:VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake"
+cmake --build C:/Users/<you>/pdfeng-build/fuzz-win
+```
+
+Use `PDFENGINE_FUZZ_SANITIZERS=fuzzer` (coverage-only) on Windows unless the
+whole dependency chain — including PDFium — is ASan-built, since mixing an
+ASan binary with a non-ASan MSVC static lib does not link cleanly.
+
+## Reproducing a crash
+
+```bash
+./pdfengine_fuzz engine/fuzz/artifacts/crash-<hash>
+```
+
+The ASan report points at the offending allocation/access; the input file is the
+minimal reproducer (run with `-minimize_crash=1` to shrink further).

engine/fuzz/fuzz_load.cpp

@@ -0,0 +1,43 @@
+
+#include "pdfengine/hardened_limits.h"
+#include "pdfengine/pdf_document.hpp"
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+    using namespace pdfengine;
+
+    if (!limits::documentSizeOk(size)) return 0;
+
+    std::vector<uint8_t> bytes(data, data + size);
+    auto doc = PdfDocument::loadFromMemory(bytes, "");
+    if (!doc) return 0;
+    PdfDocument& d = **doc;
+
+    const int pages = d.pageCount();
+    if (!limits::pageCountOk(pages)) return 0;
+
+    (void)d.metadata();
+    (void)d.extractOutline();
+
+    const int limit = pages < 3 ? pages : 3;
+    for (int i = 0; i < limit; ++i) {
+        auto page = d.getPage(i);
+        if (!page) continue;
+        PdfPage& p = **page;
+
+        (void)p.render(72);
+        (void)p.extractText();
+        (void)p.extractAnnotations();
+
+        auto glyphs = p.orderedGlyphs();
+        if (glyphs && !glyphs->empty()) {
+            const auto& g = glyphs->front();
+            (void)p.hitGlyph(g.x, g.y);
+            (void)p.selectRange(g.x, g.y, g.x + 50.0, g.y + 20.0);
+        }
+    }
+    return 0;
+}

engine/include/pdfengine/ast.hpp

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <cstdint>
+#include <memory>
+
+namespace pdfengine {
+
+enum class AstNodeType {
+    Number,
+    Name,
+    String,
+    HexString,
+    Boolean,
+    Null,
+    Array,
+    Dictionary
+};
+
+class AstNode {
+public:
+    AstNodeType type;
+    std::string stringValue;
+    std::vector<uint8_t> bytesValue;
+    double numberValue = 0.0;
+    bool boolValue = false;
+    
+    std::vector<std::shared_ptr<AstNode>> arrayItems;
+    std::unordered_map<std::string, std::shared_ptr<AstNode>> dictItems;
+
+    AstNode() = default;
+    explicit AstNode(AstNodeType t) : type(t) {}
+};
+
+struct Operation {
+    std::string op;
+    std::vector<std::shared_ptr<AstNode>> operands;
+};
+
+}

engine/include/pdfengine/content_object.hpp

@@ -0,0 +1,79 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <pdfengine/graphics_state.hpp>
+#include <pdfengine/path.hpp>
+#include <pdfengine/image_object.hpp>
+
+namespace pdfengine {
+
+enum class ContentObjectType {
+    Text,
+    Path,
+    Image,
+    Unknown
+};
+
+enum class XObjectType {
+    Image,
+    Form,
+    Pattern,
+    Unknown
+};
+
+class ContentObject {
+public:
+    virtual ~ContentObject() = default;
+    virtual ContentObjectType getType() const = 0;
+};
+
+class TextObject : public ContentObject {
+public:
+    ContentObjectType getType() const override { return ContentObjectType::Text; }
+    
+    std::string text;
+    std::string fontName;
+    double fontSize = 0.0;
+    
+    double tm[6] = {1.0, 0.0, 0.0, 1.0, 0.0, 0.0};
+};
+
+class ImageContentObject : public ContentObject {
+public:
+    ContentObjectType getType() const override { return ContentObjectType::Image; }
+
+    std::shared_ptr<const ImageObject> image;
+    Matrix transform;
+
+    // Helper compatibility properties
+    std::string name;
+    int width = 0;
+    int height = 0;
+    std::string colorSpace;
+    std::string filter;
+    int bitsPerComponent = 8;
+    bool hasSoftMask = false;
+    std::vector<uint8_t> pixelData;
+};
+
+using LegacyImageObject = ImageContentObject;
+
+enum class PathPaintOp {
+    Stroke,
+    Fill,
+    FillStroke
+};
+
+class PathObject : public ContentObject {
+public:
+    ContentObjectType getType() const override { return ContentObjectType::Path; }
+
+    Path path;
+    PathPaintOp paintOp = PathPaintOp::Stroke;
+    FillRule fillRule = FillRule::NonZero;
+    Matrix transform;
+};
+
+} // namespace pdfengine