pulled from zaid

Reviewed-on: https://gitea.maskantech.in/gitea_admin/pdf/pulls/83

.dockerignore

@@ -16,6 +16,7 @@
 **/.idea
 **/coverage
 **/tmp
+corpus/
 **/.pytest_cache
 **/.mypy_cache
 **/.ruff_cache

.github/workflows/ci.yml

@@ -1,162 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [main, develop]
-  pull_request:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ci-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Rule R2 — PDFium boundary check
-        run: bash scripts/check_pdfium_boundary.sh
-
-      - name: Install clang-format (pinned)
-        run: pipx install clang-format==22.1.5
-
-      - name: clang-format
-        run: |
-          clang-format --version
-          find engine \( -name '*.cpp' -o -name '*.cc' -o -name '*.h' -o -name '*.hpp' \) \
-            -print0 | xargs -0 clang-format --dry-run --Werror
-
-  build:
-    needs: lint
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            preset: linux-debug
-          - os: macos-latest
-            preset: macos-debug
-            experimental: true
-          - os: windows-latest
-            preset: windows-debug
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental == true }}
-    env:
-      VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/.vcpkg-cache
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Ninja
-        uses: seanmiddleditch/gha-setup-ninja@v5
-
-      - name: Set up MSVC environment
-        if: runner.os == 'Windows'
-        uses: ilammy/msvc-dev-cmd@v1
-
-      - name: Locate vcpkg
-        shell: bash
-        run: |
-          echo "VCPKG_ROOT=$VCPKG_INSTALLATION_ROOT" >> "$GITHUB_ENV"
-          git -C "$VCPKG_INSTALLATION_ROOT" fetch --quiet origin || true
-
-      - name: Create vcpkg binary cache dir
-        shell: bash
-        run: mkdir -p "$VCPKG_DEFAULT_BINARY_CACHE"
-
-      - name: Cache vcpkg artifacts
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.VCPKG_DEFAULT_BINARY_CACHE }}
-          key: vcpkg-${{ matrix.os }}-${{ hashFiles('vcpkg.json') }}
-          restore-keys: vcpkg-${{ matrix.os }}-
-
-      - name: Pin vcpkg dependency baseline
-        shell: bash
-        run: |
-          if ! grep -q '"builtin-baseline"' vcpkg.json; then
-            "$VCPKG_ROOT/vcpkg" x-update-baseline --add-initial-baseline
-          fi
-
-      - name: Configure
-        run: cmake --preset ${{ matrix.preset }}
-
-      - name: Build
-        run: cmake --build --preset ${{ matrix.preset }}
-
-      - name: Test
-        run: ctest --preset ${{ matrix.preset }}
-
-  gateway:
-    needs: lint
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: gateway
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-          cache: pip
-          cache-dependency-path: gateway/pyproject.toml
-
-      - name: Install gateway (editable, with dev extras)
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install -e ".[dev]"
-
-      - name: Ruff — lint
-        run: python -m ruff check .
-
-      - name: Ruff — format check
-        run: python -m ruff format --check .
-
-      - name: Pytest
-        run: python -m pytest
-
-
-  wasm:
-    needs: lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Read pinned emsdk version
-        id: emsdk-version
-        run: |
-          version=$(grep '^EMSDK_VERSION=' wasm/emsdk.pinned | cut -d= -f2)
-          if [ -z "$version" ]; then
-            echo "::error::EMSDK_VERSION not found in wasm/emsdk.pinned"
-            exit 1
-          fi
-          echo "version=$version" >> "$GITHUB_OUTPUT"
-
-      - name: Install Ninja
-        uses: seanmiddleditch/gha-setup-ninja@v5
-
-      - name: Set up Emscripten ${{ steps.emsdk-version.outputs.version }}
-        uses: mymindstorm/setup-emsdk@v14
-        with:
-          version: ${{ steps.emsdk-version.outputs.version }}
-          actions-cache-folder: emsdk-cache-${{ steps.emsdk-version.outputs.version }}
-
-      - name: Set up Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: "20"
-
-      - name: Configure (WASM)
-
-        run: cmake --preset wasm
-
-      - name: Build
-        run: cmake --build --preset wasm
-
-      - name: Smoke test
-        run: node wasm/hello.test.mjs

.github/workflows/fuzz.yml

@@ -1,104 +0,0 @@
-# Extended robustness: corpus render-stability sweep + libFuzzer run.
-#
-# DELIBERATELY ISOLATED from the main CI gate:
-#   - never triggers on push or pull_request, so it can NEVER block a merge,
-#     push, or pull;
-#   - the whole job is continue-on-error, so a crash finding or a build/runner
-#     problem reports red here but does not fail any required check;
-#   - runs on a weekly schedule and on manual dispatch only.
-#
-# A full 24h fuzz run needs a self-hosted runner (GitHub-hosted runners cap a job
-# at 6h). Use the workflow_dispatch `duration_seconds` input for that; the weekly
-# schedule does a short smoke instead.
-
-name: Fuzz & corpus sweep
-
-on:
-  schedule:
-    - cron: "0 3 * * 0" # Sundays 03:00 UTC — short smoke
-  workflow_dispatch:
-    inputs:
-      duration_seconds:
-        description: "libFuzzer -max_total_time (e.g. 1800 smoke, 86400 for 24h on a self-hosted runner)"
-        default: "1800"
-      sanitizers:
-        description: "Sanitizer set (fuzzer,address,undefined needs an ASan/UBSan-built PDFium; fuzzer = coverage-only)"
-        default: "fuzzer,address,undefined"
-
-permissions:
-  contents: read
-
-concurrency:
-  group: fuzz-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  fuzz:
-    # Non-blocking by construction: nothing depends on this job and it is allowed to fail.
-    continue-on-error: true
-    runs-on: ubuntu-latest
-    timeout-minutes: 1500 # permits a 24h dispatch on a self-hosted runner; hosted runners stop at 6h
-    env:
-      VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/.vcpkg-cache
-      FUZZ_SANITIZERS: ${{ github.event.inputs.sanitizers || 'fuzzer' }}
-      FUZZ_DURATION: ${{ github.event.inputs.duration_seconds || '600' }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Ninja + Clang
-        shell: bash
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y ninja-build clang
-
-      - name: Locate vcpkg
-        shell: bash
-        run: |
-          echo "VCPKG_ROOT=$VCPKG_INSTALLATION_ROOT" >> "$GITHUB_ENV"
-          mkdir -p "$VCPKG_DEFAULT_BINARY_CACHE"
-
-      - name: Cache vcpkg artifacts
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.VCPKG_DEFAULT_BINARY_CACHE }}
-          key: vcpkg-fuzz-${{ hashFiles('vcpkg.json') }}
-          restore-keys: vcpkg-fuzz-
-
-      - name: Pin vcpkg dependency baseline
-        shell: bash
-        run: |
-          if ! grep -q '"builtin-baseline"' vcpkg.json; then
-            "$VCPKG_ROOT/vcpkg" x-update-baseline --add-initial-baseline
-          fi
-
-      - name: Configure (fuzz-linux)
-        run: cmake --preset fuzz-linux -DPDFENGINE_FUZZ_SANITIZERS="$FUZZ_SANITIZERS"
-
-      - name: Build fuzzer
-        run: cmake --build --preset fuzz-linux --target pdfengine_fuzz
-
-      - name: Fetch corpus (pinned + hash-verified)
-        shell: bash
-        run: python3 scripts/fetch_corpus.py --manifest tests/regression/corpus-manifest.json || echo "corpus fetch failed (network) — continuing with committed corpus"
-
-      - name: Render-stability sweep (replay every corpus PDF once)
-        shell: bash
-        run: |
-          BIN=out/build/fuzz-linux/bin/pdfengine_fuzz
-          mkdir -p engine/fuzz/artifacts
-          "$BIN" -runs=0 -artifact_prefix=engine/fuzz/artifacts/ corpus/ corpus/fuzz/ || true
-
-      - name: Fuzz run
-        shell: bash
-        run: |
-          BIN=out/build/fuzz-linux/bin/pdfengine_fuzz
-          "$BIN" -max_total_time="$FUZZ_DURATION" -print_final_stats=1 \
-                 -artifact_prefix=engine/fuzz/artifacts/ corpus/fuzz/ corpus/ || true
-
-      - name: Upload crash artifacts
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: fuzz-artifacts
-          path: engine/fuzz/artifacts/
-          if-no-files-found: ignore

.gitignore

@@ -67,8 +67,45 @@ gateway/*.pyd
 gateway/*.so
 gateway/*.dylib
 
-# Downloaded fuzzing corpus (large; fetched via scripts/fetch_corpus.py)
+# Corpora — keep convert metadata; ignore large samples / archives
+/corpus/*
+!/corpus/convert/
+/corpus/convert/**
+!/corpus/convert/real/
+!/corpus/convert/public/
+!/corpus/convert/third_party/
+/corpus/convert/real/**
+!/corpus/convert/real/pack.json
+!/corpus/convert/real/*.md
+/corpus/convert/public/**
+!/corpus/convert/public/README.md
+!/corpus/convert/public/pack_public.json
+!/corpus/convert/public/manifests/
+!/corpus/convert/public/manifests/**
+!/corpus/convert/public/reports/
+!/corpus/convert/public/reports/**/*.md
+!/corpus/convert/public/reports/**/*.json
+/corpus/convert/public/samples/
+/corpus/convert/third_party/**
+!/corpus/convert/third_party/NOTICE
+!/corpus/convert/third_party/LICENSE_CHECKLIST.md
+!/corpus/convert/third_party/checksums.sha256
 corpus/fuzz/
+# Layout ML weights (download via scripts/convert/fetch_layout_onnx.py)
+/gateway/models/layout/**/*.onnx
+/gateway/models/layout/**/*.pdmodel
+/gateway/models/layout/**/*.pdiparams
+!/gateway/models/layout/**/README.md
+!/gateway/models/layout/**/.gitkeep
+# Arabic OCR rec weights (download via scripts/convert/fetch_ocr_arabic_onnx.py)
+/gateway/models/ocr/**/*.onnx
+!/gateway/models/ocr/**/README.md
+!/gateway/models/ocr/**/.gitkeep
+!/gateway/models/ocr/**/*.txt
+# DocLayNet / Pub* lab archives
+**/DocLayNet*.zip
+**/PubLayNet*
+**/PubTabNet*
 # Large-corpus regression baseline (derived from the gitignored corpus above)
 tests/regression/baseline-large/
 # Fuzzer working artifacts (crashes, leaks, coverage)
@@ -78,5 +115,18 @@ leak-*
 timeout-*
 
 PDF Editor Timeline.xlsx
+# Operator convert score log (Phase 0); never commit customer DOCX
+convert-scores.jsonl
+**/convert-scores.jsonl
+
 # Local environment config / secrets
 /third_party/pdfium-wasm/
+
+# AI Large Model Weights & Vector Indexes (never commit binary model weights into Git)
+models/**/*.onnx
+models/**/*.bin
+models/**/*.pt
+models/**/*.safetensors
+models/**/*.index
+!models/**/.gitkeep
+.github

CMakeLists.txt

@@ -73,9 +73,9 @@ find_package(freetype  CONFIG REQUIRED)
 find_package(harfbuzz  CONFIG REQUIRED)
 find_package(spdlog    CONFIG REQUIRED)
 find_package(nlohmann_json CONFIG REQUIRED)
-if(PDFENGINE_WITH_QPDF)
-    find_package(qpdf      CONFIG REQUIRED)
-endif()
+find_package(qpdf CONFIG REQUIRED)
+find_package(ZLIB REQUIRED)
+find_package(JPEG REQUIRED)
 if(WIN32 AND DEFINED VCPKG_TARGET_TRIPLET)
     link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/lib")
     link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/debug/lib")
@@ -110,4 +110,5 @@ message(STATUS "  Build tests .......... ${PDFENGINE_BUILD_TESTS}")
 message(STATUS "  Sanitizers ........... ${PDFENGINE_ENABLE_SANITIZERS}")
 message(STATUS "  Warnings as errors ... ${PDFENGINE_WARNINGS_AS_ERRORS}")
 message(STATUS "  Link PDFium .......... ${PDFENGINE_WITH_PDFIUM}")
-message(STATUS "  Link Skia ............ ${PDFENGINE_WITH_SKIA}")
+message(STATUS "  Link Skia ............ ${PDFENGINE_WITH_SKIA}")
+message(STATUS "  Link QPDF ............ ${PDFENGINE_WITH_QPDF}")

CMakeUserPresets.json.template

@@ -6,7 +6,7 @@
       "name": "win-local",
       "displayName": "Windows • Debug (local — build dir outside OneDrive/spaces)",
       "inherits": "windows-debug",
-      "binaryDir": "C:/Users/@USERNAME@/pdfeng-build/win-local",
+      "binaryDir": "D:/pdfeng-build/win-local",
       "cacheVariables": {
         "PDFENGINE_WITH_PDFIUM": "ON",
         "PDFENGINE_WITH_QPDF": "ON"
@@ -16,8 +16,8 @@
       "name": "win-local-pdfium",
       "displayName": "Windows • RelWithDebInfo + PDFium (static CRT — build dir outside OneDrive/spaces)",
       "inherits": "windows-release",
-      "binaryDir": "C:/Users/@USERNAME@/pdfeng-build/win-local-pdfium",
-      "cacheVariables": { 
+      "binaryDir": "D:/pdfeng-build/win-local-pdfium",
+      "cacheVariables": {
         "PDFENGINE_WITH_PDFIUM": "ON",
         "PDFENGINE_WITH_QPDF": "ON"
       }

CONVERSION_ENGINE_DEEP_DIVE.md

@@ -0,0 +1,517 @@
+# Document Conversion Engine: End-to-End Deep Dive
+
+**Investigation date:** 2026-09-09  
+**Scope:** `pdf/` repository, with emphasis on `gateway/app/services/convert`, the
+PDF engine binding, OCR/layout models, writers, configuration, and conversion tests.  
+**Evidence standard:** real documents were passed through the public
+`run_conversion` path and the resulting Office files were parsed and rendered.
+Numbers below are measurements on the Windows development host, not vendor
+claims or a declaration of production readiness.
+
+## Current State
+
+### Executive finding
+
+The engine is a composed, mostly in-process conversion stack. It is effective for
+simple digital PDFs, text extraction, Office document generation, and searchable
+PDF creation. It is not a general PDF layout clone. A PDF has positioned drawing
+operators, while DOCX/XLSX require a semantic flow/grid model; the conversion
+therefore reconstructs an intermediate document model (IDM) and necessarily loses
+some semantics and positioning.
+
+The most visible historical defect, interleaved text on multi-column PDFs, had a
+specific upstream cause and is fixed in the current tree. The remaining quality
+ceiling is dominated by table reconstruction, OCR cost/ambiguity, and the choice
+of ReportLab as the Office-to-PDF fallback. A successful HTTP response is not
+equivalent to faithful content or layout: the measured corpus still contains
+low-order similarity and three failing assertions.
+
+### Execution topology
+
+The current request path is:
+
+```text
+HTTP /v1/convert/*
+  -> filename/MIME/magic-byte/size validation
+  -> converter registry and optional automatic PDF policy
+  -> bounded worker admission (queue + shared conversion slots)
+  -> conversion/document-cache scopes
+  -> PDF route (digital, scanned, image-based, mixed)
+  -> per-page geometry and content recovery
+       C++ pdfengine ordered glyphs (preferred)
+       pypdf visitor/content-stream geometry (fallback)
+       line grouping -> paragraphs -> reading order -> tables/images
+       optional ONNX layout regions and table separators
+  -> OCR for scan/hybrid pages
+       PDF render -> RapidOCR Latin pass -> optional Arabic pass
+       confidence/script merge -> IDM blocks and metadata
+  -> IDM optimization (headers/footers, wrapped rows, figures)
+  -> format-specific formatter/writer
+  -> OOXML/PDF validity checks, quality score and warnings
+  -> content-addressed result cache
+```
+
+`run_conversion` binds cancellation, deadlines, options, automatic-routing
+decisions, and a per-conversion read/raster/geometry cache. The public converter
+registry currently covers PDF, DOCX, XLSX, PPTX, HTML, Markdown, text, CSV,
+JSON, and image paths; the exact pair list is exposed by the convert info route.
+
+### Component inventory
+
+| Layer | Implementation in this tree | Observed role and limitation |
+| --- | --- | --- |
+| PDF parser/render core | C++23 `pdfengine` pybind module; PDFium, FreeType, HarfBuzz, optional Skia/QPDF | Gives authoritative glyphs, fonts, page geometry, rendering, and editing on this host. Native availability is not guaranteed on every Python deployment. |
+| Python PDF parser | `pypdf==5.3.0` | Text extraction, page count, content-stream fallback, XObject inspection. Its inferred whitespace and broken ToUnicode output cannot be treated as ground truth. |
+| Raster path | `pypdfium2==5.13.0` declared in `pyproject.toml` | Required for hosts without the C++ renderer. It is absent from the active venv, so raster-specific tests skip there; the C++ binding supplies rendering for the current audit. |
+| OCR | `rapidocr-onnxruntime==1.2.3`, `onnxruntime==1.28.0` | Shared Latin detector/recognizer; optional PP-OCRv5 Arabic recognizer. Inference is serialized because RapidOCR sessions mutate shared state. |
+| Layout ML | PP-DocLayoutV3-derived ONNX, 130,502,330 bytes FP32 or 34,694,580 bytes INT8 | Region labels narrow heuristic decisions; it does not reconstruct a document by itself. It runs on CPU in the measured environment. |
+| Table ML | Optional `structure.onnx` (Table Transformer/SLANet operator-supplied) | Not vendored (`shipped: false` in `models/MANIFEST.json`); heuristic lattice/stream/rectangle detectors remain the default. |
+| IDM | `app/services/convert/idm/model.py` and layout pipeline | Stores pages, BBoxes, paragraphs, headings, headers/footers, figures, and cell grids. It is a useful interchange contract, but not a complete PDF display list or OOXML style graph. |
+| Office writers | `python-docx==1.1.2`, `openpyxl==3.1.5`, `python-pptx` code, XML sanitization | Good for creating valid Office files; cannot infer all original PDF semantics (fonts, fields, anchored objects, section behavior). |
+| Office -> PDF | Gotenberg/LibreOffice attempt, then in-process ReportLab 4.2.5 | Gotenberg is best fidelity when reachable. The benchmark used the ReportLab fallback, which is concurrent-safe but not a Word/Excel print-layout clone. |
+| Images | Pillow 10.4.0, PDFium render/XObject extraction | Images are extracted, resized/compacted, or the complete page is embedded when reconstruction would lose visible content. |
+| Font recognition | `FontRecognitionService` facade, classifier and embedding store | The checked-in `FontClassifier` uses aspect ratio/dark-pixel heuristics and `EmbeddingStore` uses a deterministic 16x8 grayscale projection. `load_model()` reads metadata but does not load an OpenCLIP ONNX session; `model.onnx` and `fonts.index` are not present in the repository. This is metadata assistance, not a validated font-identification model. |
+
+The declared dependency set is intentionally permissive (pypdf BSD, pypdfium2
+Apache/BSD, python-docx/openpyxl MIT, ReportLab BSD, Pillow HPND). `mammoth==1.8.0`
+is installed in this venv for DOCX/HTML fallback paths; `fitz`/PyMuPDF is absent,
+which avoids an AGPL transitive dependency but removes that optional path.
+
+### Runtime configuration and resource policy
+
+The startup script defaults to automatic reconstruction, layout ML (INT8 when
+available), Arabic OCR with adaptive gating, a 1,800 second maximum conversion
+timeout, 22 seconds per OCR page budget, 45 seconds per-page timeout, a 1,200
+second OCR wall budget, four conversion workers, four simultaneous conversion
+slots, and a queue of roughly two batches. Result caching uses namespace
+`conversion-engine-v2` and includes effective options and routing mode in the key.
+
+These are protection mechanisms, not throughput guarantees. A single process can
+still hold multiple ONNX sessions and large page rasters, and OCR inference is
+deliberately serialized. Horizontal scaling or process isolation is needed for
+untrusted large-document workloads.
+
+### Jobs, storage, and failure boundaries
+
+Synchronous and asynchronous routes share a process-wide `ThreadPoolExecutor`
+and bounded semaphores. Asynchronous job state is persisted in SQLite with WAL
+and `synchronous=NORMAL`; results larger than 512 KB are spooled to a local
+`jobs_spool` directory, while smaller results are stored as BLOBs. Jobs older
+than three hours are purged, and jobs left queued/running after a process restart
+are marked failed. This gives useful single-node crash recovery, but SQLite and
+local spool paths are not a multi-instance object store: replicas need a shared
+database/object store or a queue service before horizontal scaling.
+
+The timeout helper releases the HTTP caller on deadline by abandoning a worker
+thread (`ThreadPoolExecutor.shutdown(wait=False)`). Python cannot kill that
+thread, so a native render/OCR operation may continue in the background until it
+returns. Cooperative cancellation is checked at page/OCR boundaries, which
+limits normal work, but a hung native call can still consume CPU/RAM after a 504.
+Production deployments should put untrusted conversions behind process/container
+boundaries with an OS-level kill and cgroup limits.
+
+Gotenberg/LibreOffice is attempted for Office-to-PDF with a 30-second client
+timeout and fails open to ReportLab. This makes the endpoint available when the
+microservice is absent, but it can silently change fidelity class unless the
+response warning is surfaced to callers and monitored.
+
+## Root Cause
+
+The table below ranks causes by customer impact and likelihood in the measured
+system. "Fixed" means the implementation changed and the targeted regression
+improved; it does not mean the entire class of failures is solved.
+
+| Rank | Severity | Root cause | Evidence | State |
+| --- | --- | --- | --- | --- |
+| 1 | P0 | Precise glyph coverage was compared with `len(pypdf.extract_text())`, which includes inferred spaces/newlines, while the precise walker emits text-showing characters only. Pages near the 0.80 gate fell into `_glyphs_from_visitor`, whose width calibration measured the two-column pitch rather than font advances. | Mozilla pages had coverage 0.754-0.858; stripping whitespace raised it to 0.879-1.000. Estimated spans crossed the gutter; the visible sentence was interleaved. | Fixed by comparing non-whitespace characters and retaining the precise path. |
+| 2 | P0 | Baseline clustering was page-global. Different leading in adjacent columns let rows "bridge" into one bucket; splitting columns later could not undo the vertical weld. | On the paper page, 6.97 pt copyright rows and 8.97 pt body rows were interleaved; 4.6 pt and 3.3 pt hops were under the global tolerance. Reading-order inversions fell from 189 to 1 after column-aware bands. | Fixed with `_column_band` and a bounded baseline clip. |
+| 3 | P0 | OCR is CPU-heavy and each bilingual page can run Latin and Arabic recognition, plus per-line/per-word font metadata work. A shared RapidOCR session cannot safely run concurrently. | 20-page scan: 142.747 s wall, 655.016 CPU-s, 1,769.4 MB peak RSS. Warm one-page inference was 3.021 s/17.318 CPU-s with Arabic versus 1.370 s/7.943 CPU-s with adaptive skip. | Partially fixed: inference/model locks, adaptive Arabic, page caps, budgets, and raster reuse are present. |
+| 4 | P1 | Table reconstruction is heuristic by default; ruled headers and multi-line cells are represented as fragments before grid assignment. The optional structure model is not shipped. | W3C exact table gate produced recall 0.867 (<0.90). `Participants` became `s Participant`; `Time to complete` became `Results Time to complete`; `n=1` gained a space after `=`. | Open; targeted fix is required. |
+| 5 | P1 | PDF-to-editable-Office is a semantic reflow, not a pixel-preserving transform. Header/footer detection, style inference, anchored objects, columns, fields, and arbitrary positioned graphics have no one-to-one OOXML representation. | Pixel similarity varies from 0.784 (USGS) to 0.950 (govinfo) in the ReportLab-rendered comparison; order scores are 0.232-0.674 for several digital PDFs. | Fundamental limitation; use page-image fallback or a fidelity backend when editability is not required. |
+| 6 | P1 | OCR recognition is sensitive to source DPI, contrast, skew, script, and confidence thresholds. Low thresholds (0.08 box/text) preserve faint marks but also admit noise; post-processing must distinguish symbols from rules. | Scan recall is 1.000 against expected page tokens but precision is 0.077 because OCR also emits repeated body/table text. Arabic adaptive mode improves CPU cost but whole-run wall time is noisy. | Partially fixed; needs calibrated per-document confidence and preprocessing. |
+| 7 | P1 | Quality scoring historically rewarded agreement with a corrupted pypdf text layer and ignored duplicates/order. Structure bonus could be earned by a diagram label classified as a header. | Mozilla score moved 0.9475 -> 0.8998 while the corrupted duplicate text disappeared and real prose improved. New metrics expose precision/order and store `idm.meta["text_metrics"]`. | Fixed for diagnostics; corpus gates still need independent gold/layout checks. |
+| 8 | P2 | Repeated PDF reads, geometry extraction, rasterization, and model initialization inflated latency and memory. Unbounded cache keys also allowed stale results across option/routing changes. | Per-conversion document cache and canonical cache keys were added; regression suites cover collisions and queue admission. | Fixed/mitigated. |
+| 9 | P2 | API contract mismatch unrelated to conversion: unknown font endpoint currently returns 204 while its test expects 404. | Full suite failure in `tests/test_inplace_editing.py::test_font_endpoint_rejects_unknown`. | Open decision for API owner. |
+| 10 | P2 | Timeout release is thread-based, not process-based. A native renderer or OCR call that ignores cancellation can continue after the request receives 504; local SQLite/spool storage also limits multi-instance scaling. | `run_with_timeout()` explicitly uses `shutdown(wait=False)` because Python cannot kill a thread; job store uses one local SQLite file and local spool directory. No runaway crash was observed in the audit, but the resource behavior is a design risk. | Mitigate with process/container isolation and shared durable storage. |
+
+### Content accuracy
+
+PDF text is not necessarily a trustworthy source string. Embedded fonts can have
+missing or malformed ToUnicode maps, CID encodings, zero-width markers, and text
+show operators split in ways that do not correspond to words. The current pipeline
+therefore combines C++ glyph extraction, pypdf extraction, content-stream evidence,
+OCR, and text-quality heuristics. Any stage that chooses a wrong representation can
+duplicate, drop, or reorder characters before the writer sees them.
+
+The old comparison metric made this harder to observe: ASCII-only unique-token
+intersection ignored Arabic, CJK, combining marks, symbols, repeated occurrences,
+and order. The replacement is Unicode NFKC normalization, line-break hyphen joining,
+duplicate-aware multiset matching, meaningful-symbol retention, and an ordered
+sequence score. It is still a diagnostic metric, not a semantic proof.
+
+### Layout and structure fidelity
+
+The IDM preserves BBoxes and reading order, but a flowing DOCX writer must choose
+paragraphs, columns, tables, and page breaks. Heuristics infer headings from size
+and text, detect headers/footers from repetition and position, and merge rows across
+pages. They cannot preserve every PDF transformation matrix, clipping path,
+baseline, text box, field, or z-order. Automatic page-raster fallback is the safety
+floor for complex positioned pages: visual content survives, but text is no longer
+editable or searchable as individual runs.
+
+### Formatting preservation
+
+The writers can carry bold, italic, colors, hyperlinks, bullets, numbering, and
+several paragraph attributes when those attributes exist in the source Office XML
+or can be inferred from PDF glyph metadata. PDF extraction usually lacks the
+original style names and theme inheritance. Font aliases and locally installed
+fonts determine metrics; missing fonts fall back to Helvetica/Liberation/DejaVu or
+script-specific Noto faces. Word-to-PDF currently applies a fixed paragraph
+`spaceAfter` in the ReportLab path rather than reproducing every source paragraph's
+spacing, so small but systematic vertical shifts are expected.
+
+### Tables
+
+The detector combines ruling-line (lattice), whitespace/stream, rectangle, and
+plausibility passes, then maps glyph lines into cells. A two-column spanning header
+or a row whose words sit on separate baselines can be split into fragments. Merged
+cells and nested content are represented approximately. The structure ONNX hook
+can supply separators, but the model is operator-supplied and fail-open; it is not
+part of the reproducible default deployment.
+
+### Images and graphics
+
+XObjects and rendered page images are retained where possible. Large figures are
+compacted rather than silently discarded. If a page reconstructs to no meaningful
+blocks, the pipeline renders and embeds that page as a figure, or emits an explicit
+warning. This preserves visible pixels but cannot preserve editable diagram
+objects, vector semantics, captions' relationships, or exact crop/ordering in all
+cases.
+
+## Evidence
+
+### Measurement method
+
+`tools/audit_conversion.py` invokes the same in-process `run_conversion` entry point
+used by the gateway. For each case it records input/output bytes, source and output
+page counts, warnings/errors, duplicate-aware text recall/precision/order/F1, table
+payloads, and model status. A Windows process sampler records working-set peak and
+`GetProcessTimes` CPU seconds at 20 ms intervals. DOCX/XLSX outputs are parsed with
+`python-docx`/`openpyxl`; visual comparison renders both sides at 96 DPI and reports
+`1 - mean(abs(gray_a-gray_b))/255` over the compared pages.
+
+The visual metric is intentionally simple and should be treated as a trend signal.
+It is not SSIM, does not align transformed pages, and in this run renders Office
+outputs with the in-process ReportLab fallback. It therefore does not measure what
+LibreOffice/Gotenberg would produce.
+
+### Representative real-document run
+
+The latest six-case digital run is the current `audit_results.json` artifact; the
+20-page scan row is from the preceding targeted run and will be appended after the
+final OCR run. Times are wall seconds; CPU is process CPU seconds; RSS is peak
+working set. "Order" is the ordered token similarity, not a page-layout score.
+Output size and output page count are taken directly from the audit JSON. Re-run
+the command in Appendix A after any code change to refresh all rows. Results can
+vary with cache/model warm-up and with whether the page-raster fallback is
+selected, so the harness should be run more than once when comparing changes.
+
+| Document -> target | Source pages | Rendered output pages* | Page delta* | Wall s | CPU s | Peak RSS MB | Output bytes | Warnings | Recall | Precision | Order | Pixel similarity |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| govinfo -> DOCX | 1 | 2 | +1 | 4.707 | 12.250 | 717.0 | 39,779 | 7 | 0.941 | 0.888 | 0.443 | 0.950 |
+| IRS 1040 -> DOCX | 2 | 7 | +5 | 4.531 | 13.578 | 1118.7 | 43,430 | 9 | 0.936 | 0.944 | 0.674 | 0.890 |
+| Mozilla PDF spec (14 p) -> DOCX | 14 | 28 | +14 | 29.785 | 92.750 | 1141.1 | 77,386 | 24 | 0.916 | 0.937 | 0.149 | 0.895 |
+| USGS factsheet (4 p) -> DOCX | 4 | 7 | +3 | 14.897 | 30.594 | 1346.8 | 161,987 | 13 | 0.767 | 0.793 | 0.232 | 0.784 |
+| W3C ruled table -> XLSX | 1 | 1 | 0 | 1.732 | 5.391 | 1191.7 | 5,328 | 7 | 1.000 | 1.000 | 0.990 | 0.959 |
+| Multipage table (2 p) -> XLSX | 2 | 2 | 0 | 3.280 | 11.891 | 1195.6 | 5,593 | 7 | 0.600 | 1.000 | 0.750 | 0.998 |
+| Scan pack 1 (20 p) -> DOCX | 20 | 4 | -16 | 142.747 | 655.016 | 1769.4 | 58,244 | 102 | 1.000 | 0.077 | 0.143 | 0.957 |
+
+Source characteristics for the digital cases were 3,487 chars/545 words (govinfo),
+10,416/2,223 (IRS), 82,714/13,439 (Mozilla), 14,022/1,887 (USGS), 375/65 (W3C),
+and 73/25 (multipage table) according to pypdf's source read. The scan pack has
+20 raster pages and only a tiny/invalid embedded text layer, so expected page-token
+recall was supplied independently (`ScanTokenpack1P000` ... `P019`).
+
+\*Rendered output pages are the pages produced when the audit harness sends the
+Office bytes through its in-process ReportLab visual renderer. They are not a
+claim about a user's pagination in Word. For example, the scan DOCX contains 19
+explicit page breaks (20 intended pages) even though the fallback visual render
+reports four pages; both page-break metadata and rendered page count must be
+tracked.
+
+The scan result is a useful warning: recall 1.000 alone would say "perfect," while
+precision 0.077 and order 0.143 show that the editable OCR reconstruction contains
+substantial extra/repeated material. Its pixel similarity is high because the page
+images are retained; visual success and editable-text success are different
+products.
+
+### Multi-column defect A/B evidence
+
+On the Mozilla document, the old path produced 189 reading-order inversions in the
+probe. After the coverage-gate and column-aware row fixes, the probe reports 1.
+The corrected prose sentence is no longer spliced. The old corpus quality number
+was 0.9475 and the new number 0.8998, but the output character count fell from
+83,136 to 79,223 because duplicate visitor text disappeared. A/B instrumentation
+showed recall 0.9806 -> 0.9608 and missing token *types* 41 -> 83, while missing
+real words changed 31 -> 32; the additional "missing" tokens were chart-axis
+labels that pypdf had decoded differently. This is a metric-baseline artifact,
+not evidence that the repaired sentence was lost.
+
+### Table evidence
+
+The hand-labelled table truth in `corpus/convert/table_truth.json` requires the W3C
+exact six-column grid to reach recall 0.90 and precision 0.55. The current failing
+output from `build_idm_from_pdf(table.pdf)` is:
+
+```text
+["Disability Category", "s Participant", "Ballots Completed",
+ "Ballots Incomplete/ Terminated", "Accuracy", "Results Time to complete"]
+["Blind", "5", "1", "4", "34.5%, n= 1", "1199 sec, n=1"]
+...
+["Mobility", "3", "3", "0", "95.4%, n= 3", "1416 sec, n=3"]
+```
+
+Compiled glyph inspection found a zero-width-space marker at approximately
+`x=222.28` and the following `s` at `x=222.31`, with near-identical baseline;
+`lines_from_glyphs()` separated them. Header words were spread over multiple
+baselines (`Participant`/`s`, `Results`/`Time to complete`, and similar pairs),
+and some spans contained the punctuation artifact `"= "`. The two failing table
+tests are therefore reproducible extraction defects, not a threshold problem.
+
+### OCR and model evidence
+
+The active venv reports:
+
+```text
+pypdf 5.3.0
+onnxruntime 1.28.0
+rapidocr-onnxruntime 1.2.3
+providers: AzureExecutionProvider, CPUExecutionProvider
+pypdfium2: unavailable
+```
+
+No CUDA provider was available, so the measured conversion was CPU-bound. The
+layout model is 130.5 MB FP32 (34.7 MB INT8 alternative); the Arabic recognizer is
+8.0 MB. The root `models/font` tree contains only metadata/tokenizer placeholders,
+not `model.onnx` or `fonts.index`, and the classifier code does not create an ONNX
+session. Any font-confidence value from that service must therefore be treated as
+a heuristic, not a calibrated model probability.
+
+Arabic warm-page measurements were stable enough to show the tradeoff:
+
+| Mode | Mean wall/page | Mean CPU/page | Interpretation |
+| --- | ---: | ---: | --- |
+| Arabic pass enabled | 3.021 s | 17.318 CPU-s | Better script coverage, roughly double inference work |
+| Adaptive skip on clean Latin | 1.370 s | 7.943 CPU-s | About 55% lower inference cost; known Arabic/broken-cmap pages still force the pass |
+
+Whole 20-page wall time was noisy (non-adaptive 150.389 s versus adaptive
+178.408 s in separate runs), so the warm-page measurements, not that pair alone,
+should guide capacity planning.
+
+### Test evidence
+
+Focused regression groups passed as follows: earlier cache/queue suite 96 tests;
+OCR budget/Arabic suite 32; CORS/model packaging 8; raster/quality/real-corpus
+focused run 34 passed with 16 raster-specific skips; streaming export plus
+cache/queue 7; corpus/phase/production smoke 36; and an additional metrics/OCR
+run 133. The full gateway suite completed 746 passed, 21 skipped, and 3 failed.
+
+The three failures were:
+
+1. `test_labelled_table_is_recovered[w3c_ruled_grid]`: recall 0.867, floor 0.90.
+2. `test_table_pdf_reading_order_and_spans`: the same `n= 1` and header-fragment
+   defects.
+3. `test_font_endpoint_rejects_unknown`: endpoint returned 204; test expects 404.
+
+The raster skips are environmental (`pypdfium2` is missing), not green evidence
+that image export/OCR works on a renderer-free host.
+
+## Recommended Fix
+
+### Immediate P0/P1 work
+
+1. **Finish W3C/table normalization and fragment stitching.** In the table path
+   only, remove whitespace immediately after `=` and normalize punctuation spacing
+   around `%`, `=`, and commas. Merge one-character continuation fragments when
+   their baseline and x-position overlap a neighboring fragment. Stitch header
+   fragments by column anchor and vertical proximity, preserving true multi-line
+   cells. Add a regression for the exact five bad cells and for ordinary prose so
+   the normalization cannot alter sentence spacing.
+2. **Merge the two geometry walkers.** The precise C++ walker and pypdf visitor
+   observe the same content stream. Use precise glyph bounds whenever available and
+   visitor text only as a decoding fallback, instead of calibrating visitor width
+   from cross-column span gaps. This removes the remaining unsound estimator rather
+   than merely reducing how often it runs.
+3. **Add independent layout gates.** Keep hand-labelled table truth, and add page
+   bounding-box/column anchors, header/footer repetition, page-break, image-count,
+   and cross-column order fixtures. Report recall, precision, duplicates, order,
+   and visual similarity separately; never gate only on pypdf-derived text.
+4. **Make the table structure model reproducible.** Either vendor a checked,
+   permissively licensed Table Transformer/SLANet model with checksum and measured
+   CPU/RAM cost, or explicitly document heuristic-only deployment and route
+   high-risk tables to the page-image fallback.
+
+### OCR and resource work
+
+5. Keep one process-wide session per model, but use a dedicated OCR worker pool or
+   process queue so serialized inference cannot block unrelated conversions. Bound
+   image dimensions and decoded bytes before OCR; expose queue wait, model-load,
+   render, inference, and rebuild timings per page.
+6. Calibrate DPI and preprocessing by document class (deskew, contrast, denoise,
+   adaptive threshold, crop). Use confidence distributions and script detection to
+   decide a second pass; preserve low-confidence text with a warning rather than
+   silently replacing it. Evaluate RapidOCR against PaddleOCR PP-OCRv5, EasyOCR,
+   and a CPU-friendly transformer on a labelled multilingual set before switching.
+7. Treat the current font service as a heuristic fallback until a real model is
+   shipped. If font fidelity matters, load and checksum an actual ONNX model,
+   batch line crops, and cache one prediction per style/span rather than running
+   line and word inference for every OCR block. Otherwise disable it in the hot
+   conversion path and use extracted PDF font metadata.
+
+### Fidelity and operations
+
+8. Preserve the existing page-raster fallback for complex positioned PDFs and make
+   the choice explicit in response metadata (`editable` versus `visual`). For
+   customer-facing editable PDF->Office jobs, use Gotenberg/LibreOffice or a
+   commercial fidelity provider behind the same plugin interface; retain ReportLab
+   for concurrent-safe low-cost fallback.
+9. Install `pypdfium2` in every supported runtime image and run the raster suite in
+   CI. Keep the C++ engine path as the preferred renderer, but fail loudly when a
+   host has neither renderer nor OCR capability.
+10. Resolve the font endpoint 204/404 contract, and expose structured error codes
+    for page-cap, OCR-budget, table-confidence, and visual-fallback events.
+11. Run conversion workers in separate processes/containers for hostile or very
+    large files. Apply per-tenant byte/page/time budgets, temporary-directory
+    quotas, cancellation cleanup, and a dead-letter queue. Track p95/p99 latency,
+    peak RSS, CPU saturation, cache hit rate, and output-fidelity distributions.
+
+## Implementation
+
+The following changes are present in the working tree and were validated by the
+focused suites:
+
+- `layout/pipeline.py`, `pypdf_geometry.py`, `glyphs.py`, and paragraph/order
+  helpers now use non-whitespace coverage, column-aware row bands, zero-width
+  whitespace handling, improved paragraph grouping, and figure preservation.
+- `ocr.py` serializes model load/inference safely, supports adaptive Arabic
+  recognition (`CONVERT_OCR_ARABIC_ADAPTIVE`), preserves meaningful symbols such
+  as `$` and `%`, and records whether Arabic was used per call.
+- `ocr/rebuild.py` raises the page cap to 200, separates page-cap and time-budget
+  warnings/metadata, compacts oversized figures instead of dropping them, and
+  embeds unreconstructable pages as honest page images.
+- `result_cache.py` canonicalizes effective options, namespace, and automatic
+  versus explicit resolution mode. `doc_cache.py` reuses readers, geometry,
+  rasters, and engine handles inside one conversion.
+- `routers/convert/jobs.py` and conversion scopes bound admission and shared
+  execution slots; cancellation and deadlines are checked at page/OCR boundaries.
+- `quality/text_metrics.py` and `quality/scorer.py` now expose Unicode-aware,
+  duplicate-aware, order-sensitive metrics and warnings for low precision/order;
+  the IDM retains component metrics for diagnostics.
+- PDF->PNG/JPEG/TIFF exports stream pages; TIFF is multi-frame LZW with DPI
+  metadata. DOCX raster fallback is selected from measured complexity/content
+  evidence rather than blindly reflowing every page.
+- `Dockerfile`, `models/MANIFEST.json`, and `tools/verify_models.py` package and
+  checksum runtime layout/Arabic weights. CORS defaults are explicit and
+  production-safe. `start_convert_gateway.ps1` documents OCR, model, timeout,
+  queue, and cache settings.
+- `tools/audit_conversion.py` provides repeatable end-to-end measurements. New
+  tests cover metrics, OCR resource guards, cache keys, queue admission, streaming
+  exports, model packaging, and table truth.
+
+No component was replaced solely because it was open source. The changes preserve
+the C++/PDFium core and existing plugin interfaces; alternatives are recommended
+only where the evidence shows a fundamental fidelity or reproducibility gap.
+
+## Validation Results
+
+The current state passes the broad focused checks listed in Evidence and runs the
+full gateway suite to completion without a crash or timeout. The multi-column
+order probe improved from 189 inversions to 1, and the OCR/page-cap/resource
+regressions are green. Streaming image export and OOXML validity checks are green.
+
+The result is not yet release-clean:
+
+- W3C exact table recall is 0.867 against a 0.90 floor.
+- Table reading-order/span assertions fail on the same punctuation/header
+  fragments.
+- Unknown-font endpoint status disagrees with its test contract.
+- USGS and multipage-table order/recall remain low despite high pixel similarity.
+- Scan-pack editable text has perfect supplied-token recall but only 0.077
+  precision and 0.143 order similarity; it must not be marketed as accurate OCR
+  without a richer ground truth.
+
+Expected gains from the recommended work are stated as targets, not measured
+claims: table stitching should raise W3C recall above the existing 0.90 gate and
+remove the four known header/punctuation errors; a merged geometry walker should
+keep the 1-inversion result while eliminating the remaining visitor-width risk;
+adaptive OCR already demonstrates approximately 55% lower warm-page inference
+cost on clean Latin scans; process-level isolation should make peak RSS roughly
+linear per worker instead of allowing simultaneous jobs to share one unbounded
+address space. Each target must be re-measured on the same corpus before release.
+
+## Remaining Limitations
+
+1. PDF->editable DOCX/XLSX cannot be pixel-perfect for arbitrary positioned PDF
+   content. The page-image fallback preserves appearance by giving up editability;
+   the reflow path preserves more semantics by accepting layout drift.
+2. The default table detector remains heuristic and the table structure model is
+   not shipped. Merged cells, borderless grids, nested tables, and multi-line
+   headers remain the largest content-structure risk.
+3. RapidOCR is CPU-bound in the measured host, and shared-session locking limits
+   parallel OCR throughput. Whole-document timings vary with cache/model state and
+   Windows scheduling; warm per-page numbers are more stable.
+4. The active venv lacks `pypdfium2`, so renderer-free Linux behavior and several
+   raster tests were skipped. The Docker dependency declaration and local runtime
+   are currently inconsistent until the venv is rebuilt from `pyproject.toml`.
+5. No CUDA provider was available in this audit. GPU utilization, GPU memory, and
+   multi-GPU scaling are therefore unmeasured.
+6. The advertised OpenCLIP font model and FAISS index are metadata placeholders in
+   this checkout. Font predictions are heuristic and should not be used as a
+   fidelity guarantee.
+7. Visual similarity is a grayscale pixel heuristic over at most the compared
+   pages and, for Office outputs in this audit, uses ReportLab rather than a real
+   LibreOffice/Gotenberg render.
+8. The quality score is useful for triage but is not a single acceptance metric.
+   Source text can be corrupt, OCR can add legitimate-looking alternatives, and
+   order/layout require independent annotations.
+9. The repository has a `.git_disabled` metadata directory rather than a normal
+   `.git` checkout, and much of the conversion subsystem is working-tree content.
+   Before/after comparisons in this investigation therefore come from harness
+   snapshots and controlled runs, not a pristine commit diff.
+
+## Appendix A: Reproduction Commands and Evidence Files
+
+Run from `pdf/` with the gateway venv:
+
+```powershell
+gateway\.venv\Scripts\python.exe tools\audit_conversion.py
+gateway\.venv\Scripts\python.exe tools\audit_conversion.py --no-scan
+gateway\.venv\Scripts\python.exe -m pytest gateway\tests\convert -q
+gateway\.venv\Scripts\python.exe -m pytest gateway\tests -q
+```
+
+Primary evidence and implementation references:
+
+- `audit_results.json` - machine-readable audit output (the checked-in file may
+  be targeted to one case; regenerate for the full matrix).
+- `tools/audit_conversion.py` - timing, resource, content, table, and visual
+  measurement harness.
+- `corpus/convert/table_truth.json` and `corpus/convert/baselines.json` - external
+  table labels and suite floors.
+- `gateway/app/services/convert/layout/pipeline.py` - routing, IDM construction,
+  OCR budgeting, and page-image fallback.
+- `gateway/app/services/convert/layout/pypdf_geometry.py` and `glyphs.py` - glyph
+  walkers, line grouping, and geometry evidence.
+- `gateway/app/services/convert/ocr/rebuild.py` and `gateway/app/services/ocr.py` -
+  raster/OCR/rebuild path and Arabic adaptive policy.
+- `gateway/app/services/convert/formatters/docx_formatter.py`,
+  `writers/pdf_from_docx.py`, and `backends/pdf_exporter.py` - Office writers and
+  Gotenberg/ReportLab selection.
+- `gateway/pyproject.toml`, `gateway/Dockerfile`, `gateway/models/MANIFEST.json`,
+  and `gateway/start_convert_gateway.ps1` - dependency, packaging, and runtime
+  configuration evidence.
+
+**Bottom line:** the engine now has honest diagnostics and materially better
+multi-column recovery, but it should be released with explicit fidelity classes,
+the W3C table defect fixed, renderer/OCR dependencies verified in CI, and separate
+content, structure, and visual acceptance gates.

bindings/CMakeLists.txt

@@ -1,6 +1,74 @@
 # Python bindings for PdfEngine using pybind11
 #
 
+# The .pyd ABI tag (cp312 vs cp313) must match the interpreter that runs
+# the FastAPI gateway. Prefer gateway/.venv when it exists so `import pdfengine`
+# works in uvicorn without a second Python install.
+if(WIN32)
+    set(_gateway_python "${CMAKE_SOURCE_DIR}/gateway/.venv/Scripts/python.exe")
+else()
+    set(_gateway_python "${CMAKE_SOURCE_DIR}/gateway/.venv/bin/python")
+endif()
+if(EXISTS "${_gateway_python}")
+    set(Python_EXECUTABLE "${_gateway_python}" CACHE FILEPATH "Python for pybind11" FORCE)
+    set(Python3_EXECUTABLE "${_gateway_python}" CACHE FILEPATH "Python3 for pybind11" FORCE)
+    set(PYTHON_EXECUTABLE "${_gateway_python}" CACHE FILEPATH "Python for pybind11 (legacy)" FORCE)
+    message(STATUS "pybind11: using gateway venv ${_gateway_python}")
+endif()
+
+# Drop stale FindPythonLibsNew cache from a previous configure (it can keep
+# PYTHON_MODULE_EXTENSION=.cp313-*.pyd and python313.lib even after we switch
+# the interpreter to the gateway 3.12 venv).
+unset(PYTHON_MODULE_EXTENSION CACHE)
+unset(PYTHON_LIBRARIES CACHE)
+unset(PYTHON_INCLUDE_DIRS CACHE)
+unset(PYTHON_VERSION CACHE)
+unset(PYTHON_VERSION_MAJOR CACHE)
+unset(PYTHON_VERSION_MINOR CACHE)
+unset(PYTHON_IS_DEBUG CACHE)
+unset(PYTHON_MODULE_PREFIX CACHE)
+unset(PYTHON_MODULE_DEBUG_POSTFIX CACHE)
+
+if(Python_EXECUTABLE)
+    execute_process(
+        COMMAND "${Python_EXECUTABLE}" -c "import sys, sysconfig, pathlib; base=pathlib.Path(sys.base_prefix); ver=f'{sys.version_info.major}{sys.version_info.minor}'; print(sysconfig.get_config_var('EXT_SUFFIX') or ''); print(sysconfig.get_path('include')); print(base / 'libs' / f'python{ver}.lib')"
+        OUTPUT_VARIABLE _pdfengine_py_info
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_QUIET
+    )
+    string(REPLACE "\r" "" _pdfengine_py_info "${_pdfengine_py_info}")
+    string(REPLACE "\n" ";" _pdfengine_py_info_lines "${_pdfengine_py_info}")
+    list(LENGTH _pdfengine_py_info_lines _pdfengine_py_info_len)
+    if(_pdfengine_py_info_len GREATER_EQUAL 3)
+        list(GET _pdfengine_py_info_lines 0 _pdfengine_py_ext)
+        list(GET _pdfengine_py_info_lines 1 _pdfengine_py_inc)
+        list(GET _pdfengine_py_info_lines 2 _pdfengine_py_lib)
+        file(TO_CMAKE_PATH "${_pdfengine_py_ext}" _pdfengine_py_ext)
+        file(TO_CMAKE_PATH "${_pdfengine_py_inc}" _pdfengine_py_inc)
+        file(TO_CMAKE_PATH "${_pdfengine_py_lib}" _pdfengine_py_lib)
+        if(_pdfengine_py_ext)
+            set(PYTHON_MODULE_EXTENSION "${_pdfengine_py_ext}" CACHE INTERNAL "Python extension suffix")
+            message(STATUS "pybind11: extension suffix ${_pdfengine_py_ext}")
+        endif()
+        # vcpkg ships a static python312.lib; linking the .pyd against that
+        # crashes when uvicorn loads it (no python312.dll import). Use the
+        # interpreter's own import lib / headers instead.
+        if(EXISTS "${_pdfengine_py_lib}" AND EXISTS "${_pdfengine_py_inc}/Python.h")
+            set(Python_LIBRARY "${_pdfengine_py_lib}" CACHE FILEPATH "Python import library" FORCE)
+            set(Python_LIBRARIES "${_pdfengine_py_lib}" CACHE FILEPATH "Python import library" FORCE)
+            set(Python_INCLUDE_DIR "${_pdfengine_py_inc}" CACHE PATH "Python headers" FORCE)
+            set(Python3_LIBRARY "${_pdfengine_py_lib}" CACHE FILEPATH "Python3 import library" FORCE)
+            set(Python3_INCLUDE_DIR "${_pdfengine_py_inc}" CACHE PATH "Python3 headers" FORCE)
+            get_filename_component(_pdfengine_py_root "${_pdfengine_py_inc}" DIRECTORY)
+            set(Python_ROOT_DIR "${_pdfengine_py_root}" CACHE PATH "CPython install root" FORCE)
+            message(STATUS "pybind11: CPython lib ${_pdfengine_py_lib}")
+            message(STATUS "pybind11: CPython include ${_pdfengine_py_inc}")
+        endif()
+    endif()
+endif()
+
+set(PYBIND11_FINDPYTHON ON)
+
 find_package(pybind11 CONFIG REQUIRED)
 
 # Declare the python module target. We name the target pdfengine_py to avoid
@@ -12,9 +80,23 @@ set_target_properties(pdfengine_py PROPERTIES
     OUTPUT_NAME "pdfengine"
     ARCHIVE_OUTPUT_NAME "pdfengine_py_import"
 )
+if(_pdfengine_py_ext)
+    set_target_properties(pdfengine_py PROPERTIES SUFFIX "${_pdfengine_py_ext}")
+endif()
 
 target_link_libraries(pdfengine_py PRIVATE pdfengine::pdfengine)
 
+# OBJECT-library usage requirements are not always enough on MSVC for the
+# python module (it also consumes $<TARGET_OBJECTS:pdfengine>). Repeat the
+# libraries the engine objects reference so the .pyd link line is complete.
+target_link_libraries(pdfengine_py PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
+if(PDFENGINE_WITH_PDFIUM)
+    target_link_libraries(pdfengine_py PRIVATE pdfium::pdfium)
+endif()
+if(_pdfengine_py_lib AND EXISTS "${_pdfengine_py_lib}")
+    target_link_libraries(pdfengine_py PRIVATE "${_pdfengine_py_lib}")
+endif()
+
 target_include_directories(pdfengine_py PRIVATE 
     "${CMAKE_SOURCE_DIR}/engine/src"
 )
@@ -23,6 +105,28 @@ if(MSVC)
     target_link_options(pdfengine_py PRIVATE "/FORCE:MULTIPLE")
 endif()
 
+# vcpkg's find_package(Python) wrapper still injects its static python312.lib.
+# A pybind module must link only the host interpreter's import lib (python312.dll).
+foreach(_py_tgt IN ITEMS Python::Module Python::Python Python3::Module Python3::Python)
+    if(TARGET ${_py_tgt} AND _pdfengine_py_lib)
+        set_property(TARGET ${_py_tgt} PROPERTY INTERFACE_LINK_LIBRARIES "${_pdfengine_py_lib}")
+        set_property(TARGET ${_py_tgt} PROPERTY IMPORTED_LOCATION "${_pdfengine_py_lib}")
+        set_property(TARGET ${_py_tgt} PROPERTY IMPORTED_IMPLIB "${_pdfengine_py_lib}")
+    endif()
+endforeach()
+get_target_property(_pdfengine_py_link pdfengine_py LINK_LIBRARIES)
+if(_pdfengine_py_link)
+    set(_pdfengine_py_kept "")
+    foreach(_lib IN LISTS _pdfengine_py_link)
+        if(_lib MATCHES "vcpkg_installed" AND _lib MATCHES "[Pp]ython")
+            message(STATUS "pybind11: dropping vcpkg CPython ${_lib}")
+            continue()
+        endif()
+        list(APPEND _pdfengine_py_kept "${_lib}")
+    endforeach()
+    set_property(TARGET pdfengine_py PROPERTY LINK_LIBRARIES "${_pdfengine_py_kept}")
+endif()
+
 # Set warnings and sanitizers for the bindings module
 pdfengine_set_warnings(pdfengine_py)
 pdfengine_enable_sanitizers(pdfengine_py)

bindings/python/pdfengine_py.cpp

@@ -1,7 +1,8 @@
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
 #include <pdfengine/pdf_document.hpp>
 #include <pdfengine/pdf_engine.hpp>
+#include <spdlog/spdlog.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
 
 namespace py = pybind11;
 
@@ -9,28 +10,27 @@ namespace {
 
 void throw_on_error(pdfengine::EngineError err) {
     switch (err) {
-        case pdfengine::EngineError::FileNotFound:
-            PyErr_SetString(PyExc_FileNotFoundError, "PDF file not found");
-            throw py::error_already_set();
-        case pdfengine::EngineError::InvalidFormat:
-            throw py::value_error("Invalid PDF format");
-        case pdfengine::EngineError::PasswordRequired:
-            throw py::value_error("Password required to open this PDF");
-        case pdfengine::EngineError::InvalidPassword:
-            throw py::value_error("Invalid password provided for this PDF");
-        case pdfengine::EngineError::PageOutOfBounds:
-            throw py::index_error("Page index out of bounds");
-        case pdfengine::EngineError::RenderFailed:
-            throw std::runtime_error("Failed to render PDF page");
-        case pdfengine::EngineError::WriteFailed:
-            throw std::runtime_error("Failed to write PDF data");
-        default:
-            throw std::runtime_error("Unknown PDF engine error");
+    case pdfengine::EngineError::FileNotFound:
+        PyErr_SetString(PyExc_FileNotFoundError, "PDF file not found");
+        throw py::error_already_set();
+    case pdfengine::EngineError::InvalidFormat:
+        throw py::value_error("Invalid PDF format");
+    case pdfengine::EngineError::PasswordRequired:
+        throw py::value_error("Password required to open this PDF");
+    case pdfengine::EngineError::InvalidPassword:
+        throw py::value_error("Invalid password provided for this PDF");
+    case pdfengine::EngineError::PageOutOfBounds:
+        throw py::index_error("Page index out of bounds");
+    case pdfengine::EngineError::RenderFailed:
+        throw std::runtime_error("Failed to render PDF page");
+    case pdfengine::EngineError::WriteFailed:
+        throw std::runtime_error("Failed to write PDF data");
+    default:
+        throw std::runtime_error("Unknown PDF engine error");
     }
 }
 
-template<typename T>
-T get_or_throw(std::expected<T, pdfengine::EngineError>&& res) {
+template <typename T> T get_or_throw(std::expected<T, pdfengine::EngineError>&& res) {
     if (!res.has_value()) {
         throw_on_error(res.error());
     }
@@ -43,16 +43,17 @@ void get_or_throw(std::expected<void, pdfengine::EngineError>&& res) {
     }
 }
 
-} 
+} // namespace
 
-#include <pdfengine/content_object.hpp>
-#include <qpdf/qpdf_extractor.hpp>
-#include <qpdf/qpdf_writer.hpp>
+#include <parser/content_builder.hpp>
 #include <parser/lexer.hpp>
 #include <parser/parser.hpp>
-#include <parser/content_builder.hpp>
-#include <serializer/content_serializer.hpp>
+#include <pdfengine/content_object.hpp>
+#include <pdfengine/ocr/ocr_coordinator.hpp>
+#include <qpdf/qpdf_extractor.hpp>
+#include <qpdf/qpdf_writer.hpp>
 #include <serializer/ast_serializer.hpp>
+#include <serializer/content_serializer.hpp>
 
 static constexpr double kTjSpaceKern = -500.0;
 
@@ -66,14 +67,14 @@ public:
         if (!stream.has_value()) {
             throw std::runtime_error("Failed to extract page stream");
         }
-        
+
         pdfengine::Lexer lexer(stream->decodedContent);
         auto tokens = lexer.tokenize();
         pdfengine::ContentParser parser(tokens);
         pdfengine::ContentBuilder builder;
-        
+
         auto objects = builder.build(parser.parse());
-        
+
         py::list result;
         for (const auto& obj : objects) {
             if (obj->getType() == pdfengine::ContentObjectType::Text) {
@@ -82,7 +83,7 @@ public:
                 d["text"] = py::bytes(textObj->text);
                 d["fontName"] = textObj->fontName;
                 d["fontSize"] = textObj->fontSize;
-                
+
                 py::list tm;
                 for (int i = 0; i < 6; ++i) {
                     tm.append(textObj->tm[i]);
@@ -94,24 +95,28 @@ public:
         return result;
     }
 
-    bool replace_text_object(int page_index, int object_index, const py::bytes& new_text_bytes, const std::string& dest_path) {
+    bool replace_text_object(int page_index, int object_index, const py::bytes& new_text_bytes,
+                             const std::string& dest_path) {
         std::string new_text = new_text_bytes;
         pdfengine::qpdf_layer::QpdfExtractor extractor;
         auto stream = extractor.extractPageStream(filepath_, page_index);
-        if (!stream.has_value()) return false;
-        
+        if (!stream.has_value())
+            return false;
+
         pdfengine::Lexer lexer(stream->decodedContent);
         auto tokens = lexer.tokenize();
         pdfengine::ContentParser parser(tokens);
         auto operations = parser.parse();
-        
+
         int textCount = 0;
         bool modified = false;
         for (auto& op : operations) {
             if (op.op == "Tj" || op.op == "'") {
-                if (op.operands.empty()) continue;
+                if (op.operands.empty())
+                    continue;
                 auto& strNode = op.operands.back();
-                if (strNode->type == pdfengine::AstNodeType::String || strNode->type == pdfengine::AstNodeType::HexString) {
+                if (strNode->type == pdfengine::AstNodeType::String ||
+                    strNode->type == pdfengine::AstNodeType::HexString) {
                     if (textCount == object_index) {
                         strNode->type = pdfengine::AstNodeType::String;
                         strNode->stringValue = new_text;
@@ -121,7 +126,8 @@ public:
                     textCount++;
                 }
             } else if (op.op == "TJ") {
-                if (op.operands.empty()) continue;
+                if (op.operands.empty())
+                    continue;
                 auto& arrNode = op.operands.back();
                 if (arrNode->type == pdfengine::AstNodeType::Array) {
                     std::string combinedText;
@@ -129,9 +135,11 @@ public:
                         if (item->type == pdfengine::AstNodeType::String) {
                             combinedText += item->stringValue;
                         } else if (item->type == pdfengine::AstNodeType::HexString) {
-                            combinedText += std::string(item->bytesValue.begin(), item->bytesValue.end());
+                            combinedText +=
+                                std::string(item->bytesValue.begin(), item->bytesValue.end());
                         } else if (item->type == pdfengine::AstNodeType::Number) {
-                            if (item->numberValue < kTjSpaceKern) combinedText += " ";
+                            if (item->numberValue < kTjSpaceKern)
+                                combinedText += " ";
                         }
                     }
                     if (!combinedText.empty()) {
@@ -139,17 +147,22 @@ public:
                             bool redistributed = false;
                             if (new_text.size() == combinedText.size()) {
                                 std::vector<std::pair<pdfengine::AstNode*, std::string>> assign;
-                                size_t pos = 0; bool ok = true;
+                                size_t pos = 0;
+                                bool ok = true;
                                 for (const auto& item : arrNode->arrayItems) {
                                     if (item->type == pdfengine::AstNodeType::String ||
                                         item->type == pdfengine::AstNodeType::HexString) {
                                         size_t L = (item->type == pdfengine::AstNodeType::HexString)
-                                            ? item->bytesValue.size() : item->stringValue.size();
+                                                       ? item->bytesValue.size()
+                                                       : item->stringValue.size();
                                         assign.emplace_back(item.get(), new_text.substr(pos, L));
                                         pos += L;
                                     } else if (item->type == pdfengine::AstNodeType::Number &&
                                                item->numberValue < kTjSpaceKern) {
-                                        if (pos >= new_text.size() || new_text[pos] != ' ') { ok = false; break; }
+                                        if (pos >= new_text.size() || new_text[pos] != ' ') {
+                                            ok = false;
+                                            break;
+                                        }
                                         pos += 1;
                                     }
                                 }
@@ -163,7 +176,8 @@ public:
                             }
                             if (!redistributed) {
                                 arrNode->arrayItems.clear();
-                                auto newStrNode = std::make_shared<pdfengine::AstNode>(pdfengine::AstNodeType::String);
+                                auto newStrNode = std::make_shared<pdfengine::AstNode>(
+                                    pdfengine::AstNodeType::String);
                                 newStrNode->stringValue = new_text;
                                 arrNode->arrayItems.push_back(std::move(newStrNode));
                             }
@@ -175,12 +189,13 @@ public:
                 }
             }
         }
-        
-        if (!modified) return false;
-        
+
+        if (!modified)
+            return false;
+
         pdfengine::AstSerializer astSerializer;
         std::string newRawStream = astSerializer.serialize(operations);
-        
+
         pdfengine::qpdf_layer::QpdfWriter writer;
         auto res = writer.replacePageStreamAndSave(filepath_, dest_path, page_index, newRawStream);
         return res.has_value();
@@ -196,12 +211,87 @@ PYBIND11_MODULE(pdfengine, m) {
     py::class_<StreamEditor>(m, "StreamEditor")
         .def(py::init<const std::string&>(), py::arg("filepath"))
         .def("extract_text_objects", &StreamEditor::extract_text_objects, py::arg("page_index"))
-        .def("replace_text_object", &StreamEditor::replace_text_object, py::arg("page_index"), py::arg("object_index"), py::arg("new_text"), py::arg("dest_path"));
+        .def("replace_text_object", &StreamEditor::replace_text_object, py::arg("page_index"),
+             py::arg("object_index"), py::arg("new_text"), py::arg("dest_path"));
 
     m.def("engine_version", &pdfengine::engineVersion, "Get the engine version string");
     m.def("engine_build_info", &pdfengine::engineBuildInfo, "Get the engine build info string");
-    m.def("engine_has_pdfium", &pdfengine::engineHasPdfium, "Check if the engine was built with PDFium support");
-    m.def("engine_has_skia", &pdfengine::engineHasSkia, "Check if the engine was built with Skia support");
+    m.def("engine_has_pdfium", &pdfengine::engineHasPdfium,
+          "Check if the engine was built with PDFium support");
+    m.def("engine_has_skia", &pdfengine::engineHasSkia,
+          "Check if the engine was built with Skia support");
+
+    m.def(
+        "protect_pdf",
+        [](const py::bytes& input_bytes,
+           const std::string& user_password,
+           const std::string& owner_password,
+           const py::dict& perms_dict) {
+            if (user_password.empty()) {
+                throw py::value_error("User password cannot be empty");
+            }
+
+            std::string_view sv = input_bytes;
+            std::vector<uint8_t> data(sv.begin(), sv.end());
+
+            pdfengine::qpdf_layer::PdfEncryptionOptions opts;
+            opts.userPassword = user_password;
+            opts.ownerPassword = owner_password.empty() ? user_password : owner_password;
+
+            auto get_bool = [&](const char* key, bool dflt) {
+                if (perms_dict.contains(key) && !perms_dict[key].is_none()) {
+                    try { return perms_dict[key].cast<bool>(); } catch (...) {}
+                }
+                return dflt;
+            };
+
+            opts.allowPrint = get_bool("canPrint", true);
+            opts.allowPrintHighRes = get_bool("canPrintHighRes", true);
+            opts.allowModify = get_bool("canModify", true);
+            opts.allowCopy = get_bool("canCopy", true);
+            opts.allowAnnotate = get_bool("canAnnotate", true);
+            opts.allowFillForms = get_bool("canFillForms", true);
+            opts.allowAccessibility = get_bool("canExtractForAccessibility", true);
+            opts.allowAssemble = get_bool("canAssemble", true);
+            opts.keyLengthBits = 256;
+
+            pdfengine::qpdf_layer::QpdfWriter writer;
+            auto res = writer.encryptPdf(data, opts);
+
+            if (!res.has_value()) {
+                throw std::runtime_error("Encryption failed");
+            }
+
+            const auto& out_bytes = res.value();
+            return py::bytes(reinterpret_cast<const char*>(out_bytes.data()), out_bytes.size());
+        },
+        py::arg("input_bytes"),
+        py::arg("user_password"),
+        py::arg("owner_password") = "",
+        py::arg("permissions") = py::dict(),
+        "Encrypt raw PDF bytes with AES-256 and custom permissions"
+    );
+
+    m.def(
+        "unlock_pdf",
+        [](const py::bytes& input_bytes, const std::string& password) {
+            std::string_view sv = input_bytes;
+            std::vector<uint8_t> data(sv.begin(), sv.end());
+
+            pdfengine::qpdf_layer::QpdfWriter writer;
+            auto res = writer.unlockPdf(data, password);
+
+            if (!res.has_value()) {
+                throw std::runtime_error("Unlock failed");
+            }
+
+            const auto& out_bytes = res.value();
+            return py::bytes(reinterpret_cast<const char*>(out_bytes.data()), out_bytes.size());
+        },
+        py::arg("input_bytes"),
+        py::arg("password") = "",
+        "Decrypt raw PDF bytes with QPDF"
+    );
 
     py::class_<pdfengine::Point2D>(m, "Point2D")
         .def(py::init<double, double>(), py::arg("x") = 0.0, py::arg("y") = 0.0)
@@ -216,7 +306,8 @@ PYBIND11_MODULE(pdfengine, m) {
         .def_readwrite("x", &pdfengine::DevicePoint::x)
         .def_readwrite("y", &pdfengine::DevicePoint::y)
         .def("__repr__", [](const pdfengine::DevicePoint& self) {
-            return "DevicePoint(x=" + std::to_string(self.x) + ", y=" + std::to_string(self.y) + ")";
+            return "DevicePoint(x=" + std::to_string(self.x) + ", y=" + std::to_string(self.y) +
+                   ")";
         });
 
     py::class_<pdfengine::DocumentMetadata>(m, "DocumentMetadata")
@@ -241,7 +332,8 @@ PYBIND11_MODULE(pdfengine, m) {
         .def_readonly("can_copy", &pdfengine::DocumentPermissions::canCopy)
         .def_readonly("can_annotate", &pdfengine::DocumentPermissions::canAnnotate)
         .def_readonly("can_fill_forms", &pdfengine::DocumentPermissions::canFillForms)
-        .def_readonly("can_extract_for_accessibility", &pdfengine::DocumentPermissions::canExtractForAccessibility)
+        .def_readonly("can_extract_for_accessibility",
+                      &pdfengine::DocumentPermissions::canExtractForAccessibility)
         .def_readonly("can_assemble", &pdfengine::DocumentPermissions::canAssemble);
 
     py::class_<pdfengine::PageImage>(m, "PageImage")
@@ -272,7 +364,8 @@ PYBIND11_MODULE(pdfengine, m) {
         .def_readonly("descent", &pdfengine::FontInfo::descent)
         .def_readonly("cap_height", &pdfengine::FontInfo::capHeight)
         .def("__repr__", [](const pdfengine::FontInfo& self) {
-            return "FontInfo(font_name='" + self.fontName + "', type='" + self.type + "', is_embedded=" + (self.isEmbedded ? "True" : "False") + ")";
+            return "FontInfo(font_name='" + self.fontName + "', type='" + self.type +
+                   "', is_embedded=" + (self.isEmbedded ? "True" : "False") + ")";
         });
 
     py::class_<pdfengine::Glyph>(m, "Glyph")
@@ -297,6 +390,8 @@ PYBIND11_MODULE(pdfengine, m) {
         .def_readonly("font_size", &pdfengine::TextRun::fontSize)
         .def_readonly("internal_font_id", &pdfengine::TextRun::internalFontId)
         .def_readonly("is_embedded", &pdfengine::TextRun::isEmbedded)
+        .def_readonly("is_embedded_font", &pdfengine::TextRun::isEmbeddedFont)
+        .def_readonly("is_predicted_font", &pdfengine::TextRun::isPredictedFont)
         .def_readonly("type", &pdfengine::TextRun::type)
         .def_readonly("glyphs", &pdfengine::TextRun::glyphs)
         .def_readonly("x", &pdfengine::TextRun::x)
@@ -366,169 +461,306 @@ PYBIND11_MODULE(pdfengine, m) {
     py::class_<pdfengine::PdfPage, std::shared_ptr<pdfengine::PdfPage>>(m, "PdfPage")
         .def_property_readonly("width", &pdfengine::PdfPage::width)
         .def_property_readonly("height", &pdfengine::PdfPage::height)
-        .def("render", [](const pdfengine::PdfPage& self, int dpi) {
-            return get_or_throw(self.render(dpi));
-        }, py::arg("dpi") = 96)
-        .def("render_region_raw", [](const pdfengine::PdfPage& self, int dpi, double y_top_pt, double height_pt) {
-            auto img = get_or_throw(self.renderRegionRaw(dpi, y_top_pt, height_pt));
-            return py::make_tuple(img.width, img.height,
-                                  py::bytes(reinterpret_cast<const char*>(img.data.data()), img.data.size()));
-        }, py::arg("dpi"), py::arg("y_top_pt"), py::arg("height_pt") = 0.0)
-        .def("render_tile", [](const pdfengine::PdfPage& self, int dpi, double xPt, double yPt, double wPt, double hPt) {
-            auto img = get_or_throw(self.renderTile(dpi, xPt, yPt, wPt, hPt));
-            return py::make_tuple(img.width, img.height,
-                                  py::bytes(reinterpret_cast<const char*>(img.data.data()), img.data.size()));
-        }, py::arg("dpi"), py::arg("xPt"), py::arg("yPt"), py::arg("wPt"), py::arg("hPt"))
-        .def("extract_document_model", [](const pdfengine::PdfPage& self) {
-            return get_or_throw(self.extractDocumentModel());
-        })
-        .def("extract_text", [](const pdfengine::PdfPage& self) {
-            return get_or_throw(self.extractText());
-        })
-        .def("extract_annotations_text", [](const pdfengine::PdfPage& self) {
-            return get_or_throw(self.extractAnnotationsText());
-        })
-        .def("extract_annotations", [](const pdfengine::PdfPage& self) {
-            return get_or_throw(self.extractAnnotations());
-        })
-        .def("extract_text_with_bounds", [](const pdfengine::PdfPage& self) {
-            auto res = get_or_throw(self.extractTextWithBounds());
-            py::list py_list;
-            for (const auto& glyph : res) {
+        .def(
+            "render",
+            [](const pdfengine::PdfPage& self, int dpi) { return get_or_throw(self.render(dpi)); },
+            py::arg("dpi") = 96)
+        .def(
+            "render_region_raw",
+            [](const pdfengine::PdfPage& self, int dpi, double y_top_pt, double height_pt) {
+                auto img = get_or_throw(self.renderRegionRaw(dpi, y_top_pt, height_pt));
+                return py::make_tuple(
+                    img.width, img.height,
+                    py::bytes(reinterpret_cast<const char*>(img.data.data()), img.data.size()));
+            },
+            py::arg("dpi"), py::arg("y_top_pt"), py::arg("height_pt") = 0.0)
+        .def(
+            "render_tile",
+            [](const pdfengine::PdfPage& self, int dpi, double xPt, double yPt, double wPt,
+               double hPt) {
+                auto img = get_or_throw(self.renderTile(dpi, xPt, yPt, wPt, hPt));
+                return py::make_tuple(
+                    img.width, img.height,
+                    py::bytes(reinterpret_cast<const char*>(img.data.data()), img.data.size()));
+            },
+            py::arg("dpi"), py::arg("xPt"), py::arg("yPt"), py::arg("wPt"), py::arg("hPt"))
+        .def("extract_document_model",
+             [](const pdfengine::PdfPage& self) {
+                 return get_or_throw(self.extractDocumentModel());
+             })
+        .def("extract_text",
+             [](const pdfengine::PdfPage& self) { return get_or_throw(self.extractText()); })
+        .def("extract_annotations_text",
+             [](const pdfengine::PdfPage& self) {
+                 return get_or_throw(self.extractAnnotationsText());
+             })
+        .def("extract_annotations",
+             [](const pdfengine::PdfPage& self) { return get_or_throw(self.extractAnnotations()); })
+        .def("extract_text_with_bounds",
+             [](const pdfengine::PdfPage& self) {
+                 auto res = get_or_throw(self.extractTextWithBounds());
+                 py::list py_list;
+                 for (const auto& glyph : res) {
+                     py::dict d;
+                     d["text"] = glyph.text;
+                     d["x"] = glyph.x;
+                     d["y"] = glyph.y;
+                     d["w"] = glyph.w;
+                     d["h"] = glyph.h;
+                     d["fontSize"] = glyph.fontSize;
+                     py_list.append(d);
+                 }
+                 return py_list;
+             })
+        .def("ordered_glyphs",
+             [](const pdfengine::PdfPage& self) {
+                 auto res = get_or_throw(self.orderedGlyphs());
+                 py::list py_list;
+                 for (const auto& g : res) {
+                     py::dict d;
+                     d["text"] = g.text;
+                     d["x"] = g.x;
+                     d["y"] = g.y;
+                     d["w"] = g.w;
+                     d["h"] = g.h;
+                     d["fontSize"] = g.fontSize;
+                     py_list.append(d);
+                 }
+                 return py_list;
+             })
+        .def(
+            "hit_glyph",
+            [](const pdfengine::PdfPage& self, double x, double y) {
+                auto hit = get_or_throw(self.hitGlyph(x, y));
                 py::dict d;
-                d["text"] = glyph.text;
-                d["x"] = glyph.x;
-                d["y"] = glyph.y;
-                d["w"] = glyph.w;
-                d["h"] = glyph.h;
-                d["fontSize"] = glyph.fontSize;
-                py_list.append(d);
-            }
-            return py_list;
-        })
-        .def("ordered_glyphs", [](const pdfengine::PdfPage& self) {
-            auto res = get_or_throw(self.orderedGlyphs());
-            py::list py_list;
-            for (const auto& g : res) {
+                d["glyphIndex"] = hit.glyphIndex;
+                d["caret"] = hit.caret;
+                d["line"] = hit.line;
+                return d;
+            },
+            py::arg("x"), py::arg("y"))
+        .def(
+            "select_range",
+            [](const pdfengine::PdfPage& self, double ax, double ay, double bx, double by) {
+                auto sel = get_or_throw(self.selectRange(ax, ay, bx, by));
                 py::dict d;
-                d["text"] = g.text; d["x"] = g.x; d["y"] = g.y;
-                d["w"] = g.w; d["h"] = g.h; d["fontSize"] = g.fontSize;
-                py_list.append(d);
-            }
-            return py_list;
-        })
-        .def("hit_glyph", [](const pdfengine::PdfPage& self, double x, double y) {
-            auto hit = get_or_throw(self.hitGlyph(x, y));
-            py::dict d;
-            d["glyphIndex"] = hit.glyphIndex;
-            d["caret"] = hit.caret;
-            d["line"] = hit.line;
-            return d;
-        }, py::arg("x"), py::arg("y"))
-        .def("select_range", [](const pdfengine::PdfPage& self, double ax, double ay, double bx, double by) {
-            auto sel = get_or_throw(self.selectRange(ax, ay, bx, by));
-            py::dict d;
-            d["startGlyph"] = sel.startGlyph;
-            d["endGlyph"] = sel.endGlyph;
-            d["text"] = sel.text;
-            py::list rects;
-            for (const auto& r : sel.rects) {
-                py::dict rd;
-                rd["x"] = r.x; rd["y"] = r.y; rd["w"] = r.w; rd["h"] = r.h;
-                rects.append(rd);
-            }
-            d["rects"] = rects;
-            return d;
-        }, py::arg("ax"), py::arg("ay"), py::arg("bx"), py::arg("by"))
-        .def("get_fonts", [](const pdfengine::PdfPage& self) {
-            return get_or_throw(self.getFonts());
-        })
-        .def("get_glyph_width", [](const pdfengine::PdfPage& self, const std::string& fontName, uint32_t charcode, double fontSize) {
-            return get_or_throw(self.getGlyphWidth(fontName, charcode, fontSize));
-        }, py::arg("font_name"), py::arg("charcode"), py::arg("font_size"))
-        .def("page_to_device", &pdfengine::PdfPage::pageToDevice, 
-             py::arg("page_point"), py::arg("device_width"), py::arg("device_height"), py::arg("rotate") = 0)
-        .def("device_to_page", &pdfengine::PdfPage::deviceToPage, 
-             py::arg("device_point"), py::arg("device_width"), py::arg("device_height"), py::arg("rotate") = 0)
-        .def("extract_display_list", [](const pdfengine::PdfPage& self) {
-            return get_or_throw(self.extractDisplayListJson());
-        })
-        .def("extract_image_xobject", [](const pdfengine::PdfPage& self, const std::string& name) {
-            auto res = get_or_throw(self.extractImageXObject(name));
-            return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
-        }, py::arg("name"));
+                d["startGlyph"] = sel.startGlyph;
+                d["endGlyph"] = sel.endGlyph;
+                d["text"] = sel.text;
+                py::list rects;
+                for (const auto& r : sel.rects) {
+                    py::dict rd;
+                    rd["x"] = r.x;
+                    rd["y"] = r.y;
+                    rd["w"] = r.w;
+                    rd["h"] = r.h;
+                    rects.append(rd);
+                }
+                d["rects"] = rects;
+                return d;
+            },
+            py::arg("ax"), py::arg("ay"), py::arg("bx"), py::arg("by"))
+        .def("get_fonts",
+             [](const pdfengine::PdfPage& self) { return get_or_throw(self.getFonts()); })
+        .def(
+            "get_glyph_width",
+            [](const pdfengine::PdfPage& self, const std::string& fontName, uint32_t charcode,
+               double fontSize) {
+                return get_or_throw(self.getGlyphWidth(fontName, charcode, fontSize));
+            },
+            py::arg("font_name"), py::arg("charcode"), py::arg("font_size"))
+        .def("page_to_device", &pdfengine::PdfPage::pageToDevice, py::arg("page_point"),
+             py::arg("device_width"), py::arg("device_height"), py::arg("rotate") = 0)
+        .def("device_to_page", &pdfengine::PdfPage::deviceToPage, py::arg("device_point"),
+             py::arg("device_width"), py::arg("device_height"), py::arg("rotate") = 0)
+        .def("extract_display_list",
+             [](const pdfengine::PdfPage& self) {
+                 return get_or_throw(self.extractDisplayListJson());
+             })
+        .def(
+            "extract_image_xobject",
+            [](const pdfengine::PdfPage& self, const std::string& name) {
+                auto res = get_or_throw(self.extractImageXObject(name));
+                return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
+            },
+            py::arg("name"));
 
     py::class_<pdfengine::PdfDocument, std::shared_ptr<pdfengine::PdfDocument>>(m, "PdfDocument")
-        .def_static("load_from_file", [](const std::string& path, const std::string& password) {
-            return get_or_throw(pdfengine::PdfDocument::loadFromFile(path, password));
-        }, py::arg("path"), py::arg("password") = "")
-        .def_static("load_from_memory", [](const py::bytes& bytes, const std::string& password) {
-            std::string_view sv = bytes;
-            std::vector<uint8_t> data(sv.begin(), sv.end());
-            return get_or_throw(pdfengine::PdfDocument::loadFromMemory(data, password));
-        }, py::arg("data"), py::arg("password") = "")
+        .def_static(
+            "load_from_file",
+            [](const std::string& path, const std::string& password) {
+                return get_or_throw(pdfengine::PdfDocument::loadFromFile(path, password));
+            },
+            py::arg("path"), py::arg("password") = "")
+        .def_static(
+            "load_from_memory",
+            [](const py::bytes& bytes, const std::string& password) {
+                std::string_view sv = bytes;
+                std::vector<uint8_t> data(sv.begin(), sv.end());
+                return get_or_throw(pdfengine::PdfDocument::loadFromMemory(data, password));
+            },
+            py::arg("data"), py::arg("password") = "")
         .def_property_readonly("page_count", &pdfengine::PdfDocument::pageCount)
         .def_property_readonly("metadata", &pdfengine::PdfDocument::metadata)
         .def_property_readonly("permissions", &pdfengine::PdfDocument::permissions)
-        .def("extract_outline", [](const pdfengine::PdfDocument& self) {
-            auto res = get_or_throw(self.extractOutline());
-            py::list out;
-            for (const auto& item : res) {
-                py::dict d;
-                d["title"] = item.title;
-                d["pageIndex"] = item.pageIndex;
-                d["level"] = item.level;
-                out.append(d);
-            }
-            return out;
-        })
-        .def("get_page", [](pdfengine::PdfDocument& self, int pageIndex) {
-            return get_or_throw(self.getPage(pageIndex));
-        }, py::arg("page_index"))
-        .def("get_fonts", [](const pdfengine::PdfDocument& self, int start_page, int end_page) {
-            return get_or_throw(self.getFonts(start_page, end_page));
-        }, py::arg("start_page") = 0, py::arg("end_page") = -1)
-        .def("get_font_data", [](const pdfengine::PdfDocument& self, const std::string& internal_font_id) {
-            auto res = self.getFontData(internal_font_id);
-            if (!res || res->empty()) {
-                return py::bytes();
-            }
-            return py::bytes(reinterpret_cast<const char*>(res->data()), res->size());
-        }, py::arg("internal_font_id"))
-        .def("get_reconstructed_font_data", [](pdfengine::PdfDocument& self, const std::string& internal_font_id) {
-            auto res = self.getReconstructedFontData(internal_font_id);
-            if (!res || res->empty()) {
-                return py::bytes();
-            }
-            return py::bytes(reinterpret_cast<const char*>(res->data()), res->size());
-        }, py::arg("internal_font_id"))
-        .def("apply_edits", [](pdfengine::PdfDocument& self, const std::string& editsJson) {
-            auto regions = get_or_throw(self.applyEdits(editsJson));
-            py::list py_regions;
-            for (const auto& r : regions) {
-                py::dict d;
-                d["pageIndex"] = r.pageIndex;
-                d["x"] = r.x;
-                d["y"] = r.y;
-                d["width"] = r.width;
-                d["height"] = r.height;
-                py_regions.append(d);
-            }
-            return py_regions;
-        }, py::arg("edits_json"))
-        .def("last_reflow_layout", [](const pdfengine::PdfDocument& self) {
-            return self.lastReflowLayout();
-        })
-        .def("save_incremental", [](const pdfengine::PdfDocument& self) {
-            std::vector<uint8_t> res = get_or_throw(self.saveIncremental());
-            return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
-        })
-        .def("save_full", [](const pdfengine::PdfDocument& self) {
-            std::vector<uint8_t> res = get_or_throw(self.saveFull());
-            return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
-        })
+        .def("extract_outline",
+             [](const pdfengine::PdfDocument& self) {
+                 auto res = get_or_throw(self.extractOutline());
+                 py::list out;
+                 for (const auto& item : res) {
+                     py::dict d;
+                     d["title"] = item.title;
+                     d["pageIndex"] = item.pageIndex;
+                     d["level"] = item.level;
+                     out.append(d);
+                 }
+                 return out;
+             })
+        .def(
+            "get_page",
+            [](pdfengine::PdfDocument& self, int pageIndex) {
+                return get_or_throw(self.getPage(pageIndex));
+            },
+            py::arg("page_index"))
+        .def(
+            "get_fonts",
+            [](const pdfengine::PdfDocument& self, int start_page, int end_page) {
+                return get_or_throw(self.getFonts(start_page, end_page));
+            },
+            py::arg("start_page") = 0, py::arg("end_page") = -1)
+        .def(
+            "get_font_data",
+            [](const pdfengine::PdfDocument& self, const std::string& internal_font_id) {
+                auto res = self.getFontData(internal_font_id);
+                if (!res || res->empty()) {
+                    return py::bytes();
+                }
+                return py::bytes(reinterpret_cast<const char*>(res->data()), res->size());
+            },
+            py::arg("internal_font_id"))
+        .def(
+            "get_reconstructed_font_data",
+            [](pdfengine::PdfDocument& self, const std::string& internal_font_id) {
+                auto res = self.getReconstructedFontData(internal_font_id);
+                if (!res || res->empty()) {
+                    return py::bytes();
+                }
+                return py::bytes(reinterpret_cast<const char*>(res->data()), res->size());
+            },
+            py::arg("internal_font_id"))
+        .def(
+            "apply_edits",
+            [](pdfengine::PdfDocument& self, const std::string& editsJson) {
+                auto regions = get_or_throw(self.applyEdits(editsJson));
+                py::list py_regions;
+                for (const auto& r : regions) {
+                    py::dict d;
+                    d["pageIndex"] = r.pageIndex;
+                    d["x"] = r.x;
+                    d["y"] = r.y;
+                    d["width"] = r.width;
+                    d["height"] = r.height;
+                    py_regions.append(d);
+                }
+                return py_regions;
+            },
+            py::arg("edits_json"))
+        .def("last_reflow_layout",
+             [](const pdfengine::PdfDocument& self) { return self.lastReflowLayout(); })
+        .def("save_incremental",
+             [](const pdfengine::PdfDocument& self) {
+                 std::vector<uint8_t> res = get_or_throw(self.saveIncremental());
+                 return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
+             })
+        .def("save_full",
+             [](const pdfengine::PdfDocument& self) {
+                 std::vector<uint8_t> res = get_or_throw(self.saveFull());
+                 return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
+             })
         .def("save_full_for_export", [](const pdfengine::PdfDocument& self) {
             std::vector<uint8_t> res = get_or_throw(self.saveFullForExport());
             return py::bytes(reinterpret_cast<const char*>(res.data()), res.size());
         });
+
+    py::class_<pdfengine::ocr::OCRCoordinator>(m, "OCRCoordinator")
+        .def(py::init<>())
+        .def(
+            "process_document",
+            [](const pdfengine::ocr::OCRCoordinator& self, int pageIndex, double imgW, double imgH,
+               double pdfW, double pdfH, const py::list& lines_list) {
+                std::vector<pdfengine::document::RawOCRLine> cpp_lines;
+
+                auto get_str_safe = [](py::dict d, const char* key, const std::string& fallback = "") -> std::string {
+                    if (d.contains(key) && !d[key].is_none()) {
+                        try { return d[key].cast<std::string>(); } catch (...) {}
+                    }
+                    return fallback;
+                };
+                auto get_double_safe = [](py::dict d, const char* key, double fallback = 0.0) -> double {
+                    if (d.contains(key) && !d[key].is_none()) {
+                        try { return d[key].cast<double>(); } catch (...) {}
+                    }
+                    return fallback;
+                };
+                auto get_int_safe = [](py::dict d, const char* key, int fallback = 0) -> int {
+                    if (d.contains(key) && !d[key].is_none()) {
+                        try { return d[key].cast<int>(); } catch (...) {}
+                    }
+                    return fallback;
+                };
+                auto get_bool_safe = [](py::dict d, const char* key, bool fallback = false) -> bool {
+                    if (d.contains(key) && !d[key].is_none()) {
+                        try { return d[key].cast<bool>(); } catch (...) {}
+                    }
+                    return fallback;
+                };
+
+                for (auto item : lines_list) {
+                    if (item.is_none()) continue;
+                    py::dict d = item.cast<py::dict>();
+                    pdfengine::document::RawOCRLine line;
+
+                    line.text = get_str_safe(d, "text", "");
+                    line.confidence = get_double_safe(d, "confidence", 0.0);
+                    line.fontSize = get_double_safe(d, "fontSize", 12.0);
+                    line.fontName = get_str_safe(d, "fontName", "Helvetica");
+                    line.fontId = get_str_safe(d, "fontId", "");
+                    line.fontFace = get_str_safe(d, "fontFace", "");
+                    line.fontWeight = get_int_safe(d, "fontWeight", 400);
+                    line.fontStyle = get_str_safe(d, "fontStyle", "normal");
+                    line.isBold = get_bool_safe(d, "isBold", false);
+                    line.isItalic = get_bool_safe(d, "isItalic", false);
+                    line.lineSpacing = get_double_safe(d, "lineSpacing", 1.2);
+                    line.letterSpacing = get_double_safe(d, "letterSpacing", 0.0);
+
+                    if (d.contains("box") && !d["box"].is_none()) {
+                        py::dict box = d["box"].cast<py::dict>();
+                        line.x = get_double_safe(box, "x", 0.0);
+                        line.y = get_double_safe(box, "y", 0.0);
+                        line.width = get_double_safe(box, "width", 0.0);
+                        line.height = get_double_safe(box, "height", 0.0);
+                    } else {
+                        line.x = get_double_safe(d, "x", 0.0);
+                        line.y = get_double_safe(d, "y", 0.0);
+                        line.width = get_double_safe(d, "width", 0.0);
+                        line.height = get_double_safe(d, "height", 0.0);
+                    }
+
+                    spdlog::info(
+                        "[OCR_CPP_RUN] text='{}' fontName='{}' fontId='{}' "
+                        "fontFace='{}' fontWeight={} fontStyle='{}'",
+                        line.text,
+                        line.fontName.empty() ? "<empty>" : line.fontName,
+                        line.fontId.empty() ? "<empty>" : line.fontId,
+                        line.fontFace.empty() ? "<empty>" : line.fontFace,
+                        line.fontWeight,
+                        line.fontStyle.empty() ? "<empty>" : line.fontStyle
+                    );
+
+                    cpp_lines.push_back(line);
+                }
+                return self.processDocument(pageIndex, imgW, imgH, pdfW, pdfH, cpp_lines);
+            },
+            py::arg("page_index"), py::arg("img_w"), py::arg("img_h"), py::arg("pdf_w"),
+            py::arg("pdf_h"), py::arg("lines"));
 }

command.MD

@@ -35,9 +35,16 @@ This document serves as the single source of truth for commands across our devel
 ```powershell
 powershell -ExecutionPolicy Bypass -File scripts/bootstrap.ps1
 cd gateway
-python -m venv .venv
+
+# Fast path with uv (recommended: 10x faster)
+uv venv
 .venv\Scripts\Activate.ps1
-pip install -e ".[dev]"
+uv pip install -e ".[dev]"
+
+# (Alternative fallback with standard pip)
+# python -m venv .venv
+# .venv\Scripts\Activate.ps1
+# pip install -e ".[dev]"
 cd ..
 ```
 
@@ -92,9 +99,16 @@ npm run build
 ```powershell
 powershell -ExecutionPolicy Bypass -File scripts/bootstrap.ps1
 cd gateway
-python -m venv .venv
+
+# Fast path with uv (recommended: 10x faster)
+uv venv
 .venv\Scripts\Activate.ps1
-pip install -e ".[dev]"
+uv pip install -e ".[dev]"
+
+# (Alternative fallback with standard pip)
+# python -m venv .venv
+# .venv\Scripts\Activate.ps1
+# pip install -e ".[dev]"
 cd ..
 ```
 

corpus/convert/public/README.md

@@ -0,0 +1,29 @@
+# Public free eval corpus (Wave-1)
+
+Licensed third-party + DocQube self-generated smoke for PDF↔DOCX/XLSX, Path A, images, OCR.
+
+## Quick start (Windows)
+
+```powershell
+cd pdf
+..\gateway\.venv\Scripts\python.exe scripts\convert\generate_public_smoke_fixtures.py
+..\gateway\.venv\Scripts\python.exe scripts\convert\score_public_pack.py
+```
+
+Optional third-party fetch (after LICENSE_CHECKLIST sign-off):
+
+```powershell
+..\gateway\.venv\Scripts\python.exe scripts\convert\fetch_public_eval.py
+```
+
+## Layout
+
+- `manifests/` — per-family case definitions
+- `pack_public.json` — unified pytest pack
+- `samples/` — gitignored binaries (generated or fetched)
+- `reports/` — baseline / A/B JSON
+
+## Licenses
+
+See `../third_party/NOTICE` and `../third_party/LICENSE_CHECKLIST.md`.
+Default fixtures are **self-generated** (no third-party license). DocLayNet/CORD/SROIE/PubTabNet entries in manifests are placeholders until fetched.

corpus/convert/public/manifests/cord_smoke.json

@@ -0,0 +1,23 @@
+{
+  "cases": [
+    {
+      "id": "cord_placeholder",
+      "family": "cord",
+      "license": "CC-BY-4.0",
+      "source": "public/samples/cord/.gitkeep",
+      "pair": "pdf_txt",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.5
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "placeholder until fetch_public_eval.py",
+      "optional": true
+    }
+  ]
+}

corpus/convert/public/manifests/doclaynet_smoke.json

@@ -0,0 +1,49 @@
+{
+  "cases": [
+    {
+      "id": "synth_digital_001",
+      "family": "doclaynet",
+      "license": "DocQube",
+      "source": "public/samples/doclaynet/synth_digital_001.pdf",
+      "pair": "pdf_docx",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.95
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "expected_tokens": [
+        "Digital",
+        "Report",
+        "AlphaBridge"
+      ],
+      "notes": "self-gen digital layout stand-in"
+    },
+    {
+      "id": "synth_multicol_001",
+      "family": "doclaynet",
+      "license": "DocQube",
+      "source": "public/samples/doclaynet/synth_multicol_001.pdf",
+      "pair": "pdf_docx",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.8
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "expected_tokens": [
+        "ColumnLeftOne",
+        "ColumnRightOne"
+      ],
+      "notes": "two-column reading-order stress"
+    }
+  ]
+}

corpus/convert/public/manifests/images_smoke.json

@@ -0,0 +1,22 @@
+{
+  "cases": [
+    {
+      "id": "images_png_pdf",
+      "family": "images",
+      "license": "DocQube",
+      "source": "public/samples/images/smoke.png",
+      "pair": "png_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "PNG\u2192PDF"
+    }
+  ]
+}

corpus/convert/public/manifests/path_a_smoke.json

@@ -0,0 +1,45 @@
+{
+  "cases": [
+    {
+      "id": "path_a_docx",
+      "family": "path_a",
+      "license": "DocQube",
+      "source": "public/samples/path_a/path_a_sample.docx",
+      "pair": "docx_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [
+          "PathAParagraphToken"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "DOCX\u2192PDF Path A"
+    },
+    {
+      "id": "path_a_xlsx",
+      "family": "path_a",
+      "license": "DocQube",
+      "source": "public/samples/path_a/path_a_sample.xlsx",
+      "pair": "xlsx_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [
+          "HeaderA",
+          "CellAlpha"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "XLSX\u2192PDF Path A"
+    }
+  ]
+}

corpus/convert/public/manifests/sroie_smoke.json

@@ -0,0 +1,23 @@
+{
+  "cases": [
+    {
+      "id": "sroie_placeholder",
+      "family": "sroie",
+      "license": "CC-BY-4.0",
+      "source": "public/samples/sroie/.gitkeep",
+      "pair": "pdf_txt",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.5
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "placeholder until fetch_public_eval.py",
+      "optional": true
+    }
+  ]
+}

corpus/convert/public/manifests/synthetic_invoice_smoke.json

@@ -0,0 +1,42 @@
+{
+  "cases": [
+    {
+      "id": "synth_invoice_001",
+      "family": "synthetic_invoice",
+      "license": "DocQube",
+      "source": "public/samples/invoices/synth_invoice_001.pdf",
+      "pair": "pdf_xlsx",
+      "bars": {
+        "cell_match_min": 0.85,
+        "prose_tokens": [
+          "Invoice",
+          "AcmeCorp",
+          "Total"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": [
+          [
+            "Desc",
+            "Qty",
+            "Amount"
+          ],
+          [
+            "Service",
+            "1",
+            "100"
+          ],
+          [
+            "Tax",
+            "1",
+            "10"
+          ]
+        ],
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "self-gen invoice"
+    }
+  ]
+}

corpus/convert/public/pack_public.json

@@ -0,0 +1,218 @@
+{
+  "cases": [
+    {
+      "id": "synth_invoice_001",
+      "family": "synthetic_invoice",
+      "license": "DocQube",
+      "source": "public/samples/invoices/synth_invoice_001.pdf",
+      "pair": "pdf_xlsx",
+      "bars": {
+        "cell_match_min": 0.85,
+        "prose_tokens": [
+          "Invoice",
+          "AcmeCorp",
+          "Total"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": [
+          [
+            "Desc",
+            "Qty",
+            "Amount"
+          ],
+          [
+            "Service",
+            "1",
+            "100"
+          ],
+          [
+            "Tax",
+            "1",
+            "10"
+          ]
+        ],
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "self-gen invoice"
+    },
+    {
+      "id": "synth_table_001",
+      "family": "pubtabnet",
+      "license": "DocQube",
+      "source": "public/samples/pubtab/synth_table_001.pdf",
+      "pair": "pdf_xlsx",
+      "bars": {
+        "cell_match_min": 0.9,
+        "prose_tokens": [],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": [
+          [
+            "Item",
+            "Qty",
+            "Price"
+          ],
+          [
+            "Widget",
+            "2",
+            "10"
+          ],
+          [
+            "Gadget",
+            "1",
+            "25"
+          ]
+        ],
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "self-gen stand-in for PubTabNet until fetch"
+    },
+    {
+      "id": "synth_digital_001",
+      "family": "doclaynet",
+      "license": "DocQube",
+      "source": "public/samples/doclaynet/synth_digital_001.pdf",
+      "pair": "pdf_docx",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.95
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "expected_tokens": [
+        "Digital",
+        "Report",
+        "AlphaBridge"
+      ],
+      "notes": "self-gen digital layout stand-in"
+    },
+    {
+      "id": "synth_multicol_001",
+      "family": "doclaynet",
+      "license": "DocQube",
+      "source": "public/samples/doclaynet/synth_multicol_001.pdf",
+      "pair": "pdf_docx",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.8
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "expected_tokens": [
+        "ColumnLeftOne",
+        "ColumnRightOne"
+      ],
+      "notes": "two-column reading-order stress"
+    },
+    {
+      "id": "path_a_docx",
+      "family": "path_a",
+      "license": "DocQube",
+      "source": "public/samples/path_a/path_a_sample.docx",
+      "pair": "docx_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [
+          "PathAParagraphToken"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "DOCX\u2192PDF Path A"
+    },
+    {
+      "id": "path_a_xlsx",
+      "family": "path_a",
+      "license": "DocQube",
+      "source": "public/samples/path_a/path_a_sample.xlsx",
+      "pair": "xlsx_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [
+          "HeaderA",
+          "CellAlpha"
+        ],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "XLSX\u2192PDF Path A"
+    },
+    {
+      "id": "images_png_pdf",
+      "family": "images",
+      "license": "DocQube",
+      "source": "public/samples/images/smoke.png",
+      "pair": "png_pdf",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": null
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "PNG\u2192PDF"
+    },
+    {
+      "id": "cord_placeholder",
+      "family": "cord",
+      "license": "CC-BY-4.0",
+      "source": "public/samples/cord/.gitkeep",
+      "pair": "pdf_txt",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.5
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "placeholder until fetch_public_eval.py",
+      "optional": true
+    },
+    {
+      "id": "sroie_placeholder",
+      "family": "sroie",
+      "license": "CC-BY-4.0",
+      "source": "public/samples/sroie/.gitkeep",
+      "pair": "pdf_txt",
+      "bars": {
+        "cell_match_min": null,
+        "prose_tokens": [],
+        "text_recall_min": 0.5
+      },
+      "gt": {
+        "grid": null,
+        "text_file": null,
+        "regions_coco": null
+      },
+      "notes": "placeholder until fetch_public_eval.py",
+      "optional": true
+    }
+  ]
+}

corpus/convert/public/reports/ab_compare.json

@@ -0,0 +1,6 @@
+{
+  "ml_off_passed": 7,
+  "ml_on_passed": 7,
+  "deltas": [],
+  "regressions": 0
+}

corpus/convert/public/reports/baseline_heuristic.json

@@ -0,0 +1,64 @@
+{
+  "ml": "0",
+  "results": [
+    {
+      "id": "synth_invoice_001",
+      "pair": "pdf_xlsx",
+      "ml": "0",
+      "pass": true,
+      "cell_match": 1.0
+    },
+    {
+      "id": "synth_table_001",
+      "pair": "pdf_xlsx",
+      "ml": "0",
+      "pass": true,
+      "cell_match": 1.0
+    },
+    {
+      "id": "synth_digital_001",
+      "pair": "pdf_docx",
+      "ml": "0",
+      "pass": true,
+      "text_recall": 1.0
+    },
+    {
+      "id": "synth_multicol_001",
+      "pair": "pdf_docx",
+      "ml": "0",
+      "pass": true,
+      "text_recall": 1.0
+    },
+    {
+      "id": "path_a_docx",
+      "pair": "docx_pdf",
+      "ml": "0",
+      "pass": true
+    },
+    {
+      "id": "path_a_xlsx",
+      "pair": "xlsx_pdf",
+      "ml": "0",
+      "pass": true
+    },
+    {
+      "id": "images_png_pdf",
+      "pair": "png_pdf",
+      "ml": "0",
+      "pass": true
+    },
+    {
+      "id": "cord_placeholder",
+      "skipped": true,
+      "reason": "missing_source"
+    },
+    {
+      "id": "sroie_placeholder",
+      "skipped": true,
+      "reason": "missing_source"
+    }
+  ],
+  "passed": 7,
+  "failed": 0,
+  "skipped": 2
+}

corpus/convert/public/reports/live_real_doc_suite.json

@@ -0,0 +1,728 @@
+[
+    {
+        "file":  "digital_report_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36742,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "digital_report_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4947,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "govinfo.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  38508,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8932",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "govinfo.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  7086,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8932",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "govinfo_small.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  38508,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8932",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "govinfo_small.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  7085,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8932",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "invoice_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36748,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "invoice_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4980,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6132",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "invoice_002.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36742,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "invoice_002.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4963,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6050",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "irs_f1040_sample.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  41114,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8993",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "irs_f1040_sample.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  10851,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8993",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "merged_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36833,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "merged_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5007,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    },
+    {
+        "file":  "mozilla_pdf_spec_excerpt.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  74515,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9076",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "mozilla_pdf_spec_excerpt.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  56039,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9076",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "multicolumn_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36671,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multicolumn_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4896,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "multicolumn_002.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36671,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multicolumn_002.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4896,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "multipage_table_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36835,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7125",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multipage_table_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5593,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8000",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Page 2: layout_ml=onnx regions"
+    },
+    {
+        "file":  "simple_table_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36701,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "simple_table_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4914,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6050",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "table_gap_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36668,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: glyph/engine text we"
+    },
+    {
+        "file":  "table_gap_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4898,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "table_pipe_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36847,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "table_pipe_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5033,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    },
+    {
+        "file":  "table_simple_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36854,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "table_simple_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5051,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    },
+    {
+        "file":  "usgs_factsheet.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  3065773,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6825",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "usgs_factsheet.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  15077,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.5600",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "w3c_pdf_table.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  37075,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6096",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "w3c_pdf_table.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5335,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.5431",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=2"
+    },
+    {
+        "file":  "w3c_table.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  37075,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6096",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "w3c_table.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5336,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.5431",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=2"
+    },
+    {
+        "file":  "synth_digital_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36742,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "synth_digital_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4947,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "synth_multicol_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36739,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9175",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "synth_multicol_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4933,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9175",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "synth_invoice_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36748,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "synth_invoice_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4981,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6132",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "synth_table_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36701,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "synth_table_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4915,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6050",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "digital_report_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36742,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "digital_report_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4946,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9025",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "invoice_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36748,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "invoice_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4981,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6132",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "invoice_002.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36742,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "invoice_002.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4965,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6050",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "merged_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36833,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "merged_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5006,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    },
+    {
+        "file":  "multicolumn_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36671,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multicolumn_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4895,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "multicolumn_002.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36671,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multicolumn_002.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4895,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "multipage_table_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36835,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7125",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "multipage_table_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5593,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8000",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Page 2: layout_ml=onnx regions"
+    },
+    {
+        "file":  "simple_table_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36701,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "simple_table_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4914,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.6050",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "table_gap_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36668,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: glyph/engine text we"
+    },
+    {
+        "file":  "table_gap_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  4898,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.8875",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
+    },
+    {
+        "file":  "table_pipe_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36847,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "table_pipe_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5032,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    },
+    {
+        "file":  "table_simple_001.pdf",
+        "target":  "docx",
+        "status":  200,
+        "bytes":  36854,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.7275",
+        "ml":  "PASS_ML",
+        "warn":  "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
+    },
+    {
+        "file":  "table_simple_001.pdf",
+        "target":  "xlsx",
+        "status":  200,
+        "bytes":  5051,
+        "magic":  "PK",
+        "fidelity":  "lossy",
+        "quality":  "0.9550",
+        "ml":  "PASS_ML",
+        "warn":  "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
+    }
+]

corpus/convert/public/reports/score_ml_on.json

@@ -0,0 +1,64 @@
+{
+  "ml": "1",
+  "results": [
+    {
+      "id": "synth_invoice_001",
+      "pair": "pdf_xlsx",
+      "ml": "1",
+      "pass": true,
+      "cell_match": 1.0
+    },
+    {
+      "id": "synth_table_001",
+      "pair": "pdf_xlsx",
+      "ml": "1",
+      "pass": true,
+      "cell_match": 1.0
+    },
+    {
+      "id": "synth_digital_001",
+      "pair": "pdf_docx",
+      "ml": "1",
+      "pass": true,
+      "text_recall": 1.0
+    },
+    {
+      "id": "synth_multicol_001",
+      "pair": "pdf_docx",
+      "ml": "1",
+      "pass": true,
+      "text_recall": 1.0
+    },
+    {
+      "id": "path_a_docx",
+      "pair": "docx_pdf",
+      "ml": "1",
+      "pass": true
+    },
+    {
+      "id": "path_a_xlsx",
+      "pair": "xlsx_pdf",
+      "ml": "1",
+      "pass": true
+    },
+    {
+      "id": "images_png_pdf",
+      "pair": "png_pdf",
+      "ml": "1",
+      "pass": true
+    },
+    {
+      "id": "cord_placeholder",
+      "skipped": true,
+      "reason": "missing_source"
+    },
+    {
+      "id": "sroie_placeholder",
+      "skipped": true,
+      "reason": "missing_source"
+    }
+  ],
+  "passed": 7,
+  "failed": 0,
+  "skipped": 2
+}

corpus/convert/public/reports/triage.md

@@ -0,0 +1,16 @@
+# Public pack triage (A/B)
+
+- ML off passed: 7
+- ML on passed: 7
+- Regressions: 0
+
+## Suggested climb owners
+
+| Failure pattern | Climb | Files |
+| --- | --- | --- |
+| Invoice / grid cell | M2 | layout/tables.py, quality/table_metrics.py |
+| Multi-column order | M3 | reading_order.py, ml_merge.py |
+| Scan / OCR empty | M4 | ocr/rebuild.py, rapid_adapter.py |
+| Missing figures | M5 | layout/images.py |
+| Header/footer in body | S1 | headers_footers.py, docx_formatter.py |
+

corpus/convert/real/README.md

@@ -0,0 +1,16 @@
+# Wave-1 production smoke pack (curated)
+
+Internal curated PDFs used for **shippable Wave-1** acceptance bars.
+These are synthetic/corpus-derived stand-ins for business docs until customer PDFs are added.
+
+| File | Role |
+| --- | --- |
+| digital_report_001.pdf | Simple digital text → DOCX |
+| table_pipe_001.pdf / table_simple_001.pdf | Simple tables → XLSX |
+| invoice_001.pdf / invoice_002.pdf | Invoice grid + prose → XLSX |
+| merged_001.pdf | Merged-style table → XLSX |
+| multicolumn_*.pdf | Multi-column → DOCX text recall |
+| multipage_table_001.pdf | Multi-page tables |
+| table_gap_001.pdf | Gap-separated columns |
+
+License: synthetic-internal. Not ConvertAPI goldens.

corpus/convert/real/pack.json

@@ -0,0 +1,182 @@
+{
+  "cases": [
+    {
+      "id": "real_digital_docx",
+      "source": "real/digital_report_001.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "Digital",
+        "Report",
+        "AlphaBridge",
+        "AcmeCorp"
+      ],
+      "text_recall_min": 0.95
+    },
+    {
+      "id": "real_table_xlsx",
+      "source": "real/simple_table_001.pdf",
+      "pair": "pdf->xlsx",
+      "expected_cells": [
+        [
+          "Item",
+          "Qty",
+          "Price"
+        ],
+        [
+          "Widget",
+          "2",
+          "10"
+        ],
+        [
+          "Gadget",
+          "1",
+          "25"
+        ]
+      ],
+      "cell_match_min": 0.9
+    },
+    {
+      "id": "real_invoice_xlsx",
+      "source": "real/invoice_001.pdf",
+      "pair": "pdf->xlsx",
+      "expected_cells": [
+        [
+          "Desc",
+          "Qty",
+          "Amount"
+        ],
+        [
+          "Service",
+          "1",
+          "100"
+        ]
+      ],
+      "prose_tokens": [
+        "Invoice",
+        "AcmeCorp",
+        "Vendor"
+      ],
+      "cell_match_min": 0.85
+    },
+    {
+      "id": "real_two_col_docx",
+      "source": "real/multicolumn_001.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "LeftColA",
+        "LeftColB",
+        "RightColA",
+        "RightColB"
+      ],
+      "text_recall_min": 0.95
+    },
+    {
+      "id": "real_three_col_docx",
+      "source": "real/multicolumn_003.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "AlphaLeft",
+        "BravoMid",
+        "CharlieRight",
+        "AlphaLast",
+        "CharlieLast"
+      ],
+      "text_recall_min": 0.9
+    },
+    {
+      "id": "real_fourcol_xlsx",
+      "source": "real/table_fourcol_001.pdf",
+      "pair": "pdf->xlsx",
+      "expected_cells": [
+        [
+          "Category",
+          "Q1",
+          "Q2",
+          "Total"
+        ],
+        [
+          "Food",
+          "10",
+          "12",
+          "22"
+        ],
+        [
+          "Travel",
+          "5",
+          "7",
+          "12"
+        ]
+      ],
+      "cell_match_min": 0.85
+    },
+    {
+      "id": "real_arabic_docx",
+      "source": "real/arabic_line_001.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "Hello",
+        "World",
+        "TRAIL_AR",
+        "Bilingual"
+      ],
+      "text_recall_min": 0.85
+    },
+    {
+      "id": "real_scanlike_docx",
+      "source": "real/scanlike_001.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "ScanLike",
+        "ScanTokenSMOKEP000",
+        "Widget"
+      ],
+      "text_recall_min": 0.9
+    },
+    {
+      "id": "real_spanning_docx",
+      "source": "real/spanning_title_001.pdf",
+      "pair": "pdf->docx",
+      "expected_tokens": [
+        "SpanningTitleAcrossTheFullPageWidthHere",
+        "SpanLeftA",
+        "SpanRightA"
+      ],
+      "text_recall_min": 0.9
+    },
+    {
+      "id": "real_rect_table_xlsx",
+      "source": "real/rect_table_001.pdf",
+      "pair": "pdf->xlsx",
+      "expected_cells": [
+        [
+          "Name",
+          "Qty",
+          "Cost"
+        ],
+        [
+          "Alpha",
+          "3",
+          "9"
+        ],
+        [
+          "Beta",
+          "4",
+          "8"
+        ]
+      ],
+      "cell_match_min": 0.7
+    },
+    {
+      "id": "real_hyphen_md",
+      "source": "real/hyphen_md_001.pdf",
+      "pair": "pdf->md",
+      "expected_tokens": [
+        "HyphenSmoke",
+        "word",
+        "Chapter",
+        "postprocess"
+      ],
+      "text_recall_min": 0.75
+    }
+  ]
+}

corpus/convert/third_party/LICENSE_CHECKLIST.md

@@ -0,0 +1,18 @@
+# License verification checklist (operator)
+
+Sign off before fetching each asset into `corpus/convert` or `gateway/models/layout`.
+
+| Asset | Upstream URL | License claimed | Verified by | Date | OK for prod train/eval? |
+| --- | --- | --- | --- | --- | --- |
+| DocLayNet | https://github.com/DS4SD/DocLayNet | CDLA-Permissive-1.0 | | | Y/N |
+| PubLayNet | https://github.com/ibm-aur-nlp/publaynet | CDLA-P + PMC CA images | | | Y/N |
+| PubTabNet | https://github.com/ibm-aur-nlp/PubTabNet | CDLA-P + PMC CA | | | Y/N |
+| CORD | https://github.com/clovaai/cord | CC-BY-4.0 | | | Y/N |
+| SROIE | https://rrc.cvc.uab.es/?ch=13 | CC-BY-4.0 (verify) | | | Y/N |
+| PP-DocLayout ONNX | PaddleOCR release / fetch script | Apache-2.0 | | | Y/N |
+| PP-OCRv5 Arabic rec ONNX | RapidAI ModelScope `arabic_PP-OCRv5_rec_mobile.onnx` + `ppocrv5_arabic_dict.txt` | Apache-2.0 (RapidAI/Paddle) | | | Y/N |
+| Self-gen fixtures | DocQube scripts | Proprietary/internal | | | Y |
+
+**Never fetch for prod:** FUNSD (NC), LayoutLMv3 weights (NC), DocLayout-YOLO (AGPL).
+
+Disk budgets: smoke ≤ 500 MB; full DocLayNet ~28 GB lab-only (`--full`).

corpus/convert/third_party/NOTICE

@@ -0,0 +1,40 @@
+DocQube convert — third-party NOTICE
+====================================
+
+Append a block below whenever you fetch a public dataset or layout model.
+Keep license names and upstream URLs accurate.
+
+How to append
+-------------
+1. Verify license on LICENSE_CHECKLIST.md
+2. Download via scripts/convert/fetch_public_eval.py or fetch_layout_onnx.py
+3. Add:
+
+   Name: <asset>
+   License: <SPDX or common name>
+   Source: <URL>
+   Used for: <eval|train|inference>
+   Date: <ISO date>
+
+Self-generated fixtures under corpus/convert (real/ / public/samples/invoices etc.)
+are owned by DocQube and require no third-party attribution.
+
+--- Upstream entries (filled by fetch scripts) ---
+
+Name: PP-DocLayoutV3 ONNX (layout.onnx / inference.onnx)
+License: Apache-2.0
+Source: https://huggingface.co/PaddlePaddle/PP-DocLayoutV3_onnx
+Used for: inference
+Date: 2026-08-27
+SHA256: 45bf71750b00739a41fc209f132eb104a4d6b5bb29483c9078164d8b87cf28ba
+Path: gateway/models/layout/v1/layout.onnx (copy of inference.onnx)
+
+Name: PP-OCRv5 Arabic recognition ONNX (rec.onnx)
+License: Apache-2.0
+Source: https://www.modelscope.cn/models/RapidAI/RapidOCR
+Used for: inference
+
+Name: PP-OCRv5 Arabic charset (arabic_dict.txt)
+License: Apache-2.0
+Source: https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/v3.9.2/paddle/PP-OCRv5/rec/arabic_PP-OCRv5_rec_mobile/ppocrv5_arabic_dict.txt
+Used for: inference

corpus/convert/third_party/checksums.sha256

@@ -0,0 +1,16 @@
+# SHA256 checksums for convert public samples and layout weights
+# Format: <sha256>  <relative-path-from-pdf/>
+# Populated by fetch_public_eval.py / fetch_layout_onnx.py / generate fixtures.
+58a051ee4dcc59e49b1c379ed36e528e1d99dc6968c9a5c11bf6934e0a8a1f36  corpus/convert/real/digital_report_001.pdf
+e67e0040b551d674ed57097d71b1d11bb2c0b45de18da902b847053f9b00d6e8  corpus/convert/real/simple_table_001.pdf
+ba2bf716c44970858e298c340fbd5e8a31c40639315fa2ce42da6bbee0c1a279  corpus/convert/real/invoice_001.pdf
+ffad883045c91f860039352ad9343fad1eb958e6a11fd49a4a2e7f8d22102a5c  corpus/convert/public/samples/invoices/synth_invoice_001.pdf
+0d9165e3d425d3782f4b41e38b890343c6f6cd032691d04f5e345f8ed1105211  corpus/convert/public/samples/pubtab/synth_table_001.pdf
+7fc1dc773d71db49bafdb0458d964e5ec68f76488f785d753f4575b1b685e40f  corpus/convert/public/samples/doclaynet/synth_digital_001.pdf
+314e13e9e4f1178dace76985b10cb41aa5db9a45aaa2e319dc21aab37a336023  corpus/convert/public/samples/doclaynet/synth_multicol_001.pdf
+e2e2f4f6671e753955036f4a0ac2b56e3534bf3901fe9feeeca0e68950a87749  corpus/convert/public/samples/path_a/path_a_sample.docx
+eb203ee73bea31647892d0892605cb65a2aad9a0abdb7a76e8b2e2e60e625775  corpus/convert/public/samples/path_a/path_a_sample.xlsx
+a33ad10e0028c1f84fd1ce229a4de2d0bdb931bc68ac01e80222dd9772008d05  corpus/convert/public/samples/images/smoke.png
+45bf71750b00739a41fc209f132eb104a4d6b5bb29483c9078164d8b87cf28ba  gateway/models/layout/v1/layout.onnx
+c1192e632d0baa9146ae5b756a0e635e3dc63c1733737ebfd1629e87144e9295  gateway/models/ocr/ar/v5/rec.onnx
+7f92f7dbb9b75a4787a83bfb4f6d14a8ab515525130c9d40a9036f61cf6999e9  gateway/models/ocr/ar/v5/arabic_dict.txt

docker-compose.yml

@@ -9,6 +9,10 @@ services:
     environment:
       PDFENGINE_ENVIRONMENT: dev
       PDFENGINE_ENGINE_AVAILABLE: "true"
+      # Keep browser access explicit; production deployments should replace
+      # this with the exact frontend origin(s), never a wildcard.
+      PDFENGINE_CORS_ALLOWED_ORIGINS: http://localhost:5173,http://127.0.0.1:5173,https://pdf-dev.maskantech.in
+      PDFENGINE_CORS_ALLOW_CREDENTIALS: "false"
       PORT: 8765
     ports:
       - "8765:8765"
@@ -30,7 +34,7 @@ services:
     image: pdf-engine-frontend:dev
     container_name: pdf-engine-frontend
     environment:
-      VITE_GATEWAY_URL: http://localhost:8765
+      VITE_GATEWAY_URL: https://pdfapi-dev.maskantech.in
     ports:
       - "5173:5173"
     volumes:
@@ -39,3 +43,10 @@ services:
     depends_on:
       - gateway
     restart: unless-stopped
+
+  gotenberg:
+    image: gotenberg/gotenberg:8
+    container_name: pdf-engine-gotenberg
+    ports:
+      - "3000:3000"
+    restart: unless-stopped

docs/PDF_SECURITY_FEATURE_AUDIT.md

@@ -0,0 +1,298 @@
+# PDF Security / Unlock / Protect — Feature Audit Report
+
+**Date of Audit:** August 13, 2026  
+**Audited Subsystems:** Frontend (`pdf/frontend`), Gateway API (`pdf/gateway`), C++ PDF Engine (`pdf/engine`), Pybind11 Bindings (`pdf/bindings`), Security Tests (`pdf/tests/security`)  
+**Audit Purpose:** Evaluate the exact current state of PDF security, password authentication, permission enforcement, encryption detection, password removal, and PDF protection capabilities in the existing codebase.
+
+---
+
+## 1. Executive Summary
+
+The existing PDF editor codebase possesses a **robust reading, authenticating, and permission-enforcing pipeline** for password-protected PDFs, but **lacks all writing/creation capabilities for PDF encryption and protection**.
+
+Key findings:
+* **Reading & Authenticating Encrypted PDFs:** **FULLY IMPLEMENTED**. The system detects encrypted PDFs, prompts the user via a React modal, passes the password to PDFium in C++, validates credentials, returns helpful error messages on invalid passwords, and renders the document upon success.
+* **Granular Permission Surfacing & Enforcement:** **FULLY IMPLEMENTED**. PDFium extracts security revision numbers, encryption types (RC4, AES-128, AES-256), and permission flags. The Python FastAPI Gateway exposes these flags in `PermissionsResponse` and enforces HTTP `403 Forbidden` errors if a user attempts forbidden edits (annotations, text replacements, page rotations) or unauthorized exports.
+* **Password Removal / Unlocking:** **PARTIALLY IMPLEMENTED (Implicit)**. Opening a protected PDF with a valid password loads the decrypted document into memory. Exporting the document via `/documents/{id}/export` writes out an unencrypted PDF file. However, there is no explicit UI button or API endpoint dedicated to "Remove Password".
+* **Protecting / Encrypting PDFs:** **NOT IMPLEMENTED / MISSING**. There is no functionality in the C++ engine (PDFium/QPDF), Pybind11 bindings, Gateway API, or Frontend UI to password-protect an unencrypted PDF, set user/owner passwords, or configure output permissions.
+
+---
+
+## 2. User-Facing Capability Summary
+
+### CURRENTLY AVAILABLE
+* **✓ Open Password-Protected PDFs:** Prompts for credentials when an encrypted PDF is uploaded.
+* **✓ Password Validation & Error Feedback:** Rejects incorrect passwords with clear inline UI feedback and allows unlimited retries.
+* **✓ Post-Authentication Rendering & Extraction:** Full page rendering, OCR, text extraction, font listing, layout analysis, and display list extraction work seamlessly after authentication.
+* **✓ Encryption & Security Inspection:** Detects and displays encryption standards (RC4-40, RC4-128, AES-128, AES-256) and security revision level (2 through 6) in the Inspector panel.
+* **✓ Permission Enforcement:** Gateway blocks unauthorized edits, annotations, page reordering, and exports with HTTP `403 Forbidden` responses if disallowed by the PDF's security settings.
+* **✓ Unprotected Export:** Exporting an authenticated PDF generates an unencrypted PDF that can subsequently be opened without a password.
+
+### NOT CURRENTLY AVAILABLE
+* **✗ Explicit Password Removal UI/API:** No button or endpoint explicitly labeled "Unlock PDF" or "Remove Security".
+* **✗ Password-Protect PDF / Lock PDF:** Cannot apply passwords to an unencrypted PDF.
+* **✗ Configure Output Permissions:** Cannot set or modify permission flags for printing, copying, editing, or annotating.
+* **✗ Separate Owner Password Prompting:** Prompts only with a generic "Document password" input; does not request owner password specifically when attempting restricted operations.
+* **✗ Re-encrypting Edited PDFs:** Saved/exported PDFs are saved without encryption.
+* **✗ Attempt Rate Limiting:** No rate limiting on password validation attempts at the API level.
+
+---
+
+## 3. Protected PDF Open Flow
+
+The upload and document initialization flow is traced across the full stack:
+
+```
+User Selects Encrypted PDF
+           │
+           ▼
+[Frontend] gatewayService.uploadDocument(file, password="")
+           │
+           ▼ (POST /documents)
+[Gateway API] upload_document() in crud.py
+           │
+           ▼
+[Pybind11] PdfDocument.load_from_memory(bytes_data, "")
+           │
+           ▼
+[C++ Engine] PdfDocument::loadFromMemory() -> FPDF_LoadMemDocument()
+           │
+           ▼ (PDFium returns FPDF_ERR_PASSWORD)
+[C++ Engine] mapPdfiumError() returns EngineError::PasswordRequired
+           │
+           ▼
+[Pybind11] Throws ValueError("Password required to open this PDF")
+           │
+           ▼
+[Gateway API] Catches ValueError -> Raises HTTP 401 ("Password required")
+           │
+           ▼
+[Frontend] gatewayService catches 401 -> Throws PasswordError
+           │
+           ▼
+[Frontend] App.tsx sets passwordPrompt state -> PasswordModal renders
+```
+
+### Exact Code Implementation Points
+* **Encryption Detection & Password Loading (C++ Engine):**
+  * File: [`pdf/engine/src/parser/pdfium_document.cpp`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_document.cpp#L42-L48)
+  * Function: `pdfengine::PdfDocument::loadFromMemory(const std::vector<uint8_t>& data, const std::string& password)`
+  * C++ API: `FPDF_LoadMemDocument(buffer_copy.data(), static_cast<int>(buffer_copy.size()), password.empty() ? nullptr : password.c_str())`
+* **Error Mapping (C++ Engine):**
+  * File: [`pdf/engine/src/parser/pdfium_internal.cpp`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_internal.cpp#L271-L285)
+  * Function: `pdfengine::parser::mapPdfiumError(unsigned long err, bool passwordProvided)`
+  * Logic: Maps `FPDF_ERR_PASSWORD` to `EngineError::PasswordRequired` (if `password` is empty) or `EngineError::InvalidPassword` (if `password` was provided).
+* **Pybind11 Translation:**
+  * File: [`pdf/bindings/python/pdfengine_py.cpp`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/bindings/python/pdfengine_py.cpp#L18-L21)
+  * Function: `throw_on_error(pdfengine::EngineError err)`
+  * Logic: Maps `PasswordRequired` -> `PyExc_ValueError("Password required to open this PDF")` and `InvalidPassword` -> `PyExc_ValueError("Invalid password provided for this PDF")`.
+* **Gateway Endpoint:**
+  * File: [`pdf/gateway/app/routers/documents/crud.py`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/crud.py#L142-L189)
+  * Endpoint: `POST /documents?password={password}` (`upload_document`)
+  * Logic: Catches `ValueError` from pybind11 and raises `HTTPException(status_code=401, detail="Password required")` or `HTTPException(status_code=401, detail="Invalid password")`.
+* **Frontend Password Dialog & Resubmission:**
+  * Files: [`pdf/frontend/src/lib/gatewayService.ts`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/frontend/src/lib/gatewayService.ts#L471-L486), [`pdf/frontend/src/App.tsx`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/frontend/src/App.tsx#L638-L652), [`pdf/frontend/src/components/PasswordModal.tsx`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/frontend/src/components/PasswordModal.tsx#L15-L95)
+  * Logic: `gatewayService` throws `PasswordError`. `App.tsx` catches `PasswordError` and opens `PasswordModal`. User enters password, triggering resubmission to `uploadDocument(file, password)`.
+
+---
+
+## 4. Password Authentication & Validation Flow
+
+| Stage | Implementation Status | Evidence / Location |
+|---|---|---|
+| **Password Reaches Backend** | **IMPLEMENTED** | `uploadDocument(file, password)` in `gatewayService.ts:L476` sends `POST /documents?password=...`. |
+| **Backend Passes Password to C++** | **IMPLEMENTED** | `crud.py:L173` calls `doc = pdfengine.PdfDocument.load_from_memory(bytes_data, password)`. |
+| **C++ Engine Validation** | **IMPLEMENTED** | `pdfium_document.cpp:L42` executes `FPDF_LoadMemDocument(..., password.c_str())`. |
+| **Incorrect Password Rejected** | **IMPLEMENTED** | `pdfium_internal.cpp:L280` returns `EngineError::InvalidPassword` -> HTTP 401 `"Invalid password"` -> `App.tsx` shows red error message in modal. |
+| **Correct Password Accepted** | **IMPLEMENTED** | `FPDF_LoadMemDocument` returns document pointer -> Gateway stores document info and returns HTTP 201 response. |
+| **Password Retry Loop** | **IMPLEMENTED** | `App.tsx:L650` retains modal open on failure with updated error message, allowing infinite retry attempts. |
+| **Rate Limiting / Attempt Limit** | **MISSING** | Neither Gateway nor C++ Engine tracks failed attempts or implements delays/lockouts. |
+
+---
+
+## 5. Rendering & Feature Support After Authentication
+
+Once authenticated, all engine features operate on the unlocked in-memory PDF handle:
+
+| Feature | Status | Evidence / Implementation Location |
+|---|---|---|
+| **Page Count** | **IMPLEMENTED** | [`pdfium_document.cpp:L101`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_document.cpp#L101) (`FPDF_GetPageCount`) |
+| **Document Metadata** | **IMPLEMENTED** | [`pdfium_document.cpp:L109`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_document.cpp#L109) (`FPDF_GetMetaText`) |
+| **Font Inventory** | **IMPLEMENTED** | [`pdfium_document.cpp:L280`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_document.cpp#L280), [`fonts.py:L16`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/fonts.py#L16) |
+| **Page Text Extraction** | **IMPLEMENTED** | [`pdfium_page.cpp:L100`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_page.cpp#L100), [`render.py:L172`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/render.py#L172) |
+| **Page Image Rendering** | **IMPLEMENTED** | [`pdfium_page.cpp:L50`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_page.cpp#L50), [`render.py:L19`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/render.py#L19) |
+| **Display List Extraction** | **IMPLEMENTED** | [`pdfium_page.cpp:L300`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_page.cpp#L300), [`content.py:L133`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/content.py#L133) |
+| **OCR Support** | **IMPLEMENTED** | [`ocr.py:L63`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/ocr.py#L63) (Runs Tesseract on rendered page image) |
+| **Layout Model Extraction** | **IMPLEMENTED** | [`layout.py:L120`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/layout.py#L120) (`extract_document_model()`) |
+| **Editing Operations** | **PARTIAL** | [`edits.py:L395`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/edits.py#L395) checks `permissions`. Allowed edits modify stream in-memory. |
+| **Exporting Document** | **IMPLEMENTED** | [`export.py:L17`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/export.py#L17) checks `canCopy` and calls `save_full_for_export()`. |
+
+---
+
+## 6. Unlock / Decrypt Capability
+
+| Action | Status | Description |
+|---|---|---|
+| **Open Protected PDF** | **IMPLEMENTED** | Via `upload_document` with `password`. |
+| **Authenticate** | **IMPLEMENTED** | Validated via `FPDF_LoadMemDocument`. |
+| **In-Memory Decryption** | **IMPLEMENTED** | PDFium decrypts document structure in RAM for standard operations. |
+| **Save/Export Unprotected Copy** | **IMPLEMENTED (Implicit)** | `doc.save_full_for_export()` calls PDFium's `FPDF_SaveWithVersion(doc_, &writer, 0, 14)`. Because PDFium does not attach an encryption handler during save, the output PDF is **unencrypted**. |
+| **Reopen Exported Copy Without Password** | **IMPLEMENTED** | The exported PDF contains no `/Encrypt` dictionary; reopening requires no password. |
+| **Explicit "Remove Password" Endpoint / UI** | **MISSING** | No dedicated route (e.g. `POST /documents/{id}/unlock`) or UI action exists. |
+
+---
+
+## 7. Protect / Encrypt Capability
+
+A comprehensive search across C++ Engine (`pdf/engine`), Pybind11 (`pdf/bindings`), Gateway (`pdf/gateway`), and Frontend (`pdf/frontend`) reveals **NO code for creating encrypted PDFs**:
+
+* **QPDF Encryption Writer:** Not implemented. [`qpdf_writer.cpp`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/qpdf/qpdf_writer.cpp) contains stream replacement and appearance helpers, but no `QPDFWriter::setEncryption` calls.
+* **PDFium Encryption Output:** PDFium's public writing API lacks native PDF encryption creation functions.
+* **Python Encryption Libraries:** `pikepdf` is imported **only** in [`pdf/tests/security/test_permissions.py`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/tests/security/test_permissions.py#L30) to generate test fixtures. It is not present in Gateway production services.
+* **Frontend Encryption Controls:** No modal, form, or state exists for password-protecting documents.
+
+---
+
+## 8. Password Types & Permission Management
+
+### Password Types
+* **User Password:** **IMPLEMENTED**. Used for opening documents.
+* **Owner Password:** **PARTIALLY IMPLEMENTED**. When opened with an Owner password, PDFium elevates document permissions. The C++ engine detects this by comparing user vs doc permissions (`perms.ownerUnlocked = (p != up)` in [`pdfium_document.cpp:L168`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/src/parser/pdfium_document.cpp#L168)) and displays an `"Owner"` badge in `InspectorPanel.tsx:L569`. However, there is no UI workflow to enter an owner password separately to unlock restricted actions.
+
+### PDF Permissions Matrix
+
+| Permission | Existing Support | Where Implemented |
+|---|---|---|
+| **Print** (`canPrint`) | Surfaced & Displayed | `pdfium_document.cpp:L171`, `store.py:L18`, `InspectorPanel.tsx:L573` |
+| **Modify** (`canModify`) | Surfaced & Enforced | `pdfium_document.cpp:L172`, `edits.py:L375`, `edits.py:L395` (HTTP 403) |
+| **Copy** (`canCopy`) | Surfaced & Enforced | `pdfium_document.cpp:L173`, `export.py:L30`, `export.py:L65` (HTTP 403 on Export) |
+| **Extract** (`canCopy`) | Surfaced & Enforced | Same bit as Copy (`0x10`) in PDFium spec |
+| **Annotate** (`canAnnotate`) | Surfaced & Enforced | `pdfium_document.cpp:L174`, `edits.py:L370-374`, `edits.py:L395` (HTTP 403) |
+| **Fill Forms** (`canFillForms`) | Surfaced & Enforced | `pdfium_document.cpp:L175`, `edits.py:L376`, `edits.py:L395` |
+| **Accessibility** (`canExtractForAccessibility`) | Surfaced & Displayed | `pdfium_document.cpp:L176`, `store.py:L24` |
+| **Document Assembly** (`canAssemble`) | Surfaced & Enforced | `pdfium_document.cpp:L177`, `edits.py:L377` (HTTP 403 on rotation/deletion) |
+| **High-Quality Print** (`canPrintHighRes`) | Surfaced & Displayed | `pdfium_document.cpp:L178`, `store.py:L19` |
+| **Permission Configuration (Writing)** | **MISSING** | No engine or gateway code exists to modify permission flags. |
+
+---
+
+## 9. Encryption Algorithm Surfacing
+
+The C++ engine inspects the PDF security handler revision via `FPDF_GetSecurityHandlerRevision(doc_)`:
+
+```cpp
+// pdfium_document.cpp (lines 147-164)
+switch (rev) {
+    case 2: perms.encryption = "RC4-40"; break;
+    case 3: perms.encryption = "RC4-128"; break;
+    case 4: perms.encryption = "AES-128"; break;
+    case 5:
+    case 6: perms.encryption = "AES-256"; break;
+    default: perms.encryption = "Unknown"; break;
+}
+```
+
+* **Surfacing:** Mapped to `PermissionsResponse.encryption` ([`document.py:L12`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/schemas/document.py#L12)) and rendered as a badge in the Inspector panel ([`InspectorPanel.tsx:L567`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/frontend/src/components/InspectorPanel.tsx#L567)).
+* **Creation:** Encryption creation was not found in the codebase.
+
+---
+
+## 10. Frontend UI State
+
+| Capability | Status | Front-End Evidence |
+|---|---|---|
+| **A. Unlock existing protected PDF** | **IMPLEMENTED** | `PasswordModal.tsx` renders when `passwordPrompt` state is non-null. |
+| **B. Remove password / security** | **MISSING** | No UI button or option. |
+| **C. Protect an unprotected PDF** | **MISSING** | No UI button or option. |
+| **D. Set a password** | **MISSING** | No input fields for protecting PDFs. |
+| **E. Configure permissions** | **MISSING** | No permissions toggle matrix in settings or export dialog. |
+| **F. Export document** | **IMPLEMENTED** | TopBar export button triggers file download. |
+
+---
+
+## 11. Gateway / API Endpoints Audit
+
+| Endpoint | Method | Purpose | Implemented Behavior | C++ Call |
+|---|---|---|---|---|
+| `/documents` | `POST` | Upload & open PDF | Accepts `password` query param. Passes password to engine. Returns HTTP 401 on missing/wrong password, HTTP 201 with permissions on success. | `PdfDocument::loadFromMemory` |
+| `/documents/{id}/export` | `GET` | Export PDF | Verifies `permissions.canCopy`. Returns HTTP 403 if forbidden. Calls `save_full_for_export()`. | `PdfiumDocument::saveFullForExport` |
+| `/documents/{id}/export-remote` | `POST` | Export to remote URL | Verifies `permissions.canCopy`. Streams file to target URL. | `PdfiumDocument::saveFullForExport` |
+| `/edits` | `POST` | Apply PDF edits | Maps operation types to permissions (`canAnnotate`, `canModify`, `canFillForms`, `canAssemble`). Returns HTTP 403 if restricted. | Engine edit APIs |
+| `/documents/{id}/unlock` | N/A | Dedicated unlock | **MISSING** | N/A |
+| `/documents/{id}/protect` | N/A | Protect document | **MISSING** | N/A |
+
+---
+
+## 12. End-to-End Export & Reopen Verification Scenarios
+
+### SCENARIO A: Unprotected PDF -> Protect with Password -> Export -> Reopen -> Prompted for Password
+* **Status:** **NOT WORKING / IMPOSSIBLE TODAY**
+* **Reason:** "Protect with password" is not implemented anywhere in the backend or engine.
+
+### SCENARIO B: Protected PDF -> Enter Password -> Opened -> Export -> Reopen Exported PDF -> No Password Required
+* **Status:** **FULLY WORKING TODAY (Implicitly)**
+* **Reason:** PDFium loads the decrypted PDF structure into memory. Exporting via `GET /documents/{id}/export` writes the file without encryption. Reopening the exported file requires no password.
+
+### SCENARIO C: Protected PDF -> Enter Wrong Password -> Rejected -> Enter Correct Password -> Opened
+* **Status:** **FULLY WORKING TODAY**
+* **Reason:** Invalid password returns HTTP 401 with `"Invalid password"`. The frontend displays `"Incorrect password — please try again."` and keeps the modal open. Re-submitting with the correct password opens the document cleanly.
+
+---
+
+## 13. Security Observations & Risks
+
+* **CONFIRMED FROM CODE — Password Passed in Query String:**
+  In [`gatewayService.ts:L476`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/frontend/src/lib/gatewayService.ts#L476), the upload URL is constructed as `${this.baseUrl}/documents?password=${encodeURIComponent(password)}`. Transmitting passwords in GET/POST URL query parameters poses a security risk because query parameters may be recorded in server access logs or proxy logs.
+* **CONFIRMED FROM CODE — Absence of API Rate Limiting:**
+  In [`crud.py:L142-L190`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/crud.py#L142-L189), there is no rate-limiting or lock-out mechanism for password validation requests, allowing automated brute-force attempts.
+* **CONFIRMED FROM CODE — Implicit Decryption on Export:**
+  In [`export.py:L38`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/gateway/app/routers/documents/export.py#L38), exported PDFs are saved unencrypted. Users who upload a password-protected PDF and subsequently export it will receive an unencrypted file without explicit warning that password protection has been stripped.
+* **CONFIRMED FROM CODE — Plaintext Password In-Memory Only:**
+  Passwords are passed directly to `load_from_memory` and are **not** persisted in `document_store` or written to disk.
+
+---
+
+## 14. Complete Feature Matrix
+
+| Feature | Status | Existing Location | Evidence |
+|---|---|---|---|
+| **Detect encrypted PDF** | `IMPLEMENTED` | `pdfium_document.cpp:L142` | `perms.isEncrypted = (rev != -1)` |
+| **Password popup** | `IMPLEMENTED` | `PasswordModal.tsx:L15` | `<PasswordModal state={passwordPrompt} ... />` |
+| **Validate password** | `IMPLEMENTED` | `pdfium_document.cpp:L42`, `crud.py:L173` | `load_from_memory(bytes_data, password)` |
+| **Wrong password handling** | `IMPLEMENTED` | `crud.py:L183`, `App.tsx:L650` | HTTP 401 `"Invalid password"` -> UI error |
+| **Correct password handling** | `IMPLEMENTED` | `crud.py:L174`, `App.tsx:L646` | Returns `DocumentInfoResponse` -> Document opens |
+| **Render protected PDF** | `IMPLEMENTED` | `pdfium_page.cpp:L50`, `render.py:L19` | Renders tiles/pages post-authentication |
+| **OCR protected PDF** | `IMPLEMENTED` | `ocr.py:L63` | Executes Tesseract on authenticated doc pages |
+| **Edit protected PDF** | `PARTIAL` | `edits.py:L395` | Enforces permissions, but doesn't re-encrypt |
+| **Export protected PDF** | `PARTIAL` | `export.py:L17` | Enforces `canCopy`, but exports UNENCRYPTED |
+| **Remove password** | `PARTIAL` | `export.py:L38` | Exporting strips password (implicit, no explicit API) |
+| **Export unprotected PDF** | `IMPLEMENTED` | `export.py:L38` | `save_full_for_export()` outputs unencrypted PDF |
+| **Reopen unprotected PDF** | `IMPLEMENTED` | `crud.py:L141` | Exported file reopens without password |
+| **Protect PDF** | `MISSING` | N/A | No code exists to protect/encrypt PDF |
+| **Set user password** | `MISSING` | N/A | No functionality to set user password |
+| **Set owner password** | `MISSING` | N/A | No functionality to set owner password |
+| **AES encryption (detection)** | `IMPLEMENTED` | `pdfium_document.cpp:L155-160` | Revision 4/5/6 mapped to `"AES-128"` / `"AES-256"` |
+| **AES-256 (detection)** | `IMPLEMENTED` | `pdfium_document.cpp:L158` | Revision 5/6 mapped to `"AES-256"` |
+| **RC4 (detection)** | `IMPLEMENTED` | `pdfium_document.cpp:L148-153` | Revision 2/3 mapped to `"RC4-40"` / `"RC4-128"` |
+| **Print permission** | `IMPLEMENTED` | `pdfium_document.cpp:L171`, `store.py:L18` | Surfaced in permissions API |
+| **Copy permission** | `IMPLEMENTED` | `pdfium_document.cpp:L173`, `export.py:L30` | Enforced on Export (returns HTTP 403) |
+| **Modify permission** | `IMPLEMENTED` | `pdfium_document.cpp:L172`, `edits.py:L375` | Enforced on edits (returns HTTP 403) |
+| **Annotation permission** | `IMPLEMENTED` | `pdfium_document.cpp:L174`, `edits.py:L370` | Enforced on annotations (returns HTTP 403) |
+| **Form permission** | `IMPLEMENTED` | `pdfium_document.cpp:L175`, `edits.py:L376` | Enforced on form fills (allows if permitted) |
+| **Extraction permission** | `IMPLEMENTED` | `pdfium_document.cpp:L173`, `store.py:L21` | Surfaced as `canCopy` |
+| **Document assembly** | `IMPLEMENTED` | `pdfium_document.cpp:L177`, `edits.py:L377` | Enforced on page rotate/delete ops |
+| **Accessibility permission** | `IMPLEMENTED` | `pdfium_document.cpp:L176`, `store.py:L24` | Surfaced as `canExtractForAccessibility` |
+| **High-quality printing** | `IMPLEMENTED` | `pdfium_document.cpp:L178`, `store.py:L19` | Surfaced as `canPrintHighRes` |
+| **Security UI** | `PARTIAL` | `PasswordModal.tsx`, `InspectorPanel.tsx` | Password prompt modal + Inspector security badge exist |
+| **Security API** | `PARTIAL` | `crud.py`, `export.py`, `edits.py` | Upload & export handle passwords & perms |
+| **C++ security implementation** | `PARTIAL` | `pdfium_document.cpp` | Document load & permission inspection implemented |
+| **Pybind security bindings** | `PARTIAL` | `pdfengine_py.cpp:L252` | `DocumentPermissions` & `load_from_memory` bound |
+
+---
+
+## 15. Production Readiness Summary
+
+* **Opening & Viewing Protected PDFs:** **PRODUCTION READY**. Robust, fully tested with unit tests ([`pdf/engine/tests/document_load_test.cpp`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/engine/tests/document_load_test.cpp#L27-L70)) and security integration tests ([`pdf/tests/security/test_permissions.py`](file:///c:/Users/Maskan/Desktop/pdf_editor/pdf/tests/security/test_permissions.py)).
+* **Permission Enforcement:** **PRODUCTION READY**. Gateway correctly returns HTTP 403 Forbidden for restricted edits and exports.
+* **Password Removal / Unlocking:** **NEEDS FEATURIZATION**. Works implicitly when exporting, but lacks dedicated API routes and UI buttons for explicit unlock workflows.
+* **Protecting / Encrypting PDFs:** **NOT PRODUCTION READY (0% IMPLEMENTED)**. Creation of password-protected PDFs or custom permission dictionaries requires adding QPDF or pikepdf encryption writers to the engine/gateway layer.

engine/CMakeLists.txt

@@ -13,6 +13,9 @@ add_library(pdfengine OBJECT
     src/core/skia_renderer.cpp
     src/parser/content_stream_parser.cpp
     src/parser/decoration_builder.cpp
+    src/document/document_normalizer.cpp
+    src/document/document_builder.cpp
+    src/document/document_validator.cpp
     src/text/selection.cpp
     src/text/text_layout_engine.cpp
     src/fonts/face/font_face.cpp
@@ -29,6 +32,8 @@ add_library(pdfengine OBJECT
     src/fonts/pdf_fonts/font_fallback.cpp
     src/fonts/pdf_fonts/font_subset.cpp
     src/fonts/pdf_fonts/font_cmap_builder.cpp
+    src/fonts/pdf_fonts/font_validator.cpp
+    src/fonts/pdf_fonts/font_extraction_service.cpp
     src/fonts/pdf_fonts/embedded_font_reconstructor.cpp
     src/fonts/pdf_fonts/encoding/encoding.cpp
     src/fonts/pdf_fonts/encoding/tounicode_parser.cpp
@@ -52,6 +57,31 @@ add_library(pdfengine OBJECT
     src/qpdf/qpdf_writer.cpp
     src/qpdf/qpdf_resource_resolver.cpp
     src/core/image_decoder.cpp
+    src/image/core/image_object.cpp
+    src/image/core/image_validator.cpp
+    src/image/core/image_builder.cpp
+    src/image/core/image_pipeline.cpp
+    src/image/core/image_manager.cpp
+    src/image/decoder/filter_decoder.cpp
+    src/image/decoder/sample_decoder.cpp
+    src/image/decoder/pixel_decoder.cpp
+    src/image/decoder/color_converter.cpp
+    src/image/decoder/mask_processor.cpp
+    src/image/decoder/image_decoder_factory.cpp
+    src/ocr/ocr_cache.cpp
+    src/ocr/image_cleaner.cpp
+    src/ocr/ocr_importer.cpp
+    src/ocr/ocr_coordinator.cpp
+    src/layout/layout_arena.cpp
+    src/layout/spatial_index.cpp
+    src/layout/multi_level_cache.cpp
+    src/layout/layout_session.cpp
+    src/layout/pass_registry.cpp
+    src/layout/layout_engine.cpp
+    src/layout/passes/line_detection_pass.cpp
+    src/layout/passes/paragraph_detection_pass.cpp
+    src/layout/passes/column_detection_pass.cpp
+    src/layout/passes/region_detection_pass.cpp
     src/parser/lexer.cpp
     src/parser/parser.cpp
     src/parser/content_builder.cpp
@@ -81,11 +111,15 @@ target_link_libraries(pdfengine
 )
 
 if(PDFENGINE_WITH_PDFIUM)
-    target_link_libraries(pdfengine PRIVATE pdfium::pdfium)
+    target_link_libraries(pdfengine PUBLIC pdfium::pdfium)
     target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_PDFIUM)
-    # PDFium statically bundles its own libjpeg, zlib, etc. which conflicts with vcpkg.
-    # We use LLD, so we can safely allow multiple definitions to pick the first one.
-    target_link_options(pdfengine PUBLIC "-Wl,--allow-multiple-definition")
+    # PDFium statically bundles its own libjpeg, zlib, etc. which conflict with vcpkg.
+    # MSVC ignores -Wl,--allow-multiple-definition (that is a GNU ld flag).
+    if(MSVC)
+        target_link_options(pdfengine PUBLIC "/FORCE:MULTIPLE")
+    else()
+        target_link_options(pdfengine PUBLIC "-Wl,--allow-multiple-definition")
+    endif()
 endif()
 
 if(EMSCRIPTEN)
@@ -99,16 +133,24 @@ if(PDFENGINE_WITH_SKIA)
     target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_SKIA)
 endif()
 
+# qpdf / libjpeg are used by sources that compile even when PDFENGINE_WITH_QPDF is
+# off (resource_resolver.hpp, qpdf_resource_resolver.cpp, image_decoder.cpp,
+# filter_decoder.cpp). vcpkg puts the headers on the include path automatically,
+# so the objects compile — but MSVC then fails with LNK2019 unless we link here.
+# pdfengine is an OBJECT library: PUBLIC is required so consumers (the Python
+# module and smoke test) actually get these libs on the final link line.
+find_package(ZLIB REQUIRED)
+find_package(JPEG REQUIRED)
+find_package(qpdf CONFIG REQUIRED)
+if(NOT TARGET zs)
+    add_library(zs ALIAS ZLIB::ZLIB)
+endif()
+if(NOT TARGET jpeg)
+    add_library(jpeg ALIAS JPEG::JPEG)
+endif()
+target_link_libraries(pdfengine PUBLIC qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
+
 if(PDFENGINE_WITH_QPDF)
-    find_package(ZLIB REQUIRED)
-    find_package(JPEG REQUIRED)
-    if(NOT TARGET zs)
-        add_library(zs ALIAS ZLIB::ZLIB)
-    endif()
-    if(NOT TARGET jpeg)
-        add_library(jpeg ALIAS JPEG::JPEG)
-    endif()
-    target_link_libraries(pdfengine PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
     target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_QPDF)
 endif()
 

engine/include/pdfengine/content_object.hpp

@@ -5,6 +5,7 @@
 #include <memory>
 #include <pdfengine/graphics_state.hpp>
 #include <pdfengine/path.hpp>
+#include <pdfengine/image_object.hpp>
 
 namespace pdfengine {
 
@@ -39,10 +40,14 @@ public:
     double tm[6] = {1.0, 0.0, 0.0, 1.0, 0.0, 0.0};
 };
 
-class ImageObject : public ContentObject {
+class ImageContentObject : public ContentObject {
 public:
     ContentObjectType getType() const override { return ContentObjectType::Image; }
 
+    std::shared_ptr<const ImageObject> image;
+    Matrix transform;
+
+    // Helper compatibility properties
     std::string name;
     int width = 0;
     int height = 0;
@@ -50,12 +55,11 @@ public:
     std::string filter;
     int bitsPerComponent = 8;
     bool hasSoftMask = false;
-
     std::vector<uint8_t> pixelData;
-
-    Matrix transform;
 };
 
+using LegacyImageObject = ImageContentObject;
+
 enum class PathPaintOp {
     Stroke,
     Fill,
@@ -72,4 +76,4 @@ public:
     Matrix transform;
 };
 
-}
+} // namespace pdfengine

engine/include/pdfengine/display_list.hpp

@@ -6,6 +6,7 @@
 #include <pdfengine/graphics_state.hpp>
 #include <pdfengine/path.hpp>
 #include <pdfengine/image.hpp>
+#include <pdfengine/image_object.hpp>
 namespace pdfengine {
 
 class CommandVisitor;
@@ -63,13 +64,48 @@ struct FillStrokePathCommand : public Command {
     void accept(CommandVisitor& visitor) const override;
 };
 
+enum class BlendMode {
+    Normal,
+    Multiply,
+    Screen,
+    Overlay,
+    Darken,
+    Lighten,
+    ColorBurn,
+    SoftLight,
+    Difference
+};
+
+enum class RenderingIntent {
+    RelativeColorimetric,
+    Perceptual,
+    AbsoluteColorimetric,
+    Saturation
+};
+
 struct DrawImageCommand : public Command {
+    std::shared_ptr<const ImageObject> imageObject;
     ImageInfo image;
     Matrix matrix;
-    float opacity;
+    float opacity = 1.0f;
+    BlendMode blendMode = BlendMode::Normal;
+    RenderingIntent renderingIntent = RenderingIntent::RelativeColorimetric;
+    bool interpolate = false;
 
-    DrawImageCommand(ImageInfo img, Matrix m, float op = 1.0f) 
+    DrawImageCommand(std::shared_ptr<const ImageObject> imgObj, Matrix m, float op = 1.0f,
+                     BlendMode blend = BlendMode::Normal, bool interp = false)
+        : imageObject(std::move(imgObj)), matrix(m), opacity(op), blendMode(blend), interpolate(interp) {
+        if (imageObject) {
+            image.width = imageObject->geometry().width;
+            image.height = imageObject->geometry().height;
+            image.channels = imageObject->pixels().channels();
+            image.pixelData = imageObject->pixels().vector();
+        }
+    }
+
+    DrawImageCommand(ImageInfo img, Matrix m, float op = 1.0f)
         : image(std::move(img)), matrix(m), opacity(op) {}
+
     void accept(CommandVisitor& visitor) const override;
 };
 
@@ -107,6 +143,7 @@ public:
     void strokePath(const Path& path);
     void fillStrokePath(const Path& path, FillRule rule = FillRule::NonZero);
     void drawImage(const ImageInfo& image, const Matrix& m, float opacity = 1.0f);
+    void drawImage(std::shared_ptr<const ImageObject> image, const Matrix& m, float opacity = 1.0f, BlendMode blend = BlendMode::Normal);
 
     [[nodiscard]] size_t size() const noexcept { return m_commands.size(); }
     void clear() { m_commands.clear(); }

engine/include/pdfengine/document/document_builder.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "pdfengine/pdf_document.hpp"
+#include "pdfengine/document/raw_ocr_document.hpp"
+
+namespace pdfengine::document {
+
+class DocumentBuilder {
+public:
+    DocumentBuilder() = default;
+
+    // Shared builder: converts raw OCR observation pages into native PageModel
+    PageModel buildFromOCR(const RawOCRPage& rawPage) const;
+};
+
+} // namespace pdfengine::document

engine/include/pdfengine/document/document_normalizer.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "pdfengine/document/raw_ocr_document.hpp"
+
+namespace pdfengine::document {
+
+class DocumentNormalizer {
+public:
+    DocumentNormalizer() = default;
+
+    // Normalizes coordinates, scales pixel boxes to PDF points, fixes negative dimensions and cleans duplicates
+    RawOCRPage normalize(const RawOCRPage& rawPage) const;
+};
+
+} // namespace pdfengine::document

engine/include/pdfengine/document/document_validator.hpp

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "pdfengine/pdf_document.hpp"
+#include <string>
+#include <vector>
+
+namespace pdfengine::document {
+
+struct ValidationIssue {
+    enum class Severity { Warning, Error };
+    Severity severity = Severity::Warning;
+    std::string message;
+};
+
+struct ValidationResult {
+    bool isValid = true;
+    std::vector<ValidationIssue> issues;
+};
+
+class DocumentValidator {
+public:
+    DocumentValidator() = default;
+
+    // Checks PageModel geometry, non-negative bounds, non-empty words, unicode mappings
+    ValidationResult validate(const PageModel& model) const;
+};
+
+} // namespace pdfengine::document

engine/include/pdfengine/document/raw_ocr_document.hpp

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <cstdint>
+
+namespace pdfengine::document {
+
+struct RawOCRWord {
+    std::string text;
+    double x = 0.0;
+    double y = 0.0;
+    double width = 0.0;
+    double height = 0.0;
+    double confidence = 0.0;
+    std::vector<std::pair<double, double>> polygon;
+};
+
+struct RawOCRLine {
+    std::string text;
+    double x = 0.0;
+    double y = 0.0;
+    double width = 0.0;
+    double height = 0.0;
+    double baselineY = 0.0;
+    double confidence = 0.0;
+    std::vector<RawOCRWord> words;
+    std::string fontName;
+    std::string fontId;
+    std::string fontFace;
+    int fontWeight = 400;
+    std::string fontStyle = "normal";
+    double fontSize = 0.0;
+    double lineSpacing = 1.2;
+    double letterSpacing = 0.0;
+    bool isBold = false;
+    bool isItalic = false;
+    bool isEmbeddedFont = false;
+    bool isPredictedFont = true;
+};
+
+struct RawOCRPage {
+    int pageIndex = 0;
+    double imageWidth = 0.0;
+    double imageHeight = 0.0;
+    double pageWidth = 0.0;
+    double pageHeight = 0.0;
+    std::vector<RawOCRLine> lines;
+    double processTimeMs = 0.0;
+};
+
+} // namespace pdfengine::document

engine/include/pdfengine/geometry/matrix.hpp

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <pdfengine/geometry/rect.hpp>
+#include <cmath>
+
+namespace pdfengine {
+namespace geometry {
+
+// 2D Affine Transformation Matrix [a b c d e f]
+// [ x' ]   [ a c e ] [ x ]
+// [ y' ] = [ b d f ] [ y ]
+// [ 1  ]   [ 0 0 1 ] [ 1 ]
+struct Matrix {
+    float a = 1.0f; // Scale X
+    float b = 0.0f; // Shear Y
+    float c = 0.0f; // Shear X
+    float d = 1.0f; // Scale Y
+    float e = 0.0f; // Translate X
+    float f = 0.0f; // Translate Y
+
+    constexpr Matrix() noexcept = default;
+    constexpr Matrix(float aVal, float bVal, float cVal, float dVal, float eVal, float fVal) noexcept
+        : a(aVal), b(bVal), c(cVal), d(dVal), e(eVal), f(fVal) {}
+
+    static constexpr Matrix identity() noexcept {
+        return Matrix(1.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+    }
+
+    static constexpr Matrix translation(float tx, float ty) noexcept {
+        return Matrix(1.0f, 0.0f, 0.0f, 1.0f, tx, ty);
+    }
+
+    static constexpr Matrix scale(float sx, float sy) noexcept {
+        return Matrix(sx, 0.0f, 0.0f, sy, 0.0f, 0.0f);
+    }
+
+    static Matrix rotation(float radians) noexcept {
+        float cosA = std::cos(radians);
+        float sinA = std::sin(radians);
+        return Matrix(cosA, sinA, -sinA, cosA, 0.0f, 0.0f);
+    }
+
+    [[nodiscard]] Point transformPoint(const Point& pt) const noexcept {
+        return Point(a * pt.x + c * pt.y + e, b * pt.x + d * pt.y + f);
+    }
+
+    [[nodiscard]] Matrix multiply(const Matrix& other) const noexcept {
+        return Matrix(
+            a * other.a + c * other.b,
+            b * other.a + d * other.b,
+            a * other.c + c * other.d,
+            b * other.c + d * other.d,
+            a * other.e + c * other.f + e,
+            b * other.e + d * other.f + f
+        );
+    }
+
+    [[nodiscard]] float getRotationAngle() const noexcept {
+        return std::atan2(b, a);
+    }
+};
+
+} // namespace geometry
+} // namespace pdfengine

engine/include/pdfengine/geometry/quad.hpp

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <pdfengine/geometry/rect.hpp>
+#include <pdfengine/geometry/matrix.hpp>
+
+namespace pdfengine {
+namespace geometry {
+
+struct Quad {
+    Point p1; // Top-Left
+    Point p2; // Top-Right
+    Point p3; // Bottom-Right
+    Point p4; // Bottom-Left
+
+    constexpr Quad() noexcept = default;
+    constexpr Quad(Point pt1, Point pt2, Point pt3, Point pt4) noexcept
+        : p1(pt1), p2(pt2), p3(pt3), p4(pt4) {}
+
+    explicit Quad(const Rect& rect) noexcept
+        : p1(rect.left(), rect.top()),
+          p2(rect.right(), rect.top()),
+          p3(rect.right(), rect.bottom()),
+          p4(rect.left(), rect.bottom()) {}
+
+    [[nodiscard]] Rect boundingBox() const noexcept {
+        float l = std::min({p1.x, p2.x, p3.x, p4.x});
+        float r = std::max({p1.x, p2.x, p3.x, p4.x});
+        float t = std::min({p1.y, p2.y, p3.y, p4.y});
+        float b = std::max({p1.y, p2.y, p3.y, p4.y});
+        return Rect(l, t, r - l, b - t);
+    }
+
+    [[nodiscard]] Quad transform(const Matrix& m) const noexcept {
+        return Quad(
+            m.transformPoint(p1),
+            m.transformPoint(p2),
+            m.transformPoint(p3),
+            m.transformPoint(p4)
+        );
+    }
+};
+
+} // namespace geometry
+} // namespace pdfengine

engine/include/pdfengine/geometry/rect.hpp

@@ -0,0 +1,83 @@
+#pragma once
+
+#include <algorithm>
+#include <cmath>
+
+namespace pdfengine {
+namespace geometry {
+
+struct Point {
+    float x = 0.0f;
+    float y = 0.0f;
+
+    constexpr Point() noexcept = default;
+    constexpr Point(float xVal, float yVal) noexcept : x(xVal), y(yVal) {}
+};
+
+struct Size {
+    float width = 0.0f;
+    float height = 0.0f;
+
+    constexpr Size() noexcept = default;
+    constexpr Size(float w, float h) noexcept : width(w), height(h) {}
+};
+
+struct Rect {
+    float x = 0.0f;
+    float y = 0.0f;
+    float width = 0.0f;
+    float height = 0.0f;
+
+    constexpr Rect() noexcept = default;
+    constexpr Rect(float xVal, float yVal, float w, float h) noexcept
+        : x(xVal), y(yVal), width(w), height(h) {}
+
+    [[nodiscard]] constexpr float left() const noexcept { return x; }
+    [[nodiscard]] constexpr float top() const noexcept { return y; }
+    [[nodiscard]] constexpr float right() const noexcept { return x + width; }
+    [[nodiscard]] constexpr float bottom() const noexcept { return y + height; }
+    [[nodiscard]] constexpr float centerX() const noexcept { return x + width * 0.5f; }
+    [[nodiscard]] constexpr float centerY() const noexcept { return y + height * 0.5f; }
+    [[nodiscard]] constexpr float area() const noexcept { return width * height; }
+    [[nodiscard]] constexpr bool isEmpty() const noexcept { return width <= 0.0f || height <= 0.0f; }
+
+    [[nodiscard]] constexpr bool contains(float px, float py) const noexcept {
+        return px >= x && px <= right() && py >= y && py <= bottom();
+    }
+
+    [[nodiscard]] constexpr bool contains(const Point& pt) const noexcept {
+        return contains(pt.x, pt.y);
+    }
+
+    [[nodiscard]] constexpr bool intersects(const Rect& other) const noexcept {
+        return left() < other.right() && right() > other.left() &&
+               top() < other.bottom() && bottom() > other.top();
+    }
+
+    [[nodiscard]] constexpr Rect intersectWith(const Rect& other) const noexcept {
+        float l = std::max(left(), other.left());
+        float r = std::min(right(), other.right());
+        float t = std::max(top(), other.top());
+        float b = std::min(bottom(), other.bottom());
+
+        if (l >= r || t >= b) {
+            return Rect(0.0f, 0.0f, 0.0f, 0.0f);
+        }
+        return Rect(l, t, r - l, b - t);
+    }
+
+    [[nodiscard]] constexpr Rect combineWith(const Rect& other) const noexcept {
+        if (isEmpty()) return other;
+        if (other.isEmpty()) return *this;
+
+        float l = std::min(left(), other.left());
+        float r = std::max(right(), other.right());
+        float t = std::min(top(), other.top());
+        float b = std::max(bottom(), other.bottom());
+
+        return Rect(l, t, r - l, b - t);
+    }
+};
+
+} // namespace geometry
+} // namespace pdfengine

engine/include/pdfengine/image_cache.hpp

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+namespace pdfengine {
+
+class ImageObject;
+
+class IImageCache {
+public:
+    virtual ~IImageCache() = default;
+
+    virtual std::shared_ptr<const ImageObject> get(const std::string& key) = 0;
+    virtual void put(const std::string& key, std::shared_ptr<const ImageObject> image) = 0;
+    virtual bool has(const std::string& key) const = 0;
+    virtual void clear() = 0;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_color_profile.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <string>
+
+namespace pdfengine {
+
+enum class ColorSpaceType {
+    DeviceGray,
+    DeviceRGB,
+    DeviceCMYK,
+    Indexed,
+    ICCBased,
+    Lab,
+    Separation,
+    DeviceN,
+    Pattern,
+    Shading,
+    Unknown
+};
+
+struct ImageColorProfile {
+    ColorSpaceType colorSpace = ColorSpaceType::Unknown;
+    std::string colorSpaceName;
+    int channels = 4;
+    bool hasAlpha = false;
+    std::string intent = "RelativeColorimetric";
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_decoder.hpp

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <pdfengine/image_descriptor.hpp>
+#include <pdfengine/image_pixel_buffer.hpp>
+#include <memory>
+#include <string>
+
+namespace pdfengine {
+
+class ImageDecoder {
+public:
+    static ImagePixelBuffer decode(const ResolvedImageDescriptor& descriptor);
+};
+
+class ImageDecoderFactory {
+public:
+    static bool isSupported(const std::string& filterName);
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_descriptor.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <cstdint>
+
+#include <pdfengine/image_geometry.hpp>
+#include <pdfengine/image_encoding.hpp>
+#include <pdfengine/image_color_profile.hpp>
+#include <pdfengine/image_mask.hpp>
+
+namespace pdfengine {
+
+struct ResolvedImageDescriptor {
+    int objectNumber = 0;
+    int generationNumber = 0;
+    std::string resourceName;
+
+    ImageGeometry geometry;
+    ImageEncoding encoding;
+    ImageColorProfile colorProfile;
+    ImageMask mask;
+
+    std::vector<uint8_t> originalStream;
+    bool hasSoftMask = false;
+    int softMaskObjectId = 0;
+
+    [[nodiscard]] bool isValid() const noexcept {
+        return geometry.isValid() && !originalStream.empty();
+    }
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_diagnostics.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <cstdint>
+
+namespace pdfengine {
+
+struct ImageDiagnostics {
+    uint64_t decodeTimeUs = 0;
+    uint64_t originalBytes = 0;
+    uint64_t decodedBytes = 0;
+    bool fromCache = false;
+    bool usedSoftMask = false;
+};
+
+struct ImageStatistics {
+    uint64_t memoryBytes = 0;
+    uint64_t compressedBytes = 0;
+    uint64_t decodedBytes = 0;
+    int width = 0;
+    int height = 0;
+    bool hasAlpha = false;
+    bool hasMask = false;
+    bool interpolated = false;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_encoding.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+namespace pdfengine {
+
+struct ImageEncoding {
+    std::string filter;
+    int predictor = 1;
+    int columns = 0;
+    int colors = 1;
+    std::unordered_map<std::string, std::string> decodeParms;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_geometry.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+namespace pdfengine {
+
+struct ImageGeometry {
+    int width = 0;
+    int height = 0;
+    int bitsPerComponent = 8;
+
+    [[nodiscard]] bool isValid() const noexcept {
+        return width > 0 && height > 0 && bitsPerComponent > 0;
+    }
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_manager.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <pdfengine/image_object.hpp>
+#include <pdfengine/image_descriptor.hpp>
+#include <pdfengine/image_cache.hpp>
+#include <memory>
+#include <string>
+
+namespace pdfengine {
+
+class ImageManager {
+public:
+    static ImageManager& instance();
+
+    std::shared_ptr<const ImageObject> processImageDescriptor(const ResolvedImageDescriptor& descriptor,
+                                                             std::string* outError = nullptr);
+
+    void setCache(std::shared_ptr<IImageCache> cache);
+    std::shared_ptr<IImageCache> cache() const;
+
+private:
+    ImageManager() = default;
+    std::shared_ptr<IImageCache> m_cache;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_mask.hpp

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <memory>
+
+namespace pdfengine {
+
+class ImageObject;
+
+enum class MaskType {
+    None,
+    ExplicitMask,
+    SoftMask
+};
+
+struct ImageMask {
+    MaskType type = MaskType::None;
+    std::shared_ptr<const ImageObject> maskImage;
+
+    [[nodiscard]] bool hasMask() const noexcept {
+        return type != MaskType::None && maskImage != nullptr;
+    }
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_object.hpp

@@ -0,0 +1,77 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <cstdint>
+#include <atomic>
+
+#include <pdfengine/image_geometry.hpp>
+#include <pdfengine/image_encoding.hpp>
+#include <pdfengine/image_color_profile.hpp>
+#include <pdfengine/image_pixel_buffer.hpp>
+#include <pdfengine/image_mask.hpp>
+#include <pdfengine/image_diagnostics.hpp>
+
+namespace pdfengine {
+
+using ImageId = uint64_t;
+
+ImageId generateNextImageId();
+
+class ImageObject {
+public:
+    ImageObject(ImageId id,
+                std::string name,
+                ImageGeometry geometry,
+                ImageEncoding encoding,
+                ImageColorProfile colorProfile,
+                ImagePixelBuffer pixels,
+                ImageMask mask,
+                std::vector<uint8_t> originalStream,
+                ImageDiagnostics diagnostics = {},
+                ImageStatistics statistics = {})
+        : m_id(id),
+          m_name(std::move(name)),
+          m_geometry(geometry),
+          m_encoding(std::move(encoding)),
+          m_colorProfile(std::move(colorProfile)),
+          m_pixels(std::move(pixels)),
+          m_mask(std::move(mask)),
+          m_originalStream(std::move(originalStream)),
+          m_diagnostics(diagnostics),
+          m_statistics(statistics) {}
+
+    [[nodiscard]] ImageId id() const noexcept { return m_id; }
+    [[nodiscard]] const std::string& name() const noexcept { return m_name; }
+    [[nodiscard]] const ImageGeometry& geometry() const noexcept { return m_geometry; }
+    [[nodiscard]] const ImageEncoding& encoding() const noexcept { return m_encoding; }
+    [[nodiscard]] const ImageColorProfile& colorProfile() const noexcept { return m_colorProfile; }
+    [[nodiscard]] const ImagePixelBuffer& pixels() const noexcept { return m_pixels; }
+    [[nodiscard]] const ImageMask& mask() const noexcept { return m_mask; }
+    [[nodiscard]] const std::vector<uint8_t>& originalStream() const noexcept { return m_originalStream; }
+    [[nodiscard]] const ImageDiagnostics& diagnostics() const noexcept { return m_diagnostics; }
+    [[nodiscard]] const ImageStatistics& statistics() const noexcept { return m_statistics; }
+
+    [[nodiscard]] bool isValid() const noexcept {
+        return m_id != 0 && m_geometry.isValid() && !m_pixels.empty();
+    }
+
+    [[nodiscard]] size_t byteSize() const noexcept {
+        return sizeof(*this) + m_pixels.size() + m_originalStream.size();
+    }
+
+private:
+    ImageId m_id = 0;
+    std::string m_name;
+    ImageGeometry m_geometry;
+    ImageEncoding m_encoding;
+    ImageColorProfile m_colorProfile;
+    ImagePixelBuffer m_pixels;
+    ImageMask m_mask;
+    std::vector<uint8_t> m_originalStream;
+    ImageDiagnostics m_diagnostics;
+    ImageStatistics m_statistics;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_pipeline.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <pdfengine/image_object.hpp>
+#include <pdfengine/image_descriptor.hpp>
+#include <memory>
+
+namespace pdfengine {
+
+class ImagePipeline {
+public:
+    static std::shared_ptr<const ImageObject> process(const ResolvedImageDescriptor& descriptor);
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_pixel_buffer.hpp

@@ -0,0 +1,82 @@
+#pragma once
+
+#include <vector>
+#include <cstdint>
+#include <cstddef>
+#include <algorithm>
+
+namespace pdfengine {
+
+struct PixelRGBA {
+    uint8_t r = 0;
+    uint8_t g = 0;
+    uint8_t b = 0;
+    uint8_t a = 255;
+};
+
+class ImagePixelBuffer {
+public:
+    ImagePixelBuffer() = default;
+    ImagePixelBuffer(int width, int height, int channels = 4)
+        : m_width(width), m_height(height), m_channels(channels) {
+        if (width > 0 && height > 0 && channels > 0) {
+            m_pixels.resize(static_cast<size_t>(width) * height * channels, 0);
+        }
+    }
+
+    [[nodiscard]] int width() const noexcept { return m_width; }
+    [[nodiscard]] int height() const noexcept { return m_height; }
+    [[nodiscard]] int channels() const noexcept { return m_channels; }
+    [[nodiscard]] size_t stride() const noexcept { return static_cast<size_t>(m_width) * m_channels; }
+    [[nodiscard]] size_t size() const noexcept { return m_pixels.size(); }
+    [[nodiscard]] bool empty() const noexcept { return m_pixels.empty(); }
+
+    [[nodiscard]] const uint8_t* data() const noexcept { return m_pixels.data(); }
+    [[nodiscard]] uint8_t* data() noexcept { return m_pixels.data(); }
+
+    [[nodiscard]] const std::vector<uint8_t>& vector() const noexcept { return m_pixels; }
+    [[nodiscard]] std::vector<uint8_t>& vector() noexcept { return m_pixels; }
+
+    void resize(int width, int height, int channels = 4) {
+        m_width = width;
+        m_height = height;
+        m_channels = channels;
+        if (width > 0 && height > 0 && channels > 0) {
+            m_pixels.assign(static_cast<size_t>(width) * height * channels, 0);
+        } else {
+            m_pixels.clear();
+        }
+    }
+
+    [[nodiscard]] PixelRGBA getPixel(int x, int y) const noexcept {
+        if (x < 0 || x >= m_width || y < 0 || y >= m_height || m_channels < 4) {
+            return {};
+        }
+        size_t idx = (static_cast<size_t>(y) * m_width + x) * m_channels;
+        if (idx + 3 < m_pixels.size()) {
+            return PixelRGBA{m_pixels[idx], m_pixels[idx + 1], m_pixels[idx + 2], m_pixels[idx + 3]};
+        }
+        return {};
+    }
+
+    void setPixel(int x, int y, uint8_t r, uint8_t g, uint8_t b, uint8_t a = 255) noexcept {
+        if (x < 0 || x >= m_width || y < 0 || y >= m_height || m_channels < 4) {
+            return;
+        }
+        size_t idx = (static_cast<size_t>(y) * m_width + x) * m_channels;
+        if (idx + 3 < m_pixels.size()) {
+            m_pixels[idx + 0] = r;
+            m_pixels[idx + 1] = g;
+            m_pixels[idx + 2] = b;
+            m_pixels[idx + 3] = a;
+        }
+    }
+
+private:
+    int m_width = 0;
+    int m_height = 0;
+    int m_channels = 4;
+    std::vector<uint8_t> m_pixels;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/image_validator.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <pdfengine/image_geometry.hpp>
+#include <pdfengine/image_descriptor.hpp>
+#include <string>
+
+namespace pdfengine {
+
+class ImageValidator {
+public:
+    static bool validate(const ImageGeometry& geometry, std::string* outError = nullptr);
+    static bool validate(const ResolvedImageDescriptor& descriptor, std::string* outError = nullptr);
+};
+
+} // namespace pdfengine

engine/include/pdfengine/layout_arena.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <vector>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <utility>
+
+namespace pdfengine {
+
+class LayoutArena {
+public:
+    explicit LayoutArena(size_t blockSizeBytes = 65536);
+    ~LayoutArena();
+
+    // Disable copy
+    LayoutArena(const LayoutArena&) = delete;
+    LayoutArena& operator=(const LayoutArena&) = delete;
+
+    // Enable move
+    LayoutArena(LayoutArena&&) noexcept;
+    LayoutArena& operator=(LayoutArena&&) noexcept;
+
+    template <typename T, typename... Args>
+    T* allocate(Args&&... args) {
+        void* mem = allocateBytes(sizeof(T), alignof(T));
+        return ::new (mem) T(std::forward<Args>(args)...);
+    }
+
+    void* allocateBytes(size_t size, size_t alignment = alignof(std::max_align_t));
+    void clear() noexcept;
+    [[nodiscard]] size_t totalAllocatedBytes() const noexcept;
+
+private:
+    struct Chunk {
+        std::unique_ptr<uint8_t[]> data;
+        size_t size = 0;
+        size_t used = 0;
+    };
+
+    size_t m_defaultChunkSize = 65536;
+    std::vector<Chunk> m_chunks;
+    size_t m_totalAllocated = 0;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/layout_engine.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <pdfengine/layout_session.hpp>
+#include <pdfengine/pass_registry.hpp>
+#include <memory>
+
+namespace pdfengine {
+
+class LayoutEngine {
+public:
+    static LayoutEngine& instance();
+
+    PassRegistry& registry() noexcept { return m_registry; }
+    const PassRegistry& registry() const noexcept { return m_registry; }
+
+    std::shared_ptr<PhysicalLayoutTree> processPage(const std::string& documentId, int pageIndex, float width, float height);
+
+private:
+    LayoutEngine();
+    PassRegistry m_registry;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/layout_pass.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <pdfengine/layout_session.hpp>
+
+namespace pdfengine {
+
+class ILayoutPass {
+public:
+    virtual ~ILayoutPass() = default;
+
+    [[nodiscard]] virtual std::string name() const noexcept = 0;
+    virtual bool execute(LayoutSession& session) = 0;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/layout_session.hpp

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <pdfengine/layout_types.hpp>
+#include <pdfengine/layout_tree.hpp>
+#include <pdfengine/layout_arena.hpp>
+#include <pdfengine/multi_level_cache.hpp>
+#include <pdfengine/spatial_index.hpp>
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <atomic>
+#include <chrono>
+
+namespace pdfengine {
+
+struct PipelineContext {
+    std::string documentId;
+    int pageIndex = 0;
+    float pageWidth = 0.0f;
+    float pageHeight = 0.0f;
+
+    std::shared_ptr<PhysicalLayoutTree> physicalTree;
+    std::shared_ptr<LogicalLayoutTree> logicalTree;
+    std::shared_ptr<MultiIndexSpatialIndex> spatialIndex;
+
+    std::vector<LayoutDiagnostic> diagnostics;
+    LayoutStatistics stats;
+};
+
+class LayoutSession {
+public:
+    LayoutSession(std::string documentId, int pageIndex, float width, float height);
+    ~LayoutSession() = default;
+
+    // Disable copy
+    LayoutSession(const LayoutSession&) = delete;
+    LayoutSession& operator=(const LayoutSession&) = delete;
+
+    [[nodiscard]] PipelineContext& context() noexcept { return m_context; }
+    [[nodiscard]] const PipelineContext& context() const noexcept { return m_context; }
+
+    [[nodiscard]] LayoutArena& arena() noexcept { return m_arena; }
+    [[nodiscard]] const LayoutArena& arena() const noexcept { return m_arena; }
+
+    void cancel() noexcept { m_cancelled.store(true); }
+    [[nodiscard]] bool isCancelled() const noexcept { return m_cancelled.load(); }
+
+    void logDiagnostic(LayoutDiagnostic::Severity severity, const std::string& passName, const std::string& message, const std::string& blockId = "");
+
+    void finish();
+
+private:
+    PipelineContext m_context;
+    LayoutArena m_arena;
+    std::atomic<bool> m_cancelled{false};
+    std::chrono::high_resolution_clock::time_point m_startTime;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/layout_tree.hpp

@@ -0,0 +1,96 @@
+#pragma once
+
+#include <pdfengine/layout_types.hpp>
+#include <pdfengine/geometry/rect.hpp>
+#include <pdfengine/geometry/matrix.hpp>
+#include <pdfengine/geometry/quad.hpp>
+
+#include <string>
+#include <vector>
+#include <memory>
+
+namespace pdfengine {
+
+struct LayoutGlyph {
+    std::string character;
+    uint32_t charCode = 0;
+    geometry::Rect bounds;
+    geometry::Matrix transform;
+    float advanceWidth = 0.0f;
+};
+
+struct TextRun {
+    std::string text;
+    TextStyle style;
+    geometry::Rect bounds;
+    geometry::Matrix transform;
+    std::vector<LayoutGlyph> glyphs;
+};
+
+struct LayoutLine {
+    std::string text;
+    geometry::Rect bounds;
+    float baselineY = 0.0f;
+    float lineHeight = 12.0f;
+    std::vector<TextRun> runs;
+};
+
+struct LayoutBlock {
+    std::string id;
+    LayoutBlockType type = LayoutBlockType::Paragraph;
+    geometry::Rect bounds;
+    geometry::Matrix transform;
+    int zIndex = 0;
+    int readingOrder = 0;
+
+    BlockPermissions permissions;
+    VisualStyle visualStyle;
+    TextStyle textStyle;
+    LayoutStyle layoutStyle;
+
+    std::vector<LayoutLine> children;
+    std::string parentId;
+    std::vector<std::string> dependsOn; // Block ID dependencies (e.g. caption -> image)
+
+    size_t layoutTreeVersion = 1;
+    size_t contentRevision = 1;
+    size_t layoutRevision = 1;
+    std::string sourceObjectId;
+};
+
+enum class RegionType {
+    Header,
+    Body,
+    Sidebar,
+    Footer
+};
+
+struct PageRegion {
+    std::string id;
+    RegionType type = RegionType::Body;
+    geometry::Rect bounds;
+    std::vector<std::shared_ptr<LayoutBlock>> blocks;
+};
+
+struct PhysicalLayoutTree {
+    int pageIndex = 0;
+    float width = 0.0f;
+    float height = 0.0f;
+    std::vector<PageRegion> regions;
+    std::vector<std::shared_ptr<LayoutBlock>> allBlocks;
+};
+
+struct LogicalLayoutNode {
+    std::string id;
+    std::string title;
+    LayoutBlockType type = LayoutBlockType::Paragraph;
+    std::shared_ptr<LayoutBlock> block;
+    std::vector<std::shared_ptr<LogicalLayoutNode>> children;
+};
+
+struct LogicalLayoutTree {
+    std::string documentTitle;
+    std::vector<std::shared_ptr<LogicalLayoutNode>> sections;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/layout_types.hpp

@@ -0,0 +1,106 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <cstdint>
+#include <pdfengine/geometry/rect.hpp>
+#include <pdfengine/geometry/matrix.hpp>
+
+namespace pdfengine {
+
+enum class LayoutBlockType {
+    Paragraph,
+    Heading,
+    Title,
+    Caption,
+    Quote,
+    List,
+    Table,
+    TableCell,
+    Image,
+    Figure,
+    Form,
+    Shape,
+    Header,
+    Footer,
+    Watermark,
+    Annotation,
+    CodeBlock,
+    Math,
+    TOC,
+    Footnote,
+    Signature,
+    Unknown
+};
+
+enum class TextAlignment {
+    Left,
+    Center,
+    Right,
+    Justify
+};
+
+struct VisualStyle {
+    std::string fillColor = "transparent";
+    std::string strokeColor = "none";
+    float strokeWidth = 0.0f;
+    float opacity = 1.0f;
+    float cornerRadius = 0.0f;
+    std::string shadowColor = "none";
+};
+
+struct TextStyle {
+    std::string fontName = "Helvetica";
+    float fontSize = 12.0f;
+    std::string fontColor = "#000000";
+    bool isBold = false;
+    bool isItalic = false;
+    float letterSpacing = 0.0f;
+    float lineSpacing = 1.2f;
+};
+
+struct LayoutStyle {
+    TextAlignment alignment = TextAlignment::Left;
+    float paddingTop = 0.0f;
+    float paddingRight = 0.0f;
+    float paddingBottom = 0.0f;
+    float paddingLeft = 0.0f;
+    float marginTop = 0.0f;
+    float marginRight = 0.0f;
+    float marginBottom = 0.0f;
+    float marginLeft = 0.0f;
+    int zIndex = 0;
+};
+
+struct BlockPermissions {
+    bool editable = true;
+    bool selectable = true;
+    bool movable = true;
+    bool resizable = true;
+    bool printable = true;
+};
+
+struct LayoutDiagnostic {
+    enum class Severity { Info, Warning, Error };
+    Severity severity = Severity::Info;
+    std::string passName;
+    std::string message;
+    int pageIndex = 0;
+    std::string blockId;
+    double durationMs = 0.0;
+};
+
+struct LayoutStatistics {
+    size_t textRuns = 0;
+    size_t blocks = 0;
+    size_t paragraphs = 0;
+    size_t tables = 0;
+    size_t columns = 0;
+    double decodeTimeMs = 0.0;
+    double layoutTimeMs = 0.0;
+    size_t memoryBytes = 0;
+    size_t cacheHits = 0;
+    size_t cacheMisses = 0;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/multi_level_cache.hpp

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <pdfengine/layout_tree.hpp>
+#include <pdfengine/spatial_index.hpp>
+#include <unordered_map>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <memory>
+
+namespace pdfengine {
+
+struct CacheKey {
+    std::string documentId;
+    int pageIndex = 0;
+    size_t contentRevision = 1;
+    size_t layoutRevision = 1;
+
+    bool operator==(const CacheKey& other) const noexcept {
+        return documentId == other.documentId &&
+               pageIndex == other.pageIndex &&
+               contentRevision == other.contentRevision &&
+               layoutRevision == other.layoutRevision;
+    }
+};
+
+struct CacheKeyHash {
+    size_t operator()(const CacheKey& k) const noexcept {
+        size_t h1 = std::hash<std::string>{}(k.documentId);
+        size_t h2 = std::hash<int>{}(k.pageIndex);
+        size_t h3 = std::hash<size_t>{}(k.contentRevision);
+        size_t h4 = std::hash<size_t>{}(k.layoutRevision);
+        return h1 ^ (h2 << 1) ^ (h3 << 2) ^ (h4 << 3);
+    }
+};
+
+class MultiLevelCache {
+public:
+    static MultiLevelCache& instance();
+
+    void putLayoutTree(const CacheKey& key, std::shared_ptr<PhysicalLayoutTree> tree);
+    [[nodiscard]] std::shared_ptr<PhysicalLayoutTree> getLayoutTree(const CacheKey& key) const;
+
+    void putSpatialIndex(const CacheKey& key, std::shared_ptr<MultiIndexSpatialIndex> index);
+    [[nodiscard]] std::shared_ptr<MultiIndexSpatialIndex> getSpatialIndex(const CacheKey& key) const;
+
+    void invalidatePage(const std::string& documentId, int pageIndex);
+    void invalidateDocument(const std::string& documentId);
+    void clear();
+
+    [[nodiscard]] size_t size() const;
+
+private:
+    MultiLevelCache() = default;
+    mutable std::mutex m_mutex;
+
+    std::unordered_map<CacheKey, std::shared_ptr<PhysicalLayoutTree>, CacheKeyHash> m_layoutCache;
+    std::unordered_map<CacheKey, std::shared_ptr<MultiIndexSpatialIndex>, CacheKeyHash> m_spatialCache;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/ocr/image_cleaner.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <vector>
+#include <cstdint>
+
+namespace pdfengine::ocr {
+
+class ImageCleaner {
+public:
+    ImageCleaner() = default;
+
+    // Binarize or despackle raster pixel data if needed
+    std::vector<uint8_t> cleanImage(const std::vector<uint8_t>& bgra, int width, int height) const;
+};
+
+} // namespace pdfengine::ocr

engine/include/pdfengine/ocr/ocr_coordinator.hpp

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "pdfengine/pdf_document.hpp"
+#include "pdfengine/document/raw_ocr_document.hpp"
+#include "pdfengine/document/document_normalizer.hpp"
+#include "pdfengine/document/document_builder.hpp"
+#include "pdfengine/document/document_validator.hpp"
+#include "pdfengine/ocr/image_cleaner.hpp"
+#include "pdfengine/ocr/ocr_importer.hpp"
+
+namespace pdfengine::ocr {
+
+class OCRCoordinator {
+public:
+    OCRCoordinator() = default;
+
+    // Single Facade API: orchestrates Preprocessing -> OCRImporter -> Engine::Normalizer -> Engine::Builder -> Engine::Validator -> Native PageModel
+    pdfengine::PageModel processDocument(
+        int pageIndex,
+        double imgW,
+        double imgH,
+        double pdfW,
+        double pdfH,
+        const std::vector<pdfengine::document::RawOCRLine>& lines
+    ) const;
+};
+
+} // namespace pdfengine::ocr

engine/include/pdfengine/ocr/ocr_importer.hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "pdfengine/document/raw_ocr_document.hpp"
+#include <vector>
+
+namespace pdfengine::ocr {
+
+class OCRImporter {
+public:
+    OCRImporter() = default;
+
+    pdfengine::document::RawOCRPage importOCRPage(
+        int pageIndex,
+        double imgWidth,
+        double imgHeight,
+        double pdfWidth,
+        double pdfHeight,
+        const std::vector<pdfengine::document::RawOCRLine>& lines
+    ) const;
+};
+
+} // namespace pdfengine::ocr

engine/include/pdfengine/ocr_cache.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <pdfengine/ocr_types.hpp>
+#include <unordered_map>
+#include <mutex>
+#include <optional>
+#include <string>
+
+namespace pdfengine {
+
+class OCRCache {
+public:
+    static OCRCache& instance();
+
+    void put(const std::string& imageHash, const OCRPage& ocrPage);
+    [[nodiscard]] std::optional<OCRPage> get(const std::string& imageHash) const;
+    void clear();
+    [[nodiscard]] size_t size() const;
+
+private:
+    OCRCache() = default;
+    mutable std::mutex m_mutex;
+    std::unordered_map<std::string, OCRPage> m_cache;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/ocr_engine.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <pdfengine/ocr_types.hpp>
+#include <pdfengine/image_object.hpp>
+#include <memory>
+#include <string>
+
+namespace pdfengine {
+
+class IOCREngine {
+public:
+    virtual ~IOCREngine() = default;
+
+    [[nodiscard]] virtual bool isAvailable() const noexcept = 0;
+    [[nodiscard]] virtual OCRPage processImage(const ImageObject& image, const std::string& lang = "en") = 0;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/ocr_types.hpp

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <cstdint>
+
+namespace pdfengine {
+
+struct OCRPoint {
+    float x = 0.0f;
+    float y = 0.0f;
+};
+
+struct OCRRect {
+    float x = 0.0f;
+    float y = 0.0f;
+    float width = 0.0f;
+    float height = 0.0f;
+};
+
+struct OCRWord {
+    std::string text;
+    OCRRect box;
+    std::vector<OCRPoint> polygon; // 4 corner points [[x1,y1],[x2,y2],[x3,y3],[x4,y4]]
+    float confidence = 0.0f;
+};
+
+struct OCRLine {
+    std::string text;
+    OCRRect box;
+    std::vector<OCRWord> words;
+    float baselineY = 0.0f;
+    float confidence = 0.0f;
+};
+
+struct OCRPage {
+    int pageIndex = 0;
+    int imageWidth = 0;
+    int imageHeight = 0;
+    std::vector<OCRLine> lines;
+    std::string language = "en";
+    double processTimeMs = 0.0;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/pass_registry.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <pdfengine/layout_pass.hpp>
+#include <vector>
+#include <memory>
+#include <string>
+
+namespace pdfengine {
+
+class PassRegistry {
+public:
+    PassRegistry() = default;
+
+    void registerPass(std::unique_ptr<ILayoutPass> pass);
+    void clear();
+
+    bool executeAll(LayoutSession& session);
+
+    [[nodiscard]] size_t passCount() const noexcept { return m_passes.size(); }
+
+private:
+    std::vector<std::unique_ptr<ILayoutPass>> m_passes;
+};
+
+} // namespace pdfengine

engine/include/pdfengine/pdf_document.hpp

@@ -142,10 +142,15 @@ struct Glyph {
 struct TextRun {
     std::string text;
     std::string fontName;
+    std::string fontFace;
+    int fontWeight = 400;
+    std::string fontStyle = "normal";
     uint32_t flags = 0;
     double fontSize = 0.0;
     std::string internalFontId;
     bool isEmbedded = false;
+    bool isEmbeddedFont = false;
+    bool isPredictedFont = false;
     std::string type;
     std::vector<Glyph> glyphs;
     std::vector<int> objectIndices;

engine/include/pdfengine/spatial_index.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <pdfengine/layout_tree.hpp>
+#include <pdfengine/geometry/rect.hpp>
+
+#include <vector>
+#include <memory>
+#include <string>
+
+namespace pdfengine {
+
+class MultiIndexSpatialIndex {
+public:
+    MultiIndexSpatialIndex() = default;
+
+    void buildFromTree(const PhysicalLayoutTree& tree);
+    void clear();
+
+    [[nodiscard]] std::vector<std::shared_ptr<LayoutBlock>> queryBlocksAtPoint(float x, float y) const;
+    [[nodiscard]] std::vector<std::shared_ptr<LayoutBlock>> queryBlocksInRect(const geometry::Rect& rect) const;
+    [[nodiscard]] std::vector<std::shared_ptr<LayoutBlock>> queryBlocksByType(LayoutBlockType type) const;
+
+    [[nodiscard]] size_t totalIndexedBlocks() const noexcept { return m_indexedBlocks.size(); }
+
+private:
+    struct SpatialEntry {
+        geometry::Rect bounds;
+        std::shared_ptr<LayoutBlock> block;
+    };
+
+    std::vector<SpatialEntry> m_indexedBlocks;
+    std::vector<SpatialEntry> m_indexedLines;
+};
+
+} // namespace pdfengine

engine/src/core/display_list.cpp

@@ -60,4 +60,8 @@ void DisplayList::drawImage(const ImageInfo& image, const Matrix& m, float opaci
     addCommand(std::make_unique<DrawImageCommand>(image, m, opacity));
 }
 
+void DisplayList::drawImage(std::shared_ptr<const ImageObject> image, const Matrix& m, float opacity, BlendMode blend) {
+    addCommand(std::make_unique<DrawImageCommand>(std::move(image), m, opacity, blend));
+}
+
 }

engine/src/document/document_builder.cpp

@@ -0,0 +1,178 @@
+#include "pdfengine/document/document_builder.hpp"
+#include <algorithm>
+#include <cmath>
+
+namespace pdfengine::document {
+
+PageModel DocumentBuilder::buildFromOCR(const RawOCRPage& rawPage) const {
+    PageModel model;
+    model.pageIndex = rawPage.pageIndex;
+    model.width = rawPage.pageWidth;
+    model.height = rawPage.pageHeight;
+
+    if (rawPage.lines.empty()) {
+        return model;
+    }
+
+    // Standard baseline clustering to group lines into Paragraphs
+    std::vector<Paragraph> paragraphs;
+    Paragraph currentPara;
+    double lastLineY = -1.0;
+    double lastLineH = 0.0;
+
+    for (size_t lIdx = 0; lIdx < rawPage.lines.size(); ++lIdx) {
+        const auto& rawLine = rawPage.lines[lIdx];
+
+        TextLine line;
+        line.x = rawLine.x;
+        line.y = rawLine.y;
+        line.w = rawLine.width;
+        line.h = rawLine.height;
+        line.baselineY = rawLine.baselineY;
+        line.angle = 0.0;
+
+        TextRun run;
+        run.text = rawLine.text;
+        run.fontName = rawLine.fontName.empty() ? "Helvetica" : rawLine.fontName;
+
+        bool isBold = rawLine.isBold || rawLine.fontWeight >= 600;
+        bool isItalic = rawLine.isItalic || rawLine.fontStyle == "italic";
+
+        run.flags = (isBold ? 2 : 0) | (isItalic ? 1 : 0);
+        run.fontWeight = rawLine.fontWeight > 0 ? rawLine.fontWeight : (isBold ? 700 : 400);
+        run.fontStyle = !rawLine.fontStyle.empty() ? rawLine.fontStyle : (isItalic ? "italic" : "normal");
+
+        if (!rawLine.fontFace.empty()) {
+            run.fontFace = rawLine.fontFace;
+        } else if (isBold && isItalic) {
+            run.fontFace = run.fontName + "-BoldItalic";
+        } else if (isBold) {
+            run.fontFace = run.fontName + "-Bold";
+        } else if (isItalic) {
+            run.fontFace = run.fontName + "-Italic";
+        } else {
+            run.fontFace = run.fontName;
+        }
+
+        run.internalFontId = !rawLine.fontId.empty()
+            ? rawLine.fontId
+            : (run.fontName + "_TrueType_" + std::to_string(run.fontWeight));
+
+        run.fontSize = rawLine.fontSize > 0.0 ? rawLine.fontSize : (rawLine.height * 0.72);
+        run.x = rawLine.x;
+        run.y = rawLine.y;
+        run.w = rawLine.width;
+        run.h = rawLine.height;
+        run.fillColor = "#000000";
+        run.fontFidelity = "exact";
+
+        // Generate Glyphs
+        if (!rawLine.words.empty()) {
+            for (const auto& w : rawLine.words) {
+                double charW = w.width / std::max<size_t>(1, w.text.size());
+                double currX = w.x;
+                for (char ch : w.text) {
+                    Glyph g;
+                    g.text = std::string(1, ch);
+                    g.unicode = static_cast<uint32_t>(static_cast<unsigned char>(ch));
+                    g.fontName = run.fontName;
+                    g.fontSize = run.fontSize;
+                    g.flags = run.flags;
+                    g.originX = currX;
+                    g.originY = rawLine.baselineY;
+                    g.bboxX = currX;
+                    g.bboxY = w.y;
+                    g.bboxW = charW;
+                    g.bboxH = w.height;
+                    run.glyphs.push_back(g);
+                    line.glyphs.push_back(g);
+                    currX += charW;
+                }
+            }
+        } else {
+            double charW = rawLine.width / std::max<size_t>(1, rawLine.text.size());
+            double currX = rawLine.x;
+            for (char ch : rawLine.text) {
+                Glyph g;
+                g.text = std::string(1, ch);
+                g.unicode = static_cast<uint32_t>(static_cast<unsigned char>(ch));
+                g.fontName = run.fontName;
+                g.fontSize = run.fontSize;
+                g.flags = run.flags;
+                g.originX = currX;
+                g.originY = rawLine.baselineY;
+                g.bboxX = currX;
+                g.bboxY = rawLine.y;
+                g.bboxW = charW;
+                g.bboxH = rawLine.height;
+                run.glyphs.push_back(g);
+                line.glyphs.push_back(g);
+                currX += charW;
+            }
+        }
+
+        line.runs.push_back(run);
+
+        // Check paragraph break (vertical gap > 1.5x line height)
+        bool isNewPara = false;
+        if (lastLineY >= 0.0) {
+            double gap = rawLine.y - (lastLineY + lastLineH);
+            if (gap > lastLineH * 1.5 || gap < -lastLineH * 0.5) {
+                isNewPara = true;
+            }
+        }
+
+        if (isNewPara && !currentPara.lines.empty()) {
+            // Finalize current paragraph bounds
+            double pX1 = currentPara.lines.front().x;
+            double pY1 = currentPara.lines.front().y;
+            double pX2 = pX1 + currentPara.lines.front().w;
+            double pY2 = pY1 + currentPara.lines.front().h;
+
+            for (const auto& l : currentPara.lines) {
+                pX1 = std::min(pX1, l.x);
+                pY1 = std::min(pY1, l.y);
+                pX2 = std::max(pX2, l.x + l.w);
+                pY2 = std::max(pY2, l.y + l.h);
+            }
+
+            currentPara.x = pX1;
+            currentPara.y = pY1;
+            currentPara.w = pX2 - pX1;
+            currentPara.h = pY2 - pY1;
+
+            paragraphs.push_back(currentPara);
+            currentPara = Paragraph();
+        }
+
+        currentPara.lines.push_back(line);
+        lastLineY = rawLine.y;
+        lastLineH = rawLine.height;
+    }
+
+    if (!currentPara.lines.empty()) {
+        double pX1 = currentPara.lines.front().x;
+        double pY1 = currentPara.lines.front().y;
+        double pX2 = pX1 + currentPara.lines.front().w;
+        double pY2 = pY1 + currentPara.lines.front().h;
+
+        for (const auto& l : currentPara.lines) {
+            pX1 = std::min(pX1, l.x);
+            pY1 = std::min(pY1, l.y);
+            pX2 = std::max(pX2, l.x + l.w);
+            pY2 = std::max(pY2, l.y + l.h);
+        }
+
+        currentPara.x = pX1;
+        currentPara.y = pY1;
+        currentPara.w = pX2 - pX1;
+        currentPara.h = pY2 - pY1;
+
+        paragraphs.push_back(currentPara);
+    }
+
+    model.paragraphs = std::move(paragraphs);
+    return model;
+}
+
+} // namespace pdfengine::document

engine/src/document/document_normalizer.cpp

@@ -0,0 +1,55 @@
+#include "pdfengine/document/document_normalizer.hpp"
+#include <algorithm>
+#include <cmath>
+
+namespace pdfengine::document {
+
+RawOCRPage DocumentNormalizer::normalize(const RawOCRPage& rawPage) const {
+    RawOCRPage page = rawPage;
+
+    if (page.pageWidth <= 0.0) page.pageWidth = page.imageWidth > 0.0 ? page.imageWidth : 612.0;
+    if (page.pageHeight <= 0.0) page.pageHeight = page.imageHeight > 0.0 ? page.imageHeight : 792.0;
+
+    double scaleX = 1.0;
+    double scaleY = 1.0;
+
+    if (page.imageWidth > 0.0 && std::abs(page.imageWidth - page.pageWidth) > 0.01) {
+        scaleX = page.pageWidth / page.imageWidth;
+    }
+    if (page.imageHeight > 0.0 && std::abs(page.imageHeight - page.pageHeight) > 0.01) {
+        scaleY = page.pageHeight / page.imageHeight;
+    }
+
+    for (auto& line : page.lines) {
+        line.x = std::max(0.0, line.x * scaleX);
+        line.y = std::max(0.0, line.y * scaleY);
+        line.width = std::max(1.0, line.width * scaleX);
+        line.height = std::max(1.0, line.height * scaleY);
+
+        if (line.fontSize <= 0.0) {
+            line.fontSize = line.height * 0.72;
+        }
+
+        if (line.baselineY <= 0.0) {
+            line.baselineY = line.y + line.fontSize;
+        } else {
+            line.baselineY *= scaleY;
+        }
+
+        for (auto& word : line.words) {
+            word.x = std::max(0.0, word.x * scaleX);
+            word.y = std::max(0.0, word.y * scaleY);
+            word.width = std::max(0.5, word.width * scaleX);
+            word.height = std::max(0.5, word.height * scaleY);
+
+            for (auto& pt : word.polygon) {
+                pt.first *= scaleX;
+                pt.second *= scaleY;
+            }
+        }
+    }
+
+    return page;
+}
+
+} // namespace pdfengine::document

engine/src/document/document_validator.cpp

@@ -0,0 +1,36 @@
+#include "pdfengine/document/document_validator.hpp"
+
+namespace pdfengine::document {
+
+ValidationResult DocumentValidator::validate(const PageModel& model) const {
+    ValidationResult res;
+
+    if (model.width <= 0.0 || model.height <= 0.0) {
+        res.issues.push_back({ValidationIssue::Severity::Error, "Page width or height is invalid/negative"});
+        res.isValid = false;
+    }
+
+    for (size_t pIdx = 0; pIdx < model.paragraphs.size(); ++pIdx) {
+        const auto& para = model.paragraphs[pIdx];
+        if (para.w < 0.0 || para.h < 0.0) {
+            res.issues.push_back({ValidationIssue::Severity::Warning, "Paragraph " + std::to_string(pIdx) + " has negative dimensions"});
+        }
+
+        for (size_t lIdx = 0; lIdx < para.lines.size(); ++lIdx) {
+            const auto& line = para.lines[lIdx];
+            if (line.w < 0.0 || line.h < 0.0) {
+                res.issues.push_back({ValidationIssue::Severity::Warning, "TextLine has negative dimensions"});
+            }
+
+            for (const auto& run : line.runs) {
+                if (run.fontSize <= 0.0) {
+                    res.issues.push_back({ValidationIssue::Severity::Warning, "TextRun fontSize <= 0.0"});
+                }
+            }
+        }
+    }
+
+    return res;
+}
+
+} // namespace pdfengine::document

engine/src/fonts/pdf_fonts/embedded_font_reconstructor.cpp

@@ -1,6 +1,7 @@
 #include "fonts/pdf_fonts/embedded_font_reconstructor.hpp"
 
 #include <map>
+#include <spdlog/spdlog.h>
 
 #include "fonts/pdf_fonts/font_cmap_builder.hpp"
 
@@ -34,7 +35,7 @@ ReconstructedFont EmbeddedFontReconstructor::reconstruct(
     if (cmap.empty()) return out;
     std::vector<uint8_t> sfnt = FontCmapBuilder::spliceCmapIntoSfnt(program, cmap);
     if (sfnt.empty()) return out;
-
+    spdlog::info("[RECONSTRUCTOR] Input={} Output={}", program.size(), sfnt.size());
     out.sfnt = std::move(sfnt);
     for (const auto& [uni, gid] : unicodeToGid) out.coveredUnicode.insert(uni);
     out.ok = true;

engine/src/fonts/pdf_fonts/font_extraction_service.cpp

@@ -0,0 +1,49 @@
+#include "font_extraction_service.hpp"
+#include "font_validator.hpp"
+#include "../../qpdf/qpdf_font_extractor.hpp"
+#include <spdlog/spdlog.h>
+
+namespace pdfengine::fonts::pdf_fonts {
+
+std::expected<std::shared_ptr<EmbeddedFontProgram>, EngineError>
+FontExtractionService::getFontProgram(const std::string& internalFontId,
+                                      const std::string& baseFontName,
+                                      const std::vector<uint8_t>& documentBuffer,
+                                      const std::vector<uint8_t>& pdfiumRawBytes) {
+    
+    std::lock_guard<std::mutex> lock(cacheMutex_);
+    if (cache_.count(internalFontId)) {
+        return cache_[internalFontId];
+    }
+
+    // 1. Try PDFium Bytes (Standard fonts)
+    if (!pdfiumRawBytes.empty() && FontValidator::isValidSFNT(pdfiumRawBytes)) {
+        auto prog = std::make_shared<EmbeddedFontProgram>();
+        prog->fontName = baseFontName;
+        prog->bytes = pdfiumRawBytes;
+        prog->source = FontExtractionSource::PDFium;
+        
+        cache_[internalFontId] = prog;
+        spdlog::info("[FontExtraction] internalId='{}' source=PDFium size={}", internalFontId, prog->bytes.size());
+        return prog;
+    }
+
+    // 2. Try QPDF Fallback (Type0 CIDFonts)
+    if (!documentBuffer.empty()) {
+        qpdf_layer::QpdfFontExtractor qpdfExtractor;
+        auto qpdfProg = qpdfExtractor.extractFontProgram(documentBuffer, baseFontName);
+        if (qpdfProg && FontValidator::isValidSFNT(qpdfProg->bytes)) {
+            auto prog = std::make_shared<EmbeddedFontProgram>(*qpdfProg);
+            prog->source = FontExtractionSource::QPDF_Raw;
+            cache_[internalFontId] = prog;
+            spdlog::info("[FontExtraction] internalId='{}' source=QPDF_Raw size={}", internalFontId, prog->bytes.size());
+            return prog;
+        }
+    }
+
+    // Tier 2 Reconstruction would happen here if needed, but for now we return not found
+    spdlog::warn("[FontExtraction] Failed to extract valid SFNT for '{}'", internalFontId);
+    return std::unexpected(EngineError::FileNotFound);
+}
+
+} // namespace pdfengine::fonts::pdf_fonts

engine/src/fonts/pdf_fonts/font_extraction_service.hpp

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <expected>
+
+#include "font_extraction_types.hpp"
+#include <pdfengine/pdf_document.hpp>
+
+namespace pdfengine::fonts::pdf_fonts {
+
+class FontExtractionService {
+public:
+    static FontExtractionService& getInstance() {
+        static FontExtractionService instance;
+        return instance;
+    }
+
+    std::expected<std::shared_ptr<EmbeddedFontProgram>, EngineError>
+    getFontProgram(const std::string& internalFontId,
+                   const std::string& baseFontName,
+                   const std::vector<uint8_t>& documentBuffer,
+                   const std::vector<uint8_t>& pdfiumRawBytes);
+
+private:
+    FontExtractionService() = default;
+
+    std::mutex cacheMutex_;
+    std::unordered_map<std::string, std::shared_ptr<EmbeddedFontProgram>> cache_;
+};
+
+} // namespace pdfengine::fonts::pdf_fonts

engine/src/fonts/pdf_fonts/font_extraction_types.hpp

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace pdfengine::fonts::pdf_fonts {
+
+enum class FontProgramType { Unknown, TrueType, CFF, OpenType, Type1 };
+
+enum class FontExtractionSource { PDFium, QPDF_Raw, QPDF_Reconstructed, Cache };
+
+struct EmbeddedFontProgram {
+    std::string fontName;
+    std::vector<uint8_t> bytes;
+    FontProgramType type = FontProgramType::Unknown;
+    FontExtractionSource source = FontExtractionSource::PDFium;
+};
+
+} // namespace pdfengine::fonts::pdf_fonts

engine/src/fonts/pdf_fonts/font_validator.cpp

@@ -0,0 +1,28 @@
+#include "font_validator.hpp"
+
+namespace pdfengine::fonts::pdf_fonts {
+
+bool FontValidator::isValidSFNT(const std::vector<uint8_t>& bytes) {
+    if (bytes.size() < 12) { // Minimum SFNT header size
+        return false;
+    }
+
+    // Check SFNT version / magic bytes
+    // 0x00010000 for TrueType
+    // 0x4F54544F ("OTTO") for OpenType CFF
+    // 0x74727565 ("true") for Apple TrueType
+    // 0x74797031 ("typ1") for Mac PostScript Type 1
+    const uint8_t* b = bytes.data();
+    uint32_t magic = (static_cast<uint32_t>(b[0]) << 24) |
+                     (static_cast<uint32_t>(b[1]) << 16) |
+                     (static_cast<uint32_t>(b[2]) << 8) |
+                      static_cast<uint32_t>(b[3]);
+
+    if (magic == 0x00010000 || magic == 0x4F54544F || magic == 0x74727565 || magic == 0x74797031) {
+        return true;
+    }
+
+    return false;
+}
+
+} // namespace pdfengine::fonts::pdf_fonts

engine/src/fonts/pdf_fonts/font_validator.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <cstdint>
+#include <vector>
+
+namespace pdfengine::fonts::pdf_fonts {
+
+class FontValidator {
+public:
+    static bool isValidSFNT(const std::vector<uint8_t>& bytes);
+};
+
+} // namespace pdfengine::fonts::pdf_fonts

engine/src/image/core/image_builder.cpp

@@ -0,0 +1,35 @@
+#include "image_builder.hpp"
+
+namespace pdfengine {
+
+std::shared_ptr<const ImageObject> ImageBuilder::build(const ResolvedImageDescriptor& descriptor,
+                                                        ImagePixelBuffer decodedPixels,
+                                                        ImageDiagnostics diagnostics,
+                                                        ImageStatistics statistics) {
+    ImageId id = generateNextImageId();
+
+    if (statistics.memoryBytes == 0) {
+        statistics.width = descriptor.geometry.width;
+        statistics.height = descriptor.geometry.height;
+        statistics.memoryBytes = decodedPixels.size();
+        statistics.compressedBytes = descriptor.originalStream.size();
+        statistics.decodedBytes = decodedPixels.size();
+        statistics.hasAlpha = descriptor.colorProfile.hasAlpha;
+        statistics.hasMask = descriptor.mask.hasMask();
+    }
+
+    return std::make_shared<const ImageObject>(
+        id,
+        descriptor.resourceName,
+        descriptor.geometry,
+        descriptor.encoding,
+        descriptor.colorProfile,
+        std::move(decodedPixels),
+        descriptor.mask,
+        descriptor.originalStream,
+        diagnostics,
+        statistics
+    );
+}
+
+} // namespace pdfengine

engine/src/image/core/image_builder.hpp

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <pdfengine/image_object.hpp>
+#include <pdfengine/image_descriptor.hpp>
+#include <memory>
+
+namespace pdfengine {
+
+class ImageBuilder {
+public:
+    static std::shared_ptr<const ImageObject> build(const ResolvedImageDescriptor& descriptor,
+                                                    ImagePixelBuffer decodedPixels,
+                                                    ImageDiagnostics diagnostics = {},
+                                                    ImageStatistics statistics = {});
+};
+
+} // namespace pdfengine

engine/src/image/core/image_manager.cpp

@@ -0,0 +1,42 @@
+#include <pdfengine/image_manager.hpp>
+#include <pdfengine/image_validator.hpp>
+#include <pdfengine/image_pipeline.hpp>
+
+namespace pdfengine {
+
+ImageManager& ImageManager::instance() {
+    static ImageManager mgr;
+    return mgr;
+}
+
+void ImageManager::setCache(std::shared_ptr<IImageCache> cache) {
+    m_cache = std::move(cache);
+}
+
+std::shared_ptr<IImageCache> ImageManager::cache() const {
+    return m_cache;
+}
+
+std::shared_ptr<const ImageObject> ImageManager::processImageDescriptor(const ResolvedImageDescriptor& descriptor,
+                                                                         std::string* outError) {
+    // Stage B validation before memory allocation
+    if (!ImageValidator::validate(descriptor, outError)) {
+        return nullptr;
+    }
+
+    if (m_cache) {
+        std::string key = std::to_string(descriptor.objectNumber) + "_" + std::to_string(descriptor.generationNumber);
+        if (m_cache->has(key)) {
+            return m_cache->get(key);
+        }
+        auto img = ImagePipeline::process(descriptor);
+        if (img) {
+            m_cache->put(key, img);
+        }
+        return img;
+    }
+
+    return ImagePipeline::process(descriptor);
+}
+
+} // namespace pdfengine

engine/src/image/core/image_object.cpp

@@ -0,0 +1,12 @@
+#include <pdfengine/image_object.hpp>
+#include <atomic>
+
+namespace pdfengine {
+
+static std::atomic<uint64_t> s_nextImageId{1};
+
+ImageId generateNextImageId() {
+    return s_nextImageId.fetch_add(1, std::memory_order_relaxed);
+}
+
+} // namespace pdfengine

engine/src/image/core/image_pipeline.cpp

@@ -0,0 +1,43 @@
+#include <pdfengine/image_pipeline.hpp>
+#include <pdfengine/image_decoder.hpp>
+#include "image_builder.hpp"
+#include <chrono>
+
+namespace pdfengine {
+
+std::shared_ptr<const ImageObject> ImagePipeline::process(const ResolvedImageDescriptor& descriptor) {
+    if (!descriptor.isValid()) {
+        return nullptr;
+    }
+
+    auto start = std::chrono::high_resolution_clock::now();
+
+    ImagePixelBuffer decodedPixels = ImageDecoder::decode(descriptor);
+    if (decodedPixels.empty()) {
+        return nullptr;
+    }
+
+    auto end = std::chrono::high_resolution_clock::now();
+    uint64_t elapsedUs = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+
+    ImageDiagnostics diag;
+    diag.decodeTimeUs = elapsedUs;
+    diag.originalBytes = descriptor.originalStream.size();
+    diag.decodedBytes = decodedPixels.size();
+    diag.fromCache = false;
+    diag.usedSoftMask = descriptor.mask.hasMask();
+
+    ImageStatistics stats;
+    stats.width = descriptor.geometry.width;
+    stats.height = descriptor.geometry.height;
+    stats.memoryBytes = decodedPixels.size();
+    stats.compressedBytes = descriptor.originalStream.size();
+    stats.decodedBytes = decodedPixels.size();
+    stats.hasAlpha = descriptor.colorProfile.hasAlpha;
+    stats.hasMask = descriptor.mask.hasMask();
+    stats.interpolated = false;
+
+    return ImageBuilder::build(descriptor, std::move(decodedPixels), diag, stats);
+}
+
+} // namespace pdfengine

engine/src/image/core/image_validator.cpp

@@ -0,0 +1,30 @@
+#include <pdfengine/image_validator.hpp>
+#include <pdfengine/hardened_limits.h>
+#include <sstream>
+
+namespace pdfengine {
+
+bool ImageValidator::validate(const ImageGeometry& geometry, std::string* outError) {
+    if (geometry.width <= 0 || geometry.height <= 0) {
+        if (outError) *outError = "Image dimensions must be positive.";
+        return false;
+    }
+
+    if (!limits::rasterSizeOk(geometry.width, geometry.height)) {
+        if (outError) {
+            std::ostringstream ss;
+            ss << "Image dimensions (" << geometry.width << "x" << geometry.height
+               << ") exceed hardened limits (kMaxRasterPixels=" << limits::kMaxRasterPixels << ").";
+            *outError = ss.str();
+        }
+        return false;
+    }
+
+    return true;
+}
+
+bool ImageValidator::validate(const ResolvedImageDescriptor& descriptor, std::string* outError) {
+    return validate(descriptor.geometry, outError);
+}
+
+} // namespace pdfengine

engine/src/image/decoder/color_converter.cpp

@@ -0,0 +1,76 @@
+#include "color_converter.hpp"
+#include <algorithm>
+#include <cmath>
+
+namespace pdfengine {
+
+ImagePixelBuffer ColorConverter::convertToRgba(const ImagePixelBuffer& input,
+                                               ColorSpaceType colorSpace,
+                                               int width,
+                                               int height) {
+    if (width <= 0 || height <= 0 || input.empty()) {
+        return {};
+    }
+
+    ImagePixelBuffer rgbaBuffer(width, height, 4);
+    const uint8_t* src = input.data();
+    const size_t totalPixels = static_cast<size_t>(width) * height;
+
+    if (colorSpace == ColorSpaceType::DeviceGray || input.channels() == 1) {
+        for (size_t i = 0; i < totalPixels; ++i) {
+            uint8_t gray = (i < input.size()) ? src[i] : 0;
+            rgbaBuffer.data()[i * 4 + 0] = gray;
+            rgbaBuffer.data()[i * 4 + 1] = gray;
+            rgbaBuffer.data()[i * 4 + 2] = gray;
+            rgbaBuffer.data()[i * 4 + 3] = 255;
+        }
+    } else if (colorSpace == ColorSpaceType::DeviceRGB || input.channels() == 3) {
+        for (size_t i = 0; i < totalPixels; ++i) {
+            size_t srcIdx = i * 3;
+            if (srcIdx + 2 < input.size()) {
+                rgbaBuffer.data()[i * 4 + 0] = src[srcIdx + 0];
+                rgbaBuffer.data()[i * 4 + 1] = src[srcIdx + 1];
+                rgbaBuffer.data()[i * 4 + 2] = src[srcIdx + 2];
+            } else {
+                rgbaBuffer.data()[i * 4 + 0] = 0;
+                rgbaBuffer.data()[i * 4 + 1] = 0;
+                rgbaBuffer.data()[i * 4 + 2] = 0;
+            }
+            rgbaBuffer.data()[i * 4 + 3] = 255;
+        }
+    } else if (colorSpace == ColorSpaceType::DeviceCMYK || input.channels() == 4) {
+        for (size_t i = 0; i < totalPixels; ++i) {
+            size_t srcIdx = i * 4;
+            if (srcIdx + 3 < input.size()) {
+                float c = src[srcIdx + 0] / 255.0f;
+                float m = src[srcIdx + 1] / 255.0f;
+                float y = src[srcIdx + 2] / 255.0f;
+                float k = src[srcIdx + 3] / 255.0f;
+
+                auto applyInk = [](float paper, float processColor, float amount) {
+                    return paper * ((1.0f - amount) + amount * (processColor / 255.0f));
+                };
+
+                float r = 255.0f, g = 255.0f, b = 255.0f;
+                r = applyInk(r, 0.0f, c);   g = applyInk(g, 174.0f, c); b = applyInk(b, 239.0f, c);
+                r = applyInk(r, 237.0f, m); g = applyInk(g, 0.0f, m);   b = applyInk(b, 140.0f, m);
+                r = applyInk(r, 255.0f, y); g = applyInk(g, 241.0f, y); b = applyInk(b, 0.0f, y);
+                r = applyInk(r, 35.0f, k);  g = applyInk(g, 31.0f, k);  b = applyInk(b, 32.0f, k);
+
+                rgbaBuffer.data()[i * 4 + 0] = static_cast<uint8_t>(std::clamp(std::lround(r), 0l, 255l));
+                rgbaBuffer.data()[i * 4 + 1] = static_cast<uint8_t>(std::clamp(std::lround(g), 0l, 255l));
+                rgbaBuffer.data()[i * 4 + 2] = static_cast<uint8_t>(std::clamp(std::lround(b), 0l, 255l));
+                rgbaBuffer.data()[i * 4 + 3] = 255;
+            }
+        }
+    } else {
+        // Fallback for unsupported or 4-channel pass-through
+        if (input.size() == rgbaBuffer.size()) {
+            std::copy(input.data(), input.data() + input.size(), rgbaBuffer.data());
+        }
+    }
+
+    return rgbaBuffer;
+}
+
+} // namespace pdfengine

engine/src/image/decoder/color_converter.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <pdfengine/image_pixel_buffer.hpp>
+#include <pdfengine/image_color_profile.hpp>
+
+namespace pdfengine {
+
+class ColorConverter {
+public:
+    static ImagePixelBuffer convertToRgba(const ImagePixelBuffer& input,
+                                          ColorSpaceType colorSpace,
+                                          int width,
+                                          int height);
+};
+
+} // namespace pdfengine

engine/src/image/decoder/filter_decoder.cpp

@@ -0,0 +1,165 @@
+#include "filter_decoder.hpp"
+#include <jpeglib.h>
+#include <zlib.h>
+#include <csetjmp>
+#include <iostream>
+#include <algorithm>
+
+namespace {
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable: 4324)
+#endif
+struct JpegErrorManager {
+    jpeg_error_mgr pub;
+    std::jmp_buf jump;
+};
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+void jpegErrorExit(j_common_ptr cinfo) {
+    auto* manager = reinterpret_cast<JpegErrorManager*>(cinfo->err);
+    longjmp(manager->jump, 1);
+}
+
+} // namespace
+
+namespace pdfengine {
+
+bool FilterDecoder::isSupportedFilter(const std::string& filterName) {
+    return filterName.empty() ||
+           filterName == "/DCTDecode" || filterName == "DCTDecode" ||
+           filterName == "/FlateDecode" || filterName == "FlateDecode" ||
+           filterName == "/JPXDecode" || filterName == "JPXDecode" ||
+           filterName == "/CCITTFaxDecode" || filterName == "CCITTFaxDecode" ||
+           filterName == "/RunLengthDecode" || filterName == "RunLengthDecode" ||
+           filterName == "/LZWDecode" || filterName == "LZWDecode" ||
+           filterName == "/ASCII85Decode" || filterName == "ASCII85Decode" ||
+           filterName == "/ASCIIHexDecode" || filterName == "ASCIIHexDecode";
+}
+
+std::vector<uint8_t> FilterDecoder::decode(const std::vector<uint8_t>& rawStream,
+                                           const std::string& filterName) {
+    if (rawStream.empty()) {
+        return {};
+    }
+
+    if (filterName == "/DCTDecode" || filterName == "DCTDecode") {
+        return decodeJpeg(rawStream);
+    }
+
+    if (filterName == "/FlateDecode" || filterName == "FlateDecode") {
+        return decodeFlate(rawStream);
+    }
+
+    if (filterName.empty() || filterName == "None") {
+        return rawStream;
+    }
+
+    // Reserved filter stubs
+    if (filterName == "/JPXDecode" || filterName == "/CCITTFaxDecode" ||
+        filterName == "/RunLengthDecode" || filterName == "/LZWDecode" ||
+        filterName == "/ASCII85Decode" || filterName == "/ASCIIHexDecode") {
+        std::cerr << "Warning: Filter " << filterName << " registered but not fully implemented, passing raw stream.\n";
+        return rawStream;
+    }
+
+    return rawStream;
+}
+
+std::vector<uint8_t> FilterDecoder::decodeJpeg(const std::vector<uint8_t>& jpegBytes) {
+    if (jpegBytes.empty()) {
+        return {};
+    }
+
+    jpeg_decompress_struct cinfo{};
+    JpegErrorManager jerr{};
+    cinfo.err = jpeg_std_error(&jerr.pub);
+    jerr.pub.error_exit = jpegErrorExit;
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable: 4611)
+#endif
+    if (setjmp(jerr.jump)) {
+        jpeg_destroy_decompress(&cinfo);
+        std::cerr << "Error: Failed to decode JPEG image stream\n";
+        return {};
+    }
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+    jpeg_create_decompress(&cinfo);
+    jpeg_mem_src(&cinfo, const_cast<unsigned char*>(jpegBytes.data()),
+                 static_cast<unsigned long>(jpegBytes.size()));
+
+    jpeg_read_header(&cinfo, TRUE);
+
+    const bool cmykSource = cinfo.jpeg_color_space == JCS_CMYK ||
+                            cinfo.jpeg_color_space == JCS_YCCK;
+    cinfo.out_color_space = cmykSource ? JCS_CMYK : JCS_RGB;
+
+    jpeg_start_decompress(&cinfo);
+
+    const int width = static_cast<int>(cinfo.output_width);
+    const int height = static_cast<int>(cinfo.output_height);
+    const int components = static_cast<int>(cinfo.output_components);
+    const int rowStride = width * components;
+
+    std::vector<uint8_t> decodedData(static_cast<size_t>(width) * height * components);
+    std::vector<JSAMPLE> row(static_cast<size_t>(rowStride));
+
+    while (cinfo.output_scanline < cinfo.output_height) {
+        JSAMPROW rowPointer = row.data();
+        const int y = static_cast<int>(cinfo.output_scanline);
+        jpeg_read_scanlines(&cinfo, &rowPointer, 1);
+        std::copy(row.begin(), row.end(), decodedData.begin() + static_cast<size_t>(y) * rowStride);
+    }
+
+    jpeg_finish_decompress(&cinfo);
+    jpeg_destroy_decompress(&cinfo);
+    return decodedData;
+}
+
+std::vector<uint8_t> FilterDecoder::decodeFlate(const std::vector<uint8_t>& flateBytes) {
+    if (flateBytes.empty()) {
+        return {};
+    }
+
+    z_stream strm{};
+    strm.next_in = const_cast<Bytef*>(flateBytes.data());
+    strm.avail_in = static_cast<uInt>(flateBytes.size());
+
+    if (inflateInit(&strm) != Z_OK) {
+        return {};
+    }
+
+    std::vector<uint8_t> decompressed;
+    decompressed.resize(flateBytes.size() * 4 + 1024);
+
+    strm.next_out = decompressed.data();
+    strm.avail_out = static_cast<uInt>(decompressed.size());
+
+    int res = inflate(&strm, Z_NO_FLUSH);
+    while (res == Z_OK && strm.avail_out == 0) {
+        size_t currentSize = decompressed.size();
+        decompressed.resize(currentSize * 2);
+        strm.next_out = decompressed.data() + currentSize;
+        strm.avail_out = static_cast<uInt>(currentSize);
+        res = inflate(&strm, Z_NO_FLUSH);
+    }
+
+    if (res == Z_STREAM_END || res == Z_OK) {
+        decompressed.resize(strm.total_out);
+        inflateEnd(&strm);
+        return decompressed;
+    }
+
+    inflateEnd(&strm);
+    return {};
+}
+
+} // namespace pdfengine

engine/src/image/decoder/filter_decoder.hpp

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <cstdint>
+
+namespace pdfengine {
+
+class FilterDecoder {
+public:
+    static std::vector<uint8_t> decode(const std::vector<uint8_t>& rawStream,
+                                       const std::string& filterName);
+
+    static bool isSupportedFilter(const std::string& filterName);
+
+private:
+    static std::vector<uint8_t> decodeJpeg(const std::vector<uint8_t>& jpegBytes);
+    static std::vector<uint8_t> decodeFlate(const std::vector<uint8_t>& flateBytes);
+};
+
+} // namespace pdfengine