maskantech/pdf
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

pulled from zaid

Browse Source
This commit is contained in:
azeeee05
2026-09-09 12:57:11 +05:30
parent d3cdf0a441 a954df0c05
commit 9ff1d7ff18
249 changed files with 55733 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+42 -2
View File
@@ -21,7 +21,6 @@ CMakeUserPresets.json
/third_party/pdfium/checkout/
/third_party/pdfium/install/
/third_party/pdfium/.gclient*
/corpus/
# Skia from-source build (depot_tools / GN / Ninja)
/third_party/skia/depot_tools/
@@ -68,8 +67,45 @@ gateway/*.pyd
gateway/*.so
gateway/*.dylib
# Downloaded fuzzing corpus (large; fetched via scripts/fetch_corpus.py)
# Corpora — keep convert metadata; ignore large samples / archives
/corpus/*
!/corpus/convert/
/corpus/convert/**
!/corpus/convert/real/
!/corpus/convert/public/
!/corpus/convert/third_party/
/corpus/convert/real/**
!/corpus/convert/real/pack.json
!/corpus/convert/real/*.md
/corpus/convert/public/**
!/corpus/convert/public/README.md
!/corpus/convert/public/pack_public.json
!/corpus/convert/public/manifests/
!/corpus/convert/public/manifests/**
!/corpus/convert/public/reports/
!/corpus/convert/public/reports/**/*.md
!/corpus/convert/public/reports/**/*.json
/corpus/convert/public/samples/
/corpus/convert/third_party/**
!/corpus/convert/third_party/NOTICE
!/corpus/convert/third_party/LICENSE_CHECKLIST.md
!/corpus/convert/third_party/checksums.sha256
corpus/fuzz/
# Layout ML weights (download via scripts/convert/fetch_layout_onnx.py)
/gateway/models/layout/**/*.onnx
/gateway/models/layout/**/*.pdmodel
/gateway/models/layout/**/*.pdiparams
!/gateway/models/layout/**/README.md
!/gateway/models/layout/**/.gitkeep
# Arabic OCR rec weights (download via scripts/convert/fetch_ocr_arabic_onnx.py)
/gateway/models/ocr/**/*.onnx
!/gateway/models/ocr/**/README.md
!/gateway/models/ocr/**/.gitkeep
!/gateway/models/ocr/**/*.txt
# DocLayNet / Pub* lab archives
**/DocLayNet*.zip
**/PubLayNet*
**/PubTabNet*
# Large-corpus regression baseline (derived from the gitignored corpus above)
tests/regression/baseline-large/
# Fuzzer working artifacts (crashes, leaks, coverage)
@@ -79,6 +115,10 @@ leak-*
timeout-*
PDF Editor Timeline.xlsx
# Operator convert score log (Phase 0); never commit customer DOCX
convert-scores.jsonl
**/convert-scores.jsonl
# Local environment config / secrets
/third_party/pdfium-wasm/
CMakeLists.txt
+5 -4
View File
@@ -73,9 +73,9 @@ find_package(freetype CONFIG REQUIRED)
find_package(harfbuzz CONFIG REQUIRED)
find_package(spdlog CONFIG REQUIRED)
find_package(nlohmann_json CONFIG REQUIRED)
if(PDFENGINE_WITH_QPDF)
find_package(qpdf CONFIG REQUIRED)
endif()
find_package(qpdf CONFIG REQUIRED)
find_package(ZLIB REQUIRED)
find_package(JPEG REQUIRED)
if(WIN32 AND DEFINED VCPKG_TARGET_TRIPLET)
link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/lib")
link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/debug/lib")
@@ -110,4 +110,5 @@ message(STATUS " Build tests .......... ${PDFENGINE_BUILD_TESTS}")
message(STATUS " Sanitizers ........... ${PDFENGINE_ENABLE_SANITIZERS}")
message(STATUS " Warnings as errors ... ${PDFENGINE_WARNINGS_AS_ERRORS}")
message(STATUS " Link PDFium .......... ${PDFENGINE_WITH_PDFIUM}")
message(STATUS " Link Skia ............ ${PDFENGINE_WITH_SKIA}")
message(STATUS " Link Skia ............ ${PDFENGINE_WITH_SKIA}")
message(STATUS " Link QPDF ............ ${PDFENGINE_WITH_QPDF}")
CMakeUserPresets.json.template
+4 -1
View File
@@ -17,7 +17,10 @@
"displayName": "Windows • RelWithDebInfo + PDFium (static CRT — build dir outside OneDrive/spaces)",
"inherits": "windows-release",
"binaryDir": "D:/pdfeng-build/win-local-pdfium",
"cacheVariables": { "PDFENGINE_WITH_PDFIUM": "ON" }
"cacheVariables": {
"PDFENGINE_WITH_PDFIUM": "ON",
"PDFENGINE_WITH_QPDF": "ON"
}
}
],
"buildPresets": [
CONVERSION_ENGINE_DEEP_DIVE.md
+517
View File
@@ -0,0 +1,517 @@
# Document Conversion Engine: End-to-End Deep Dive
**Investigation date:** 2026-09-09
**Scope:** `pdf/` repository, with emphasis on `gateway/app/services/convert`, the
PDF engine binding, OCR/layout models, writers, configuration, and conversion tests.
**Evidence standard:** real documents were passed through the public
`run_conversion` path and the resulting Office files were parsed and rendered.
Numbers below are measurements on the Windows development host, not vendor
claims or a declaration of production readiness.
## Current State
### Executive finding
The engine is a composed, mostly in-process conversion stack. It is effective for
simple digital PDFs, text extraction, Office document generation, and searchable
PDF creation. It is not a general PDF layout clone. A PDF has positioned drawing
operators, while DOCX/XLSX require a semantic flow/grid model; the conversion
therefore reconstructs an intermediate document model (IDM) and necessarily loses
some semantics and positioning.
The most visible historical defect, interleaved text on multi-column PDFs, had a
specific upstream cause and is fixed in the current tree. The remaining quality
ceiling is dominated by table reconstruction, OCR cost/ambiguity, and the choice
of ReportLab as the Office-to-PDF fallback. A successful HTTP response is not
equivalent to faithful content or layout: the measured corpus still contains
low-order similarity and three failing assertions.
### Execution topology
The current request path is:
```text
HTTP /v1/convert/*
-> filename/MIME/magic-byte/size validation
-> converter registry and optional automatic PDF policy
-> bounded worker admission (queue + shared conversion slots)
-> conversion/document-cache scopes
-> PDF route (digital, scanned, image-based, mixed)
-> per-page geometry and content recovery
C++ pdfengine ordered glyphs (preferred)
pypdf visitor/content-stream geometry (fallback)
line grouping -> paragraphs -> reading order -> tables/images
optional ONNX layout regions and table separators
-> OCR for scan/hybrid pages
PDF render -> RapidOCR Latin pass -> optional Arabic pass
confidence/script merge -> IDM blocks and metadata
-> IDM optimization (headers/footers, wrapped rows, figures)
-> format-specific formatter/writer
-> OOXML/PDF validity checks, quality score and warnings
-> content-addressed result cache
```
`run_conversion` binds cancellation, deadlines, options, automatic-routing
decisions, and a per-conversion read/raster/geometry cache. The public converter
registry currently covers PDF, DOCX, XLSX, PPTX, HTML, Markdown, text, CSV,
JSON, and image paths; the exact pair list is exposed by the convert info route.
### Component inventory
| Layer | Implementation in this tree | Observed role and limitation |
| --- | --- | --- |
| PDF parser/render core | C++23 `pdfengine` pybind module; PDFium, FreeType, HarfBuzz, optional Skia/QPDF | Gives authoritative glyphs, fonts, page geometry, rendering, and editing on this host. Native availability is not guaranteed on every Python deployment. |
| Python PDF parser | `pypdf==5.3.0` | Text extraction, page count, content-stream fallback, XObject inspection. Its inferred whitespace and broken ToUnicode output cannot be treated as ground truth. |
| Raster path | `pypdfium2==5.13.0` declared in `pyproject.toml` | Required for hosts without the C++ renderer. It is absent from the active venv, so raster-specific tests skip there; the C++ binding supplies rendering for the current audit. |
| OCR | `rapidocr-onnxruntime==1.2.3`, `onnxruntime==1.28.0` | Shared Latin detector/recognizer; optional PP-OCRv5 Arabic recognizer. Inference is serialized because RapidOCR sessions mutate shared state. |
| Layout ML | PP-DocLayoutV3-derived ONNX, 130,502,330 bytes FP32 or 34,694,580 bytes INT8 | Region labels narrow heuristic decisions; it does not reconstruct a document by itself. It runs on CPU in the measured environment. |
| Table ML | Optional `structure.onnx` (Table Transformer/SLANet operator-supplied) | Not vendored (`shipped: false` in `models/MANIFEST.json`); heuristic lattice/stream/rectangle detectors remain the default. |
| IDM | `app/services/convert/idm/model.py` and layout pipeline | Stores pages, BBoxes, paragraphs, headings, headers/footers, figures, and cell grids. It is a useful interchange contract, but not a complete PDF display list or OOXML style graph. |
| Office writers | `python-docx==1.1.2`, `openpyxl==3.1.5`, `python-pptx` code, XML sanitization | Good for creating valid Office files; cannot infer all original PDF semantics (fonts, fields, anchored objects, section behavior). |
| Office -> PDF | Gotenberg/LibreOffice attempt, then in-process ReportLab 4.2.5 | Gotenberg is best fidelity when reachable. The benchmark used the ReportLab fallback, which is concurrent-safe but not a Word/Excel print-layout clone. |
| Images | Pillow 10.4.0, PDFium render/XObject extraction | Images are extracted, resized/compacted, or the complete page is embedded when reconstruction would lose visible content. |
| Font recognition | `FontRecognitionService` facade, classifier and embedding store | The checked-in `FontClassifier` uses aspect ratio/dark-pixel heuristics and `EmbeddingStore` uses a deterministic 16x8 grayscale projection. `load_model()` reads metadata but does not load an OpenCLIP ONNX session; `model.onnx` and `fonts.index` are not present in the repository. This is metadata assistance, not a validated font-identification model. |
The declared dependency set is intentionally permissive (pypdf BSD, pypdfium2
Apache/BSD, python-docx/openpyxl MIT, ReportLab BSD, Pillow HPND). `mammoth==1.8.0`
is installed in this venv for DOCX/HTML fallback paths; `fitz`/PyMuPDF is absent,
which avoids an AGPL transitive dependency but removes that optional path.
### Runtime configuration and resource policy
The startup script defaults to automatic reconstruction, layout ML (INT8 when
available), Arabic OCR with adaptive gating, a 1,800 second maximum conversion
timeout, 22 seconds per OCR page budget, 45 seconds per-page timeout, a 1,200
second OCR wall budget, four conversion workers, four simultaneous conversion
slots, and a queue of roughly two batches. Result caching uses namespace
`conversion-engine-v2` and includes effective options and routing mode in the key.
These are protection mechanisms, not throughput guarantees. A single process can
still hold multiple ONNX sessions and large page rasters, and OCR inference is
deliberately serialized. Horizontal scaling or process isolation is needed for
untrusted large-document workloads.
### Jobs, storage, and failure boundaries
Synchronous and asynchronous routes share a process-wide `ThreadPoolExecutor`
and bounded semaphores. Asynchronous job state is persisted in SQLite with WAL
and `synchronous=NORMAL`; results larger than 512 KB are spooled to a local
`jobs_spool` directory, while smaller results are stored as BLOBs. Jobs older
than three hours are purged, and jobs left queued/running after a process restart
are marked failed. This gives useful single-node crash recovery, but SQLite and
local spool paths are not a multi-instance object store: replicas need a shared
database/object store or a queue service before horizontal scaling.
The timeout helper releases the HTTP caller on deadline by abandoning a worker
thread (`ThreadPoolExecutor.shutdown(wait=False)`). Python cannot kill that
thread, so a native render/OCR operation may continue in the background until it
returns. Cooperative cancellation is checked at page/OCR boundaries, which
limits normal work, but a hung native call can still consume CPU/RAM after a 504.
Production deployments should put untrusted conversions behind process/container
boundaries with an OS-level kill and cgroup limits.
Gotenberg/LibreOffice is attempted for Office-to-PDF with a 30-second client
timeout and fails open to ReportLab. This makes the endpoint available when the
microservice is absent, but it can silently change fidelity class unless the
response warning is surfaced to callers and monitored.
## Root Cause
The table below ranks causes by customer impact and likelihood in the measured
system. "Fixed" means the implementation changed and the targeted regression
improved; it does not mean the entire class of failures is solved.
| Rank | Severity | Root cause | Evidence | State |
| --- | --- | --- | --- | --- |
| 1 | P0 | Precise glyph coverage was compared with `len(pypdf.extract_text())`, which includes inferred spaces/newlines, while the precise walker emits text-showing characters only. Pages near the 0.80 gate fell into `_glyphs_from_visitor`, whose width calibration measured the two-column pitch rather than font advances. | Mozilla pages had coverage 0.754-0.858; stripping whitespace raised it to 0.879-1.000. Estimated spans crossed the gutter; the visible sentence was interleaved. | Fixed by comparing non-whitespace characters and retaining the precise path. |
| 2 | P0 | Baseline clustering was page-global. Different leading in adjacent columns let rows "bridge" into one bucket; splitting columns later could not undo the vertical weld. | On the paper page, 6.97 pt copyright rows and 8.97 pt body rows were interleaved; 4.6 pt and 3.3 pt hops were under the global tolerance. Reading-order inversions fell from 189 to 1 after column-aware bands. | Fixed with `_column_band` and a bounded baseline clip. |
| 3 | P0 | OCR is CPU-heavy and each bilingual page can run Latin and Arabic recognition, plus per-line/per-word font metadata work. A shared RapidOCR session cannot safely run concurrently. | 20-page scan: 142.747 s wall, 655.016 CPU-s, 1,769.4 MB peak RSS. Warm one-page inference was 3.021 s/17.318 CPU-s with Arabic versus 1.370 s/7.943 CPU-s with adaptive skip. | Partially fixed: inference/model locks, adaptive Arabic, page caps, budgets, and raster reuse are present. |
| 4 | P1 | Table reconstruction is heuristic by default; ruled headers and multi-line cells are represented as fragments before grid assignment. The optional structure model is not shipped. | W3C exact table gate produced recall 0.867 (<0.90). `Participants` became `s Participant`; `Time to complete` became `Results Time to complete`; `n=1` gained a space after `=`. | Open; targeted fix is required. |
| 5 | P1 | PDF-to-editable-Office is a semantic reflow, not a pixel-preserving transform. Header/footer detection, style inference, anchored objects, columns, fields, and arbitrary positioned graphics have no one-to-one OOXML representation. | Pixel similarity varies from 0.784 (USGS) to 0.950 (govinfo) in the ReportLab-rendered comparison; order scores are 0.232-0.674 for several digital PDFs. | Fundamental limitation; use page-image fallback or a fidelity backend when editability is not required. |
| 6 | P1 | OCR recognition is sensitive to source DPI, contrast, skew, script, and confidence thresholds. Low thresholds (0.08 box/text) preserve faint marks but also admit noise; post-processing must distinguish symbols from rules. | Scan recall is 1.000 against expected page tokens but precision is 0.077 because OCR also emits repeated body/table text. Arabic adaptive mode improves CPU cost but whole-run wall time is noisy. | Partially fixed; needs calibrated per-document confidence and preprocessing. |
| 7 | P1 | Quality scoring historically rewarded agreement with a corrupted pypdf text layer and ignored duplicates/order. Structure bonus could be earned by a diagram label classified as a header. | Mozilla score moved 0.9475 -> 0.8998 while the corrupted duplicate text disappeared and real prose improved. New metrics expose precision/order and store `idm.meta["text_metrics"]`. | Fixed for diagnostics; corpus gates still need independent gold/layout checks. |
| 8 | P2 | Repeated PDF reads, geometry extraction, rasterization, and model initialization inflated latency and memory. Unbounded cache keys also allowed stale results across option/routing changes. | Per-conversion document cache and canonical cache keys were added; regression suites cover collisions and queue admission. | Fixed/mitigated. |
| 9 | P2 | API contract mismatch unrelated to conversion: unknown font endpoint currently returns 204 while its test expects 404. | Full suite failure in `tests/test_inplace_editing.py::test_font_endpoint_rejects_unknown`. | Open decision for API owner. |
| 10 | P2 | Timeout release is thread-based, not process-based. A native renderer or OCR call that ignores cancellation can continue after the request receives 504; local SQLite/spool storage also limits multi-instance scaling. | `run_with_timeout()` explicitly uses `shutdown(wait=False)` because Python cannot kill a thread; job store uses one local SQLite file and local spool directory. No runaway crash was observed in the audit, but the resource behavior is a design risk. | Mitigate with process/container isolation and shared durable storage. |
### Content accuracy
PDF text is not necessarily a trustworthy source string. Embedded fonts can have
missing or malformed ToUnicode maps, CID encodings, zero-width markers, and text
show operators split in ways that do not correspond to words. The current pipeline
therefore combines C++ glyph extraction, pypdf extraction, content-stream evidence,
OCR, and text-quality heuristics. Any stage that chooses a wrong representation can
duplicate, drop, or reorder characters before the writer sees them.
The old comparison metric made this harder to observe: ASCII-only unique-token
intersection ignored Arabic, CJK, combining marks, symbols, repeated occurrences,
and order. The replacement is Unicode NFKC normalization, line-break hyphen joining,
duplicate-aware multiset matching, meaningful-symbol retention, and an ordered
sequence score. It is still a diagnostic metric, not a semantic proof.
### Layout and structure fidelity
The IDM preserves BBoxes and reading order, but a flowing DOCX writer must choose
paragraphs, columns, tables, and page breaks. Heuristics infer headings from size
and text, detect headers/footers from repetition and position, and merge rows across
pages. They cannot preserve every PDF transformation matrix, clipping path,
baseline, text box, field, or z-order. Automatic page-raster fallback is the safety
floor for complex positioned pages: visual content survives, but text is no longer
editable or searchable as individual runs.
### Formatting preservation
The writers can carry bold, italic, colors, hyperlinks, bullets, numbering, and
several paragraph attributes when those attributes exist in the source Office XML
or can be inferred from PDF glyph metadata. PDF extraction usually lacks the
original style names and theme inheritance. Font aliases and locally installed
fonts determine metrics; missing fonts fall back to Helvetica/Liberation/DejaVu or
script-specific Noto faces. Word-to-PDF currently applies a fixed paragraph
`spaceAfter` in the ReportLab path rather than reproducing every source paragraph's
spacing, so small but systematic vertical shifts are expected.
### Tables
The detector combines ruling-line (lattice), whitespace/stream, rectangle, and
plausibility passes, then maps glyph lines into cells. A two-column spanning header
or a row whose words sit on separate baselines can be split into fragments. Merged
cells and nested content are represented approximately. The structure ONNX hook
can supply separators, but the model is operator-supplied and fail-open; it is not
part of the reproducible default deployment.
### Images and graphics
XObjects and rendered page images are retained where possible. Large figures are
compacted rather than silently discarded. If a page reconstructs to no meaningful
blocks, the pipeline renders and embeds that page as a figure, or emits an explicit
warning. This preserves visible pixels but cannot preserve editable diagram
objects, vector semantics, captions' relationships, or exact crop/ordering in all
cases.
## Evidence
### Measurement method
`tools/audit_conversion.py` invokes the same in-process `run_conversion` entry point
used by the gateway. For each case it records input/output bytes, source and output
page counts, warnings/errors, duplicate-aware text recall/precision/order/F1, table
payloads, and model status. A Windows process sampler records working-set peak and
`GetProcessTimes` CPU seconds at 20 ms intervals. DOCX/XLSX outputs are parsed with
`python-docx`/`openpyxl`; visual comparison renders both sides at 96 DPI and reports
`1 - mean(abs(gray_a-gray_b))/255` over the compared pages.
The visual metric is intentionally simple and should be treated as a trend signal.
It is not SSIM, does not align transformed pages, and in this run renders Office
outputs with the in-process ReportLab fallback. It therefore does not measure what
LibreOffice/Gotenberg would produce.
### Representative real-document run
The latest six-case digital run is the current `audit_results.json` artifact; the
20-page scan row is from the preceding targeted run and will be appended after the
final OCR run. Times are wall seconds; CPU is process CPU seconds; RSS is peak
working set. "Order" is the ordered token similarity, not a page-layout score.
Output size and output page count are taken directly from the audit JSON. Re-run
the command in Appendix A after any code change to refresh all rows. Results can
vary with cache/model warm-up and with whether the page-raster fallback is
selected, so the harness should be run more than once when comparing changes.
| Document -> target | Source pages | Rendered output pages* | Page delta* | Wall s | CPU s | Peak RSS MB | Output bytes | Warnings | Recall | Precision | Order | Pixel similarity |
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| govinfo -> DOCX | 1 | 2 | +1 | 4.707 | 12.250 | 717.0 | 39,779 | 7 | 0.941 | 0.888 | 0.443 | 0.950 |
| IRS 1040 -> DOCX | 2 | 7 | +5 | 4.531 | 13.578 | 1118.7 | 43,430 | 9 | 0.936 | 0.944 | 0.674 | 0.890 |
| Mozilla PDF spec (14 p) -> DOCX | 14 | 28 | +14 | 29.785 | 92.750 | 1141.1 | 77,386 | 24 | 0.916 | 0.937 | 0.149 | 0.895 |
| USGS factsheet (4 p) -> DOCX | 4 | 7 | +3 | 14.897 | 30.594 | 1346.8 | 161,987 | 13 | 0.767 | 0.793 | 0.232 | 0.784 |
| W3C ruled table -> XLSX | 1 | 1 | 0 | 1.732 | 5.391 | 1191.7 | 5,328 | 7 | 1.000 | 1.000 | 0.990 | 0.959 |
| Multipage table (2 p) -> XLSX | 2 | 2 | 0 | 3.280 | 11.891 | 1195.6 | 5,593 | 7 | 0.600 | 1.000 | 0.750 | 0.998 |
| Scan pack 1 (20 p) -> DOCX | 20 | 4 | -16 | 142.747 | 655.016 | 1769.4 | 58,244 | 102 | 1.000 | 0.077 | 0.143 | 0.957 |
Source characteristics for the digital cases were 3,487 chars/545 words (govinfo),
10,416/2,223 (IRS), 82,714/13,439 (Mozilla), 14,022/1,887 (USGS), 375/65 (W3C),
and 73/25 (multipage table) according to pypdf's source read. The scan pack has
20 raster pages and only a tiny/invalid embedded text layer, so expected page-token
recall was supplied independently (`ScanTokenpack1P000` ... `P019`).
\*Rendered output pages are the pages produced when the audit harness sends the
Office bytes through its in-process ReportLab visual renderer. They are not a
claim about a user's pagination in Word. For example, the scan DOCX contains 19
explicit page breaks (20 intended pages) even though the fallback visual render
reports four pages; both page-break metadata and rendered page count must be
tracked.
The scan result is a useful warning: recall 1.000 alone would say "perfect," while
precision 0.077 and order 0.143 show that the editable OCR reconstruction contains
substantial extra/repeated material. Its pixel similarity is high because the page
images are retained; visual success and editable-text success are different
products.
### Multi-column defect A/B evidence
On the Mozilla document, the old path produced 189 reading-order inversions in the
probe. After the coverage-gate and column-aware row fixes, the probe reports 1.
The corrected prose sentence is no longer spliced. The old corpus quality number
was 0.9475 and the new number 0.8998, but the output character count fell from
83,136 to 79,223 because duplicate visitor text disappeared. A/B instrumentation
showed recall 0.9806 -> 0.9608 and missing token *types* 41 -> 83, while missing
real words changed 31 -> 32; the additional "missing" tokens were chart-axis
labels that pypdf had decoded differently. This is a metric-baseline artifact,
not evidence that the repaired sentence was lost.
### Table evidence
The hand-labelled table truth in `corpus/convert/table_truth.json` requires the W3C
exact six-column grid to reach recall 0.90 and precision 0.55. The current failing
output from `build_idm_from_pdf(table.pdf)` is:
```text
["Disability Category", "s Participant", "Ballots Completed",
"Ballots Incomplete/ Terminated", "Accuracy", "Results Time to complete"]
["Blind", "5", "1", "4", "34.5%, n= 1", "1199 sec, n=1"]
...
["Mobility", "3", "3", "0", "95.4%, n= 3", "1416 sec, n=3"]
```
Compiled glyph inspection found a zero-width-space marker at approximately
`x=222.28` and the following `s` at `x=222.31`, with near-identical baseline;
`lines_from_glyphs()` separated them. Header words were spread over multiple
baselines (`Participant`/`s`, `Results`/`Time to complete`, and similar pairs),
and some spans contained the punctuation artifact `"= "`. The two failing table
tests are therefore reproducible extraction defects, not a threshold problem.
### OCR and model evidence
The active venv reports:
```text
pypdf 5.3.0
onnxruntime 1.28.0
rapidocr-onnxruntime 1.2.3
providers: AzureExecutionProvider, CPUExecutionProvider
pypdfium2: unavailable
```
No CUDA provider was available, so the measured conversion was CPU-bound. The
layout model is 130.5 MB FP32 (34.7 MB INT8 alternative); the Arabic recognizer is
8.0 MB. The root `models/font` tree contains only metadata/tokenizer placeholders,
not `model.onnx` or `fonts.index`, and the classifier code does not create an ONNX
session. Any font-confidence value from that service must therefore be treated as
a heuristic, not a calibrated model probability.
Arabic warm-page measurements were stable enough to show the tradeoff:
| Mode | Mean wall/page | Mean CPU/page | Interpretation |
| --- | ---: | ---: | --- |
| Arabic pass enabled | 3.021 s | 17.318 CPU-s | Better script coverage, roughly double inference work |
| Adaptive skip on clean Latin | 1.370 s | 7.943 CPU-s | About 55% lower inference cost; known Arabic/broken-cmap pages still force the pass |
Whole 20-page wall time was noisy (non-adaptive 150.389 s versus adaptive
178.408 s in separate runs), so the warm-page measurements, not that pair alone,
should guide capacity planning.
### Test evidence
Focused regression groups passed as follows: earlier cache/queue suite 96 tests;
OCR budget/Arabic suite 32; CORS/model packaging 8; raster/quality/real-corpus
focused run 34 passed with 16 raster-specific skips; streaming export plus
cache/queue 7; corpus/phase/production smoke 36; and an additional metrics/OCR
run 133. The full gateway suite completed 746 passed, 21 skipped, and 3 failed.
The three failures were:
1. `test_labelled_table_is_recovered[w3c_ruled_grid]`: recall 0.867, floor 0.90.
2. `test_table_pdf_reading_order_and_spans`: the same `n= 1` and header-fragment
defects.
3. `test_font_endpoint_rejects_unknown`: endpoint returned 204; test expects 404.
The raster skips are environmental (`pypdfium2` is missing), not green evidence
that image export/OCR works on a renderer-free host.
## Recommended Fix
### Immediate P0/P1 work
1. **Finish W3C/table normalization and fragment stitching.** In the table path
only, remove whitespace immediately after `=` and normalize punctuation spacing
around `%`, `=`, and commas. Merge one-character continuation fragments when
their baseline and x-position overlap a neighboring fragment. Stitch header
fragments by column anchor and vertical proximity, preserving true multi-line
cells. Add a regression for the exact five bad cells and for ordinary prose so
the normalization cannot alter sentence spacing.
2. **Merge the two geometry walkers.** The precise C++ walker and pypdf visitor
observe the same content stream. Use precise glyph bounds whenever available and
visitor text only as a decoding fallback, instead of calibrating visitor width
from cross-column span gaps. This removes the remaining unsound estimator rather
than merely reducing how often it runs.
3. **Add independent layout gates.** Keep hand-labelled table truth, and add page
bounding-box/column anchors, header/footer repetition, page-break, image-count,
and cross-column order fixtures. Report recall, precision, duplicates, order,
and visual similarity separately; never gate only on pypdf-derived text.
4. **Make the table structure model reproducible.** Either vendor a checked,
permissively licensed Table Transformer/SLANet model with checksum and measured
CPU/RAM cost, or explicitly document heuristic-only deployment and route
high-risk tables to the page-image fallback.
### OCR and resource work
5. Keep one process-wide session per model, but use a dedicated OCR worker pool or
process queue so serialized inference cannot block unrelated conversions. Bound
image dimensions and decoded bytes before OCR; expose queue wait, model-load,
render, inference, and rebuild timings per page.
6. Calibrate DPI and preprocessing by document class (deskew, contrast, denoise,
adaptive threshold, crop). Use confidence distributions and script detection to
decide a second pass; preserve low-confidence text with a warning rather than
silently replacing it. Evaluate RapidOCR against PaddleOCR PP-OCRv5, EasyOCR,
and a CPU-friendly transformer on a labelled multilingual set before switching.
7. Treat the current font service as a heuristic fallback until a real model is
shipped. If font fidelity matters, load and checksum an actual ONNX model,
batch line crops, and cache one prediction per style/span rather than running
line and word inference for every OCR block. Otherwise disable it in the hot
conversion path and use extracted PDF font metadata.
### Fidelity and operations
8. Preserve the existing page-raster fallback for complex positioned PDFs and make
the choice explicit in response metadata (`editable` versus `visual`). For
customer-facing editable PDF->Office jobs, use Gotenberg/LibreOffice or a
commercial fidelity provider behind the same plugin interface; retain ReportLab
for concurrent-safe low-cost fallback.
9. Install `pypdfium2` in every supported runtime image and run the raster suite in
CI. Keep the C++ engine path as the preferred renderer, but fail loudly when a
host has neither renderer nor OCR capability.
10. Resolve the font endpoint 204/404 contract, and expose structured error codes
for page-cap, OCR-budget, table-confidence, and visual-fallback events.
11. Run conversion workers in separate processes/containers for hostile or very
large files. Apply per-tenant byte/page/time budgets, temporary-directory
quotas, cancellation cleanup, and a dead-letter queue. Track p95/p99 latency,
peak RSS, CPU saturation, cache hit rate, and output-fidelity distributions.
## Implementation
The following changes are present in the working tree and were validated by the
focused suites:
- `layout/pipeline.py`, `pypdf_geometry.py`, `glyphs.py`, and paragraph/order
helpers now use non-whitespace coverage, column-aware row bands, zero-width
whitespace handling, improved paragraph grouping, and figure preservation.
- `ocr.py` serializes model load/inference safely, supports adaptive Arabic
recognition (`CONVERT_OCR_ARABIC_ADAPTIVE`), preserves meaningful symbols such
as `$` and `%`, and records whether Arabic was used per call.
- `ocr/rebuild.py` raises the page cap to 200, separates page-cap and time-budget
warnings/metadata, compacts oversized figures instead of dropping them, and
embeds unreconstructable pages as honest page images.
- `result_cache.py` canonicalizes effective options, namespace, and automatic
versus explicit resolution mode. `doc_cache.py` reuses readers, geometry,
rasters, and engine handles inside one conversion.
- `routers/convert/jobs.py` and conversion scopes bound admission and shared
execution slots; cancellation and deadlines are checked at page/OCR boundaries.
- `quality/text_metrics.py` and `quality/scorer.py` now expose Unicode-aware,
duplicate-aware, order-sensitive metrics and warnings for low precision/order;
the IDM retains component metrics for diagnostics.
- PDF->PNG/JPEG/TIFF exports stream pages; TIFF is multi-frame LZW with DPI
metadata. DOCX raster fallback is selected from measured complexity/content
evidence rather than blindly reflowing every page.
- `Dockerfile`, `models/MANIFEST.json`, and `tools/verify_models.py` package and
checksum runtime layout/Arabic weights. CORS defaults are explicit and
production-safe. `start_convert_gateway.ps1` documents OCR, model, timeout,
queue, and cache settings.
- `tools/audit_conversion.py` provides repeatable end-to-end measurements. New
tests cover metrics, OCR resource guards, cache keys, queue admission, streaming
exports, model packaging, and table truth.
No component was replaced solely because it was open source. The changes preserve
the C++/PDFium core and existing plugin interfaces; alternatives are recommended
only where the evidence shows a fundamental fidelity or reproducibility gap.
## Validation Results
The current state passes the broad focused checks listed in Evidence and runs the
full gateway suite to completion without a crash or timeout. The multi-column
order probe improved from 189 inversions to 1, and the OCR/page-cap/resource
regressions are green. Streaming image export and OOXML validity checks are green.
The result is not yet release-clean:
- W3C exact table recall is 0.867 against a 0.90 floor.
- Table reading-order/span assertions fail on the same punctuation/header
fragments.
- Unknown-font endpoint status disagrees with its test contract.
- USGS and multipage-table order/recall remain low despite high pixel similarity.
- Scan-pack editable text has perfect supplied-token recall but only 0.077
precision and 0.143 order similarity; it must not be marketed as accurate OCR
without a richer ground truth.
Expected gains from the recommended work are stated as targets, not measured
claims: table stitching should raise W3C recall above the existing 0.90 gate and
remove the four known header/punctuation errors; a merged geometry walker should
keep the 1-inversion result while eliminating the remaining visitor-width risk;
adaptive OCR already demonstrates approximately 55% lower warm-page inference
cost on clean Latin scans; process-level isolation should make peak RSS roughly
linear per worker instead of allowing simultaneous jobs to share one unbounded
address space. Each target must be re-measured on the same corpus before release.
## Remaining Limitations
1. PDF->editable DOCX/XLSX cannot be pixel-perfect for arbitrary positioned PDF
content. The page-image fallback preserves appearance by giving up editability;
the reflow path preserves more semantics by accepting layout drift.
2. The default table detector remains heuristic and the table structure model is
not shipped. Merged cells, borderless grids, nested tables, and multi-line
headers remain the largest content-structure risk.
3. RapidOCR is CPU-bound in the measured host, and shared-session locking limits
parallel OCR throughput. Whole-document timings vary with cache/model state and
Windows scheduling; warm per-page numbers are more stable.
4. The active venv lacks `pypdfium2`, so renderer-free Linux behavior and several
raster tests were skipped. The Docker dependency declaration and local runtime
are currently inconsistent until the venv is rebuilt from `pyproject.toml`.
5. No CUDA provider was available in this audit. GPU utilization, GPU memory, and
multi-GPU scaling are therefore unmeasured.
6. The advertised OpenCLIP font model and FAISS index are metadata placeholders in
this checkout. Font predictions are heuristic and should not be used as a
fidelity guarantee.
7. Visual similarity is a grayscale pixel heuristic over at most the compared
pages and, for Office outputs in this audit, uses ReportLab rather than a real
LibreOffice/Gotenberg render.
8. The quality score is useful for triage but is not a single acceptance metric.
Source text can be corrupt, OCR can add legitimate-looking alternatives, and
order/layout require independent annotations.
9. The repository has a `.git_disabled` metadata directory rather than a normal
`.git` checkout, and much of the conversion subsystem is working-tree content.
Before/after comparisons in this investigation therefore come from harness
snapshots and controlled runs, not a pristine commit diff.
## Appendix A: Reproduction Commands and Evidence Files
Run from `pdf/` with the gateway venv:
```powershell
gateway\.venv\Scripts\python.exe tools\audit_conversion.py
gateway\.venv\Scripts\python.exe tools\audit_conversion.py --no-scan
gateway\.venv\Scripts\python.exe -m pytest gateway\tests\convert -q
gateway\.venv\Scripts\python.exe -m pytest gateway\tests -q
```
Primary evidence and implementation references:
- `audit_results.json` - machine-readable audit output (the checked-in file may
be targeted to one case; regenerate for the full matrix).
- `tools/audit_conversion.py` - timing, resource, content, table, and visual
measurement harness.
- `corpus/convert/table_truth.json` and `corpus/convert/baselines.json` - external
table labels and suite floors.
- `gateway/app/services/convert/layout/pipeline.py` - routing, IDM construction,
OCR budgeting, and page-image fallback.
- `gateway/app/services/convert/layout/pypdf_geometry.py` and `glyphs.py` - glyph
walkers, line grouping, and geometry evidence.
- `gateway/app/services/convert/ocr/rebuild.py` and `gateway/app/services/ocr.py` -
raster/OCR/rebuild path and Arabic adaptive policy.
- `gateway/app/services/convert/formatters/docx_formatter.py`,
`writers/pdf_from_docx.py`, and `backends/pdf_exporter.py` - Office writers and
Gotenberg/ReportLab selection.
- `gateway/pyproject.toml`, `gateway/Dockerfile`, `gateway/models/MANIFEST.json`,
and `gateway/start_convert_gateway.ps1` - dependency, packaging, and runtime
configuration evidence.
**Bottom line:** the engine now has honest diagnostics and materially better
multi-column recovery, but it should be released with explicit fidelity classes,
the W3C table defect fixed, renderer/OCR dependencies verified in CI, and separate
content, structure, and visual acceptance gates.
audit_mozilla_ml_on.json
+550
View File
File diff suppressed because one or more lines are too long
audit_mozilla_ml_on_after.json
+518
View File
File diff suppressed because one or more lines are too long
audit_results.json
+478
View File
File diff suppressed because one or more lines are too long
audit_results_final.json
+1914
View File
File diff suppressed because one or more lines are too long
audit_results_final_pretty.tmp
+1933
View File
File diff suppressed because one or more lines are too long
audit_results_postfix.json
+1882
View File
File diff suppressed because one or more lines are too long
audit_results_pretty.tmp
+481
View File
File diff suppressed because one or more lines are too long
bindings/CMakeLists.txt
+104
View File
@@ -1,6 +1,74 @@
# Python bindings for PdfEngine using pybind11
#
# The .pyd ABI tag (cp312 vs cp313) must match the interpreter that runs
# the FastAPI gateway. Prefer gateway/.venv when it exists so `import pdfengine`
# works in uvicorn without a second Python install.
if(WIN32)
set(_gateway_python "${CMAKE_SOURCE_DIR}/gateway/.venv/Scripts/python.exe")
else()
set(_gateway_python "${CMAKE_SOURCE_DIR}/gateway/.venv/bin/python")
endif()
if(EXISTS "${_gateway_python}")
set(Python_EXECUTABLE "${_gateway_python}" CACHE FILEPATH "Python for pybind11" FORCE)
set(Python3_EXECUTABLE "${_gateway_python}" CACHE FILEPATH "Python3 for pybind11" FORCE)
set(PYTHON_EXECUTABLE "${_gateway_python}" CACHE FILEPATH "Python for pybind11 (legacy)" FORCE)
message(STATUS "pybind11: using gateway venv ${_gateway_python}")
endif()
# Drop stale FindPythonLibsNew cache from a previous configure (it can keep
# PYTHON_MODULE_EXTENSION=.cp313-*.pyd and python313.lib even after we switch
# the interpreter to the gateway 3.12 venv).
unset(PYTHON_MODULE_EXTENSION CACHE)
unset(PYTHON_LIBRARIES CACHE)
unset(PYTHON_INCLUDE_DIRS CACHE)
unset(PYTHON_VERSION CACHE)
unset(PYTHON_VERSION_MAJOR CACHE)
unset(PYTHON_VERSION_MINOR CACHE)
unset(PYTHON_IS_DEBUG CACHE)
unset(PYTHON_MODULE_PREFIX CACHE)
unset(PYTHON_MODULE_DEBUG_POSTFIX CACHE)
if(Python_EXECUTABLE)
execute_process(
COMMAND "${Python_EXECUTABLE}" -c "import sys, sysconfig, pathlib; base=pathlib.Path(sys.base_prefix); ver=f'{sys.version_info.major}{sys.version_info.minor}'; print(sysconfig.get_config_var('EXT_SUFFIX') or ''); print(sysconfig.get_path('include')); print(base / 'libs' / f'python{ver}.lib')"
OUTPUT_VARIABLE _pdfengine_py_info
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_QUIET
)
string(REPLACE "\r" "" _pdfengine_py_info "${_pdfengine_py_info}")
string(REPLACE "\n" ";" _pdfengine_py_info_lines "${_pdfengine_py_info}")
list(LENGTH _pdfengine_py_info_lines _pdfengine_py_info_len)
if(_pdfengine_py_info_len GREATER_EQUAL 3)
list(GET _pdfengine_py_info_lines 0 _pdfengine_py_ext)
list(GET _pdfengine_py_info_lines 1 _pdfengine_py_inc)
list(GET _pdfengine_py_info_lines 2 _pdfengine_py_lib)
file(TO_CMAKE_PATH "${_pdfengine_py_ext}" _pdfengine_py_ext)
file(TO_CMAKE_PATH "${_pdfengine_py_inc}" _pdfengine_py_inc)
file(TO_CMAKE_PATH "${_pdfengine_py_lib}" _pdfengine_py_lib)
if(_pdfengine_py_ext)
set(PYTHON_MODULE_EXTENSION "${_pdfengine_py_ext}" CACHE INTERNAL "Python extension suffix")
message(STATUS "pybind11: extension suffix ${_pdfengine_py_ext}")
endif()
# vcpkg ships a static python312.lib; linking the .pyd against that
# crashes when uvicorn loads it (no python312.dll import). Use the
# interpreter's own import lib / headers instead.
if(EXISTS "${_pdfengine_py_lib}" AND EXISTS "${_pdfengine_py_inc}/Python.h")
set(Python_LIBRARY "${_pdfengine_py_lib}" CACHE FILEPATH "Python import library" FORCE)
set(Python_LIBRARIES "${_pdfengine_py_lib}" CACHE FILEPATH "Python import library" FORCE)
set(Python_INCLUDE_DIR "${_pdfengine_py_inc}" CACHE PATH "Python headers" FORCE)
set(Python3_LIBRARY "${_pdfengine_py_lib}" CACHE FILEPATH "Python3 import library" FORCE)
set(Python3_INCLUDE_DIR "${_pdfengine_py_inc}" CACHE PATH "Python3 headers" FORCE)
get_filename_component(_pdfengine_py_root "${_pdfengine_py_inc}" DIRECTORY)
set(Python_ROOT_DIR "${_pdfengine_py_root}" CACHE PATH "CPython install root" FORCE)
message(STATUS "pybind11: CPython lib ${_pdfengine_py_lib}")
message(STATUS "pybind11: CPython include ${_pdfengine_py_inc}")
endif()
endif()
endif()
set(PYBIND11_FINDPYTHON ON)
find_package(pybind11 CONFIG REQUIRED)
# Declare the python module target. We name the target pdfengine_py to avoid
@@ -12,9 +80,23 @@ set_target_properties(pdfengine_py PROPERTIES
OUTPUT_NAME "pdfengine"
ARCHIVE_OUTPUT_NAME "pdfengine_py_import"
)
if(_pdfengine_py_ext)
set_target_properties(pdfengine_py PROPERTIES SUFFIX "${_pdfengine_py_ext}")
endif()
target_link_libraries(pdfengine_py PRIVATE pdfengine::pdfengine)
# OBJECT-library usage requirements are not always enough on MSVC for the
# python module (it also consumes $<TARGET_OBJECTS:pdfengine>). Repeat the
# libraries the engine objects reference so the .pyd link line is complete.
target_link_libraries(pdfengine_py PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
if(PDFENGINE_WITH_PDFIUM)
target_link_libraries(pdfengine_py PRIVATE pdfium::pdfium)
endif()
if(_pdfengine_py_lib AND EXISTS "${_pdfengine_py_lib}")
target_link_libraries(pdfengine_py PRIVATE "${_pdfengine_py_lib}")
endif()
target_include_directories(pdfengine_py PRIVATE
"${CMAKE_SOURCE_DIR}/engine/src"
)
@@ -23,6 +105,28 @@ if(MSVC)
target_link_options(pdfengine_py PRIVATE "/FORCE:MULTIPLE")
endif()
# vcpkg's find_package(Python) wrapper still injects its static python312.lib.
# A pybind module must link only the host interpreter's import lib (python312.dll).
foreach(_py_tgt IN ITEMS Python::Module Python::Python Python3::Module Python3::Python)
if(TARGET ${_py_tgt} AND _pdfengine_py_lib)
set_property(TARGET ${_py_tgt} PROPERTY INTERFACE_LINK_LIBRARIES "${_pdfengine_py_lib}")
set_property(TARGET ${_py_tgt} PROPERTY IMPORTED_LOCATION "${_pdfengine_py_lib}")
set_property(TARGET ${_py_tgt} PROPERTY IMPORTED_IMPLIB "${_pdfengine_py_lib}")
endif()
endforeach()
get_target_property(_pdfengine_py_link pdfengine_py LINK_LIBRARIES)
if(_pdfengine_py_link)
set(_pdfengine_py_kept "")
foreach(_lib IN LISTS _pdfengine_py_link)
if(_lib MATCHES "vcpkg_installed" AND _lib MATCHES "[Pp]ython")
message(STATUS "pybind11: dropping vcpkg CPython ${_lib}")
continue()
endif()
list(APPEND _pdfengine_py_kept "${_lib}")
endforeach()
set_property(TARGET pdfengine_py PROPERTY LINK_LIBRARIES "${_pdfengine_py_kept}")
endif()
# Set warnings and sanitizers for the bindings module
pdfengine_set_warnings(pdfengine_py)
pdfengine_enable_sanitizers(pdfengine_py)
command.MD
+18 -4
View File
@@ -35,9 +35,16 @@ This document serves as the single source of truth for commands across our devel
```powershell
powershell -ExecutionPolicy Bypass -File scripts/bootstrap.ps1
cd gateway
python -m venv .venv
# Fast path with uv (recommended: 10x faster)
uv venv
.venv\Scripts\Activate.ps1
pip install -e ".[dev]"
uv pip install -e ".[dev]"
# (Alternative fallback with standard pip)
# python -m venv .venv
# .venv\Scripts\Activate.ps1
# pip install -e ".[dev]"
cd ..
```
@@ -92,9 +99,16 @@ npm run build
```powershell
powershell -ExecutionPolicy Bypass -File scripts/bootstrap.ps1
cd gateway
python -m venv .venv
# Fast path with uv (recommended: 10x faster)
uv venv
.venv\Scripts\Activate.ps1
pip install -e ".[dev]"
uv pip install -e ".[dev]"
# (Alternative fallback with standard pip)
# python -m venv .venv
# .venv\Scripts\Activate.ps1
# pip install -e ".[dev]"
cd ..
```
corpus/convert/public/README.md
+29
View File
@@ -0,0 +1,29 @@
# Public free eval corpus (Wave-1)
Licensed third-party + DocQube self-generated smoke for PDF↔DOCX/XLSX, Path A, images, OCR.
## Quick start (Windows)
```powershell
cd pdf
..\gateway\.venv\Scripts\python.exe scripts\convert\generate_public_smoke_fixtures.py
..\gateway\.venv\Scripts\python.exe scripts\convert\score_public_pack.py
```
Optional third-party fetch (after LICENSE_CHECKLIST sign-off):
```powershell
..\gateway\.venv\Scripts\python.exe scripts\convert\fetch_public_eval.py
```
## Layout
- `manifests/` — per-family case definitions
- `pack_public.json` — unified pytest pack
- `samples/` — gitignored binaries (generated or fetched)
- `reports/` — baseline / A/B JSON
## Licenses
See `../third_party/NOTICE` and `../third_party/LICENSE_CHECKLIST.md`.
Default fixtures are **self-generated** (no third-party license). DocLayNet/CORD/SROIE/PubTabNet entries in manifests are placeholders until fetched.
corpus/convert/public/manifests/cord_smoke.json
+23
View File
@@ -0,0 +1,23 @@
{
"cases": [
{
"id": "cord_placeholder",
"family": "cord",
"license": "CC-BY-4.0",
"source": "public/samples/cord/.gitkeep",
"pair": "pdf_txt",
"bars": {
"cell_match_min": null,
"prose_tokens": [],
"text_recall_min": 0.5
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"notes": "placeholder until fetch_public_eval.py",
"optional": true
}
]
}
corpus/convert/public/manifests/doclaynet_smoke.json
+49
View File
@@ -0,0 +1,49 @@
{
"cases": [
{
"id": "synth_digital_001",
"family": "doclaynet",
"license": "DocQube",
"source": "public/samples/doclaynet/synth_digital_001.pdf",
"pair": "pdf_docx",
"bars": {
"cell_match_min": null,
"prose_tokens": [],
"text_recall_min": 0.95
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"expected_tokens": [
"Digital",
"Report",
"AlphaBridge"
],
"notes": "self-gen digital layout stand-in"
},
{
"id": "synth_multicol_001",
"family": "doclaynet",
"license": "DocQube",
"source": "public/samples/doclaynet/synth_multicol_001.pdf",
"pair": "pdf_docx",
"bars": {
"cell_match_min": null,
"prose_tokens": [],
"text_recall_min": 0.8
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"expected_tokens": [
"ColumnLeftOne",
"ColumnRightOne"
],
"notes": "two-column reading-order stress"
}
]
}
corpus/convert/public/manifests/images_smoke.json
+22
View File
@@ -0,0 +1,22 @@
{
"cases": [
{
"id": "images_png_pdf",
"family": "images",
"license": "DocQube",
"source": "public/samples/images/smoke.png",
"pair": "png_pdf",
"bars": {
"cell_match_min": null,
"prose_tokens": [],
"text_recall_min": null
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"notes": "PNG\u2192PDF"
}
]
}
corpus/convert/public/manifests/path_a_smoke.json
+45
View File
@@ -0,0 +1,45 @@
{
"cases": [
{
"id": "path_a_docx",
"family": "path_a",
"license": "DocQube",
"source": "public/samples/path_a/path_a_sample.docx",
"pair": "docx_pdf",
"bars": {
"cell_match_min": null,
"prose_tokens": [
"PathAParagraphToken"
],
"text_recall_min": null
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"notes": "DOCX\u2192PDF Path A"
},
{
"id": "path_a_xlsx",
"family": "path_a",
"license": "DocQube",
"source": "public/samples/path_a/path_a_sample.xlsx",
"pair": "xlsx_pdf",
"bars": {
"cell_match_min": null,
"prose_tokens": [
"HeaderA",
"CellAlpha"
],
"text_recall_min": null
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"notes": "XLSX\u2192PDF Path A"
}
]
}
corpus/convert/public/manifests/sroie_smoke.json
+23
View File
@@ -0,0 +1,23 @@
{
"cases": [
{
"id": "sroie_placeholder",
"family": "sroie",
"license": "CC-BY-4.0",
"source": "public/samples/sroie/.gitkeep",
"pair": "pdf_txt",
"bars": {
"cell_match_min": null,
"prose_tokens": [],
"text_recall_min": 0.5
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"notes": "placeholder until fetch_public_eval.py",
"optional": true
}
]
}
corpus/convert/public/manifests/synthetic_invoice_smoke.json
+42
View File
@@ -0,0 +1,42 @@
{
"cases": [
{
"id": "synth_invoice_001",
"family": "synthetic_invoice",
"license": "DocQube",
"source": "public/samples/invoices/synth_invoice_001.pdf",
"pair": "pdf_xlsx",
"bars": {
"cell_match_min": 0.85,
"prose_tokens": [
"Invoice",
"AcmeCorp",
"Total"
],
"text_recall_min": null
},
"gt": {
"grid": [
[
"Desc",
"Qty",
"Amount"
],
[
"Service",
"1",
"100"
],
[
"Tax",
"1",
"10"
]
],
"text_file": null,
"regions_coco": null
},
"notes": "self-gen invoice"
}
]
}
corpus/convert/public/pack_public.json
+218
View File
@@ -0,0 +1,218 @@
{
"cases": [
{
"id": "synth_invoice_001",
"family": "synthetic_invoice",
"license": "DocQube",
"source": "public/samples/invoices/synth_invoice_001.pdf",
"pair": "pdf_xlsx",
"bars": {
"cell_match_min": 0.85,
"prose_tokens": [
"Invoice",
"AcmeCorp",
"Total"
],
"text_recall_min": null
},
"gt": {
"grid": [
[
"Desc",
"Qty",
"Amount"
],
[
"Service",
"1",
"100"
],
[
"Tax",
"1",
"10"
]
],
"text_file": null,
"regions_coco": null
},
"notes": "self-gen invoice"
},
{
"id": "synth_table_001",
"family": "pubtabnet",
"license": "DocQube",
"source": "public/samples/pubtab/synth_table_001.pdf",
"pair": "pdf_xlsx",
"bars": {
"cell_match_min": 0.9,
"prose_tokens": [],
"text_recall_min": null
},
"gt": {
"grid": [
[
"Item",
"Qty",
"Price"
],
[
"Widget",
"2",
"10"
],
[
"Gadget",
"1",
"25"
]
],
"text_file": null,
"regions_coco": null
},
"notes": "self-gen stand-in for PubTabNet until fetch"
},
{
"id": "synth_digital_001",
"family": "doclaynet",
"license": "DocQube",
"source": "public/samples/doclaynet/synth_digital_001.pdf",
"pair": "pdf_docx",
"bars": {
"cell_match_min": null,
"prose_tokens": [],
"text_recall_min": 0.95
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"expected_tokens": [
"Digital",
"Report",
"AlphaBridge"
],
"notes": "self-gen digital layout stand-in"
},
{
"id": "synth_multicol_001",
"family": "doclaynet",
"license": "DocQube",
"source": "public/samples/doclaynet/synth_multicol_001.pdf",
"pair": "pdf_docx",
"bars": {
"cell_match_min": null,
"prose_tokens": [],
"text_recall_min": 0.8
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"expected_tokens": [
"ColumnLeftOne",
"ColumnRightOne"
],
"notes": "two-column reading-order stress"
},
{
"id": "path_a_docx",
"family": "path_a",
"license": "DocQube",
"source": "public/samples/path_a/path_a_sample.docx",
"pair": "docx_pdf",
"bars": {
"cell_match_min": null,
"prose_tokens": [
"PathAParagraphToken"
],
"text_recall_min": null
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"notes": "DOCX\u2192PDF Path A"
},
{
"id": "path_a_xlsx",
"family": "path_a",
"license": "DocQube",
"source": "public/samples/path_a/path_a_sample.xlsx",
"pair": "xlsx_pdf",
"bars": {
"cell_match_min": null,
"prose_tokens": [
"HeaderA",
"CellAlpha"
],
"text_recall_min": null
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"notes": "XLSX\u2192PDF Path A"
},
{
"id": "images_png_pdf",
"family": "images",
"license": "DocQube",
"source": "public/samples/images/smoke.png",
"pair": "png_pdf",
"bars": {
"cell_match_min": null,
"prose_tokens": [],
"text_recall_min": null
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"notes": "PNG\u2192PDF"
},
{
"id": "cord_placeholder",
"family": "cord",
"license": "CC-BY-4.0",
"source": "public/samples/cord/.gitkeep",
"pair": "pdf_txt",
"bars": {
"cell_match_min": null,
"prose_tokens": [],
"text_recall_min": 0.5
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"notes": "placeholder until fetch_public_eval.py",
"optional": true
},
{
"id": "sroie_placeholder",
"family": "sroie",
"license": "CC-BY-4.0",
"source": "public/samples/sroie/.gitkeep",
"pair": "pdf_txt",
"bars": {
"cell_match_min": null,
"prose_tokens": [],
"text_recall_min": 0.5
},
"gt": {
"grid": null,
"text_file": null,
"regions_coco": null
},
"notes": "placeholder until fetch_public_eval.py",
"optional": true
}
]
}
corpus/convert/public/reports/ab_compare.json
+6
View File
@@ -0,0 +1,6 @@
{
"ml_off_passed": 7,
"ml_on_passed": 7,
"deltas": [],
"regressions": 0
}
corpus/convert/public/reports/baseline_heuristic.json
+64
View File
@@ -0,0 +1,64 @@
{
"ml": "0",
"results": [
{
"id": "synth_invoice_001",
"pair": "pdf_xlsx",
"ml": "0",
"pass": true,
"cell_match": 1.0
},
{
"id": "synth_table_001",
"pair": "pdf_xlsx",
"ml": "0",
"pass": true,
"cell_match": 1.0
},
{
"id": "synth_digital_001",
"pair": "pdf_docx",
"ml": "0",
"pass": true,
"text_recall": 1.0
},
{
"id": "synth_multicol_001",
"pair": "pdf_docx",
"ml": "0",
"pass": true,
"text_recall": 1.0
},
{
"id": "path_a_docx",
"pair": "docx_pdf",
"ml": "0",
"pass": true
},
{
"id": "path_a_xlsx",
"pair": "xlsx_pdf",
"ml": "0",
"pass": true
},
{
"id": "images_png_pdf",
"pair": "png_pdf",
"ml": "0",
"pass": true
},
{
"id": "cord_placeholder",
"skipped": true,
"reason": "missing_source"
},
{
"id": "sroie_placeholder",
"skipped": true,
"reason": "missing_source"
}
],
"passed": 7,
"failed": 0,
"skipped": 2
}
corpus/convert/public/reports/live_real_doc_suite.json
+728
View File
@@ -0,0 +1,728 @@
[
{
"file": "digital_report_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36742,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9025",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "digital_report_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4947,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9025",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "govinfo.pdf",
"target": "docx",
"status": 200,
"bytes": 38508,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8932",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "govinfo.pdf",
"target": "xlsx",
"status": 200,
"bytes": 7086,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8932",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "govinfo_small.pdf",
"target": "docx",
"status": 200,
"bytes": 38508,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8932",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "govinfo_small.pdf",
"target": "xlsx",
"status": 200,
"bytes": 7085,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8932",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "invoice_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36748,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "invoice_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4980,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6132",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "invoice_002.pdf",
"target": "docx",
"status": 200,
"bytes": 36742,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "invoice_002.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4963,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6050",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "irs_f1040_sample.pdf",
"target": "docx",
"status": 200,
"bytes": 41114,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8993",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "irs_f1040_sample.pdf",
"target": "xlsx",
"status": 200,
"bytes": 10851,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8993",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "merged_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36833,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "merged_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 5007,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9550",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
},
{
"file": "mozilla_pdf_spec_excerpt.pdf",
"target": "docx",
"status": 200,
"bytes": 74515,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9076",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "mozilla_pdf_spec_excerpt.pdf",
"target": "xlsx",
"status": 200,
"bytes": 56039,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9076",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "multicolumn_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36671,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "multicolumn_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4896,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "multicolumn_002.pdf",
"target": "docx",
"status": 200,
"bytes": 36671,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "multicolumn_002.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4896,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "multipage_table_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36835,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7125",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "multipage_table_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 5593,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8000",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Page 2: layout_ml=onnx regions"
},
{
"file": "simple_table_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36701,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "simple_table_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4914,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6050",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "table_gap_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36668,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: glyph/engine text we"
},
{
"file": "table_gap_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4898,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "table_pipe_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36847,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "table_pipe_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 5033,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9550",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
},
{
"file": "table_simple_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36854,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "table_simple_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 5051,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9550",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
},
{
"file": "usgs_factsheet.pdf",
"target": "docx",
"status": 200,
"bytes": 3065773,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6825",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "usgs_factsheet.pdf",
"target": "xlsx",
"status": 200,
"bytes": 15077,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.5600",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "w3c_pdf_table.pdf",
"target": "docx",
"status": 200,
"bytes": 37075,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6096",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "w3c_pdf_table.pdf",
"target": "xlsx",
"status": 200,
"bytes": 5335,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.5431",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=2"
},
{
"file": "w3c_table.pdf",
"target": "docx",
"status": 200,
"bytes": 37075,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6096",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "w3c_table.pdf",
"target": "xlsx",
"status": 200,
"bytes": 5336,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.5431",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=2"
},
{
"file": "synth_digital_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36742,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9025",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "synth_digital_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4947,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9025",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "synth_multicol_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36739,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9175",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "synth_multicol_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4933,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9175",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "synth_invoice_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36748,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "synth_invoice_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4981,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6132",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "synth_table_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36701,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "synth_table_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4915,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6050",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "digital_report_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36742,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9025",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "digital_report_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4946,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9025",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "invoice_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36748,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "invoice_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4981,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6132",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "invoice_002.pdf",
"target": "docx",
"status": 200,
"bytes": 36742,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "invoice_002.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4965,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6050",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "merged_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36833,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "merged_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 5006,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9550",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
},
{
"file": "multicolumn_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36671,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "multicolumn_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4895,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "multicolumn_002.pdf",
"target": "docx",
"status": 200,
"bytes": 36671,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "multicolumn_002.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4895,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "multipage_table_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36835,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7125",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "multipage_table_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 5593,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8000",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Page 2: layout_ml=onnx regions"
},
{
"file": "simple_table_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36701,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "simple_table_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4914,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.6050",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "table_gap_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36668,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: glyph/engine text we"
},
{
"file": "table_gap_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 4898,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.8875",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | No table structure detected; exported paragraphs as a single column. | layout_ml=o"
},
{
"file": "table_pipe_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36847,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "table_pipe_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 5032,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9550",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
},
{
"file": "table_simple_001.pdf",
"target": "docx",
"status": 200,
"bytes": 36854,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.7275",
"ml": "PASS_ML",
"warn": "Layout fidelity is lossy; headers/footers/fonts are not fully preserved. | layout_ml=onnx | Page 1: layout_ml=onnx regio"
},
{
"file": "table_simple_001.pdf",
"target": "xlsx",
"status": 200,
"bytes": 5051,
"magic": "PK",
"fidelity": "lossy",
"quality": "0.9550",
"ml": "PASS_ML",
"warn": "Table detection is heuristic/lossy. | layout_ml=onnx | Page 1: layout_ml=onnx regions=0 | Capped at medium: PDF-\u003eOffice "
}
]
corpus/convert/public/reports/score_ml_on.json
+64
View File
@@ -0,0 +1,64 @@
{
"ml": "1",
"results": [
{
"id": "synth_invoice_001",
"pair": "pdf_xlsx",
"ml": "1",
"pass": true,
"cell_match": 1.0
},
{
"id": "synth_table_001",
"pair": "pdf_xlsx",
"ml": "1",
"pass": true,
"cell_match": 1.0
},
{
"id": "synth_digital_001",
"pair": "pdf_docx",
"ml": "1",
"pass": true,
"text_recall": 1.0
},
{
"id": "synth_multicol_001",
"pair": "pdf_docx",
"ml": "1",
"pass": true,
"text_recall": 1.0
},
{
"id": "path_a_docx",
"pair": "docx_pdf",
"ml": "1",
"pass": true
},
{
"id": "path_a_xlsx",
"pair": "xlsx_pdf",
"ml": "1",
"pass": true
},
{
"id": "images_png_pdf",
"pair": "png_pdf",
"ml": "1",
"pass": true
},
{
"id": "cord_placeholder",
"skipped": true,
"reason": "missing_source"
},
{
"id": "sroie_placeholder",
"skipped": true,
"reason": "missing_source"
}
],
"passed": 7,
"failed": 0,
"skipped": 2
}
corpus/convert/public/reports/triage.md
+16
View File
@@ -0,0 +1,16 @@
# Public pack triage (A/B)
- ML off passed: 7
- ML on passed: 7
- Regressions: 0
## Suggested climb owners
| Failure pattern | Climb | Files |
| --- | --- | --- |
| Invoice / grid cell | M2 | layout/tables.py, quality/table_metrics.py |
| Multi-column order | M3 | reading_order.py, ml_merge.py |
| Scan / OCR empty | M4 | ocr/rebuild.py, rapid_adapter.py |
| Missing figures | M5 | layout/images.py |
| Header/footer in body | S1 | headers_footers.py, docx_formatter.py |
corpus/convert/real/README.md
+16
View File
@@ -0,0 +1,16 @@
# Wave-1 production smoke pack (curated)
Internal curated PDFs used for **shippable Wave-1** acceptance bars.
These are synthetic/corpus-derived stand-ins for business docs until customer PDFs are added.
| File | Role |
| --- | --- |
| digital_report_001.pdf | Simple digital text → DOCX |
| table_pipe_001.pdf / table_simple_001.pdf | Simple tables → XLSX |
| invoice_001.pdf / invoice_002.pdf | Invoice grid + prose → XLSX |
| merged_001.pdf | Merged-style table → XLSX |
| multicolumn_*.pdf | Multi-column → DOCX text recall |
| multipage_table_001.pdf | Multi-page tables |
| table_gap_001.pdf | Gap-separated columns |
License: synthetic-internal. Not ConvertAPI goldens.
corpus/convert/real/pack.json
+182
View File
@@ -0,0 +1,182 @@
{
"cases": [
{
"id": "real_digital_docx",
"source": "real/digital_report_001.pdf",
"pair": "pdf->docx",
"expected_tokens": [
"Digital",
"Report",
"AlphaBridge",
"AcmeCorp"
],
"text_recall_min": 0.95
},
{
"id": "real_table_xlsx",
"source": "real/simple_table_001.pdf",
"pair": "pdf->xlsx",
"expected_cells": [
[
"Item",
"Qty",
"Price"
],
[
"Widget",
"2",
"10"
],
[
"Gadget",
"1",
"25"
]
],
"cell_match_min": 0.9
},
{
"id": "real_invoice_xlsx",
"source": "real/invoice_001.pdf",
"pair": "pdf->xlsx",
"expected_cells": [
[
"Desc",
"Qty",
"Amount"
],
[
"Service",
"1",
"100"
]
],
"prose_tokens": [
"Invoice",
"AcmeCorp",
"Vendor"
],
"cell_match_min": 0.85
},
{
"id": "real_two_col_docx",
"source": "real/multicolumn_001.pdf",
"pair": "pdf->docx",
"expected_tokens": [
"LeftColA",
"LeftColB",
"RightColA",
"RightColB"
],
"text_recall_min": 0.95
},
{
"id": "real_three_col_docx",
"source": "real/multicolumn_003.pdf",
"pair": "pdf->docx",
"expected_tokens": [
"AlphaLeft",
"BravoMid",
"CharlieRight",
"AlphaLast",
"CharlieLast"
],
"text_recall_min": 0.9
},
{
"id": "real_fourcol_xlsx",
"source": "real/table_fourcol_001.pdf",
"pair": "pdf->xlsx",
"expected_cells": [
[
"Category",
"Q1",
"Q2",
"Total"
],
[
"Food",
"10",
"12",
"22"
],
[
"Travel",
"5",
"7",
"12"
]
],
"cell_match_min": 0.85
},
{
"id": "real_arabic_docx",
"source": "real/arabic_line_001.pdf",
"pair": "pdf->docx",
"expected_tokens": [
"Hello",
"World",
"TRAIL_AR",
"Bilingual"
],
"text_recall_min": 0.85
},
{
"id": "real_scanlike_docx",
"source": "real/scanlike_001.pdf",
"pair": "pdf->docx",
"expected_tokens": [
"ScanLike",
"ScanTokenSMOKEP000",
"Widget"
],
"text_recall_min": 0.9
},
{
"id": "real_spanning_docx",
"source": "real/spanning_title_001.pdf",
"pair": "pdf->docx",
"expected_tokens": [
"SpanningTitleAcrossTheFullPageWidthHere",
"SpanLeftA",
"SpanRightA"
],
"text_recall_min": 0.9
},
{
"id": "real_rect_table_xlsx",
"source": "real/rect_table_001.pdf",
"pair": "pdf->xlsx",
"expected_cells": [
[
"Name",
"Qty",
"Cost"
],
[
"Alpha",
"3",
"9"
],
[
"Beta",
"4",
"8"
]
],
"cell_match_min": 0.7
},
{
"id": "real_hyphen_md",
"source": "real/hyphen_md_001.pdf",
"pair": "pdf->md",
"expected_tokens": [
"HyphenSmoke",
"word",
"Chapter",
"postprocess"
],
"text_recall_min": 0.75
}
]
}
corpus/convert/third_party/LICENSE_CHECKLIST.md
Vendored
+18
View File
@@ -0,0 +1,18 @@
# License verification checklist (operator)
Sign off before fetching each asset into `corpus/convert` or `gateway/models/layout`.
| Asset | Upstream URL | License claimed | Verified by | Date | OK for prod train/eval? |
| --- | --- | --- | --- | --- | --- |
| DocLayNet | https://github.com/DS4SD/DocLayNet | CDLA-Permissive-1.0 | | | Y/N |
| PubLayNet | https://github.com/ibm-aur-nlp/publaynet | CDLA-P + PMC CA images | | | Y/N |
| PubTabNet | https://github.com/ibm-aur-nlp/PubTabNet | CDLA-P + PMC CA | | | Y/N |
| CORD | https://github.com/clovaai/cord | CC-BY-4.0 | | | Y/N |
| SROIE | https://rrc.cvc.uab.es/?ch=13 | CC-BY-4.0 (verify) | | | Y/N |
| PP-DocLayout ONNX | PaddleOCR release / fetch script | Apache-2.0 | | | Y/N |
| PP-OCRv5 Arabic rec ONNX | RapidAI ModelScope `arabic_PP-OCRv5_rec_mobile.onnx` + `ppocrv5_arabic_dict.txt` | Apache-2.0 (RapidAI/Paddle) | | | Y/N |
| Self-gen fixtures | DocQube scripts | Proprietary/internal | | | Y |
**Never fetch for prod:** FUNSD (NC), LayoutLMv3 weights (NC), DocLayout-YOLO (AGPL).
Disk budgets: smoke ≤ 500 MB; full DocLayNet ~28 GB lab-only (`--full`).
corpus/convert/third_party/NOTICE
Vendored
+40
View File
@@ -0,0 +1,40 @@
DocQube convert — third-party NOTICE
====================================
Append a block below whenever you fetch a public dataset or layout model.
Keep license names and upstream URLs accurate.
How to append
-------------
1. Verify license on LICENSE_CHECKLIST.md
2. Download via scripts/convert/fetch_public_eval.py or fetch_layout_onnx.py
3. Add:
Name: <asset>
License: <SPDX or common name>
Source: <URL>
Used for: <eval|train|inference>
Date: <ISO date>
Self-generated fixtures under corpus/convert (real/ / public/samples/invoices etc.)
are owned by DocQube and require no third-party attribution.
--- Upstream entries (filled by fetch scripts) ---
Name: PP-DocLayoutV3 ONNX (layout.onnx / inference.onnx)
License: Apache-2.0
Source: https://huggingface.co/PaddlePaddle/PP-DocLayoutV3_onnx
Used for: inference
Date: 2026-08-27
SHA256: 45bf71750b00739a41fc209f132eb104a4d6b5bb29483c9078164d8b87cf28ba
Path: gateway/models/layout/v1/layout.onnx (copy of inference.onnx)
Name: PP-OCRv5 Arabic recognition ONNX (rec.onnx)
License: Apache-2.0
Source: https://www.modelscope.cn/models/RapidAI/RapidOCR
Used for: inference
Name: PP-OCRv5 Arabic charset (arabic_dict.txt)
License: Apache-2.0
Source: https://www.modelscope.cn/models/RapidAI/RapidOCR/resolve/v3.9.2/paddle/PP-OCRv5/rec/arabic_PP-OCRv5_rec_mobile/ppocrv5_arabic_dict.txt
Used for: inference
corpus/convert/third_party/checksums.sha256
Vendored
+16
View File
@@ -0,0 +1,16 @@
# SHA256 checksums for convert public samples and layout weights
# Format: <sha256> <relative-path-from-pdf/>
# Populated by fetch_public_eval.py / fetch_layout_onnx.py / generate fixtures.
58a051ee4dcc59e49b1c379ed36e528e1d99dc6968c9a5c11bf6934e0a8a1f36 corpus/convert/real/digital_report_001.pdf
e67e0040b551d674ed57097d71b1d11bb2c0b45de18da902b847053f9b00d6e8 corpus/convert/real/simple_table_001.pdf
ba2bf716c44970858e298c340fbd5e8a31c40639315fa2ce42da6bbee0c1a279 corpus/convert/real/invoice_001.pdf
ffad883045c91f860039352ad9343fad1eb958e6a11fd49a4a2e7f8d22102a5c corpus/convert/public/samples/invoices/synth_invoice_001.pdf
0d9165e3d425d3782f4b41e38b890343c6f6cd032691d04f5e345f8ed1105211 corpus/convert/public/samples/pubtab/synth_table_001.pdf
7fc1dc773d71db49bafdb0458d964e5ec68f76488f785d753f4575b1b685e40f corpus/convert/public/samples/doclaynet/synth_digital_001.pdf
314e13e9e4f1178dace76985b10cb41aa5db9a45aaa2e319dc21aab37a336023 corpus/convert/public/samples/doclaynet/synth_multicol_001.pdf
e2e2f4f6671e753955036f4a0ac2b56e3534bf3901fe9feeeca0e68950a87749 corpus/convert/public/samples/path_a/path_a_sample.docx
eb203ee73bea31647892d0892605cb65a2aad9a0abdb7a76e8b2e2e60e625775 corpus/convert/public/samples/path_a/path_a_sample.xlsx
a33ad10e0028c1f84fd1ce229a4de2d0bdb931bc68ac01e80222dd9772008d05 corpus/convert/public/samples/images/smoke.png
45bf71750b00739a41fc209f132eb104a4d6b5bb29483c9078164d8b87cf28ba gateway/models/layout/v1/layout.onnx
c1192e632d0baa9146ae5b756a0e635e3dc63c1733737ebfd1629e87144e9295 gateway/models/ocr/ar/v5/rec.onnx
7f92f7dbb9b75a4787a83bfb4f6d14a8ab515525130c9d40a9036f61cf6999e9 gateway/models/ocr/ar/v5/arabic_dict.txt
docker-compose.yml
+11
View File
@@ -9,6 +9,10 @@ services:
environment:
PDFENGINE_ENVIRONMENT: dev
PDFENGINE_ENGINE_AVAILABLE: "true"
# Keep browser access explicit; production deployments should replace
# this with the exact frontend origin(s), never a wildcard.
PDFENGINE_CORS_ALLOWED_ORIGINS: http://localhost:5173,http://127.0.0.1:5173,https://pdf-dev.maskantech.in
PDFENGINE_CORS_ALLOW_CREDENTIALS: "false"
PORT: 8765
ports:
- "8765:8765"
@@ -39,3 +43,10 @@ services:
depends_on:
- gateway
restart: unless-stopped
gotenberg:
image: gotenberg/gotenberg:8
container_name: pdf-engine-gotenberg
ports:
- "3000:3000"
restart: unless-stopped
engine/CMakeLists.txt
+25 -13
View File
@@ -111,11 +111,15 @@ target_link_libraries(pdfengine
)
if(PDFENGINE_WITH_PDFIUM)
target_link_libraries(pdfengine PRIVATE pdfium::pdfium)
target_link_libraries(pdfengine PUBLIC pdfium::pdfium)
target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_PDFIUM)
# PDFium statically bundles its own libjpeg, zlib, etc. which conflicts with vcpkg.
# We use LLD, so we can safely allow multiple definitions to pick the first one.
target_link_options(pdfengine PUBLIC "-Wl,--allow-multiple-definition")
# PDFium statically bundles its own libjpeg, zlib, etc. which conflict with vcpkg.
# MSVC ignores -Wl,--allow-multiple-definition (that is a GNU ld flag).
if(MSVC)
target_link_options(pdfengine PUBLIC "/FORCE:MULTIPLE")
else()
target_link_options(pdfengine PUBLIC "-Wl,--allow-multiple-definition")
endif()
endif()
if(EMSCRIPTEN)
@@ -129,16 +133,24 @@ if(PDFENGINE_WITH_SKIA)
target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_SKIA)
endif()
# qpdf / libjpeg are used by sources that compile even when PDFENGINE_WITH_QPDF is
# off (resource_resolver.hpp, qpdf_resource_resolver.cpp, image_decoder.cpp,
# filter_decoder.cpp). vcpkg puts the headers on the include path automatically,
# so the objects compile — but MSVC then fails with LNK2019 unless we link here.
# pdfengine is an OBJECT library: PUBLIC is required so consumers (the Python
# module and smoke test) actually get these libs on the final link line.
find_package(ZLIB REQUIRED)
find_package(JPEG REQUIRED)
find_package(qpdf CONFIG REQUIRED)
if(NOT TARGET zs)
add_library(zs ALIAS ZLIB::ZLIB)
endif()
if(NOT TARGET jpeg)
add_library(jpeg ALIAS JPEG::JPEG)
endif()
target_link_libraries(pdfengine PUBLIC qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
if(PDFENGINE_WITH_QPDF)
find_package(ZLIB REQUIRED)
find_package(JPEG REQUIRED)
if(NOT TARGET zs)
add_library(zs ALIAS ZLIB::ZLIB)
endif()
if(NOT TARGET jpeg)
add_library(jpeg ALIAS JPEG::JPEG)
endif()
target_link_libraries(pdfengine PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_QPDF)
endif()
engine/tests/CMakeLists.txt
+1 -3
View File
@@ -46,9 +46,7 @@ if(PDFENGINE_WITH_PDFIUM)
target_link_libraries(pdfengine_smoke PRIVATE pdfium::pdfium)
endif()
if(PDFENGINE_WITH_QPDF)
target_link_libraries(pdfengine_smoke PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
endif()
target_link_libraries(pdfengine_smoke PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
if(MSVC)
target_link_options(pdfengine_smoke PRIVATE "/FORCE:MULTIPLE")
frontend/src/App.tsx
+16 -18
View File
@@ -12,6 +12,7 @@ import { ExportPDFModal } from './components/ExportPDFModal';
import { MergePDFModal } from './components/MergePDFModal';
import { WatermarkModal, type WatermarkConfig } from './components/WatermarkModal';
import { triggerPDFDownload } from './lib/pdfExport';
import { convertFileMaybeAsync, downloadBlob } from './lib/convertService';
import { CustomConfirmationModal } from './components/custom/CustomConfirmationModal';
import type { CustomConfirmationOptions } from './components/custom/CustomConfirmationModal';
@@ -160,7 +161,7 @@ function App() {
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
const [searchCurrentMatch, setSearchCurrentMatch] = useState(0);
const [isOCRLoading, setIsOCRLoading] = useState(false);
const [isOCRLoading] = useState(false);
const [creatorPageCount, setCreatorPageCount] = useState<number>(1);
const [creatorPages, setCreatorPages] = useState<PageLayout[]>([]);
const [watermarkPreview, setWatermarkPreview] = useState<WatermarkConfig | null>(null);
@@ -170,19 +171,6 @@ function App() {
if (pages) setCreatorPages(pages);
}, []);
const handleRunOCR = async () => {
if (!activeDoc) return;
setIsOCRLoading(true);
try {
await gatewayService.performPageOCR(activeDoc.id, currentPage);
viewerRef.current?.refreshPageLayout(currentPage);
} catch (err: any) {
alert(`OCR processing failed: ${err.message || err}`);
} finally {
setIsOCRLoading(false);
}
};
const forceOpenDocument = useCallback((id: string) => {
localStorage.removeItem('active_mode');
setCreatePdfModalOpen(false);
@@ -1117,10 +1105,10 @@ function App() {
setExportModalOpen(true);
};
const handleConfirmExport = async (targetFilename: string) => {
const handleConfirmExport = async (targetFilename: string, targetFormat = 'pdf') => {
const isCreatorActive = activeTool === 'create_pdf' || createPdfModalOpen;
console.log(`[EXPORT_PDF] Export requested. Document ID: ${isCreatorActive ? 'new-blank-creator' : selectedDocId}`);
console.log(`[EXPORT_PDF] Filename: ${targetFilename}`);
console.log(`[EXPORT_PDF] Filename: ${targetFilename} target=${targetFormat}`);
let pdfBlob: Blob;
if (isCreatorActive) {
@@ -1139,7 +1127,17 @@ function App() {
}
console.log(`[EXPORT_PDF] PDF size: ${pdfBlob.size} bytes`);
await triggerPDFDownload(pdfBlob, targetFilename);
if (targetFormat === 'pdf') {
await triggerPDFDownload(pdfBlob, targetFilename);
return;
}
const sourceName = (isCreatorActive ? 'document.pdf' : activeDoc?.filename) || 'document.pdf';
const file = new File([pdfBlob], sourceName.replace(/\.[^.]+$/, '.pdf'), {
type: 'application/pdf',
});
const result = await convertFileMaybeAsync(file, 'pdf', targetFormat);
downloadBlob(result.blob, targetFilename || result.filename);
};
const handlePrint = async () => {
@@ -1231,7 +1229,7 @@ function App() {
isSaving={isSaving}
isDirtySaved={(activeTool === 'create_pdf' || createPdfModalOpen) ? (creatorActions?.canUndo ?? false) : hist.stack.length > 1}
onRotate={handleRotate}
onExport={activeTool === 'create_pdf' || createPdfModalOpen ? (() => creatorActions?.generate()) : handleExport}
onExport={handleExport}
onPrint={(activeTool === 'create_pdf' || createPdfModalOpen) ? (() => creatorActions?.print?.()) : handlePrint}
onProtect={() => {
const isCreatorActive = activeTool === 'create_pdf' || createPdfModalOpen;
frontend/src/components/ExportPDFModal.tsx
+113 -30
View File
@@ -1,12 +1,23 @@
import React, { useState, useEffect } from 'react';
import React, { useEffect, useMemo, useRef, useState } from 'react';
import { Modal } from './ui';
import { CustomButton } from './custom/CustomButton';
import {
ENGINE_CONVERT_PAIRS,
FORMAT_META,
fetchConvertInfo,
fileExtensionForTarget,
pairDescription,
pairDisplayName,
replaceFilenameExtension,
targetsFromSource,
type ConverterInfo,
} from '../lib/convertService';
interface ExportPDFModalProps {
open: boolean;
onClose: () => void;
documentName?: string;
onConfirmExport: (filename: string) => Promise<void>;
onConfirmExport: (filename: string, target: string) => Promise<void>;
}
export const ExportPDFModal: React.FC<ExportPDFModalProps> = ({
@@ -16,40 +27,70 @@ export const ExportPDFModal: React.FC<ExportPDFModalProps> = ({
onConfirmExport,
}) => {
const [filename, setFilename] = useState('');
const [target, setTarget] = useState('pdf');
const [isExporting, setIsExporting] = useState(false);
const [errorMessage, setErrorMessage] = useState<string | null>(null);
const [pairs, setPairs] = useState<ConverterInfo[]>(
targetsFromSource(ENGINE_CONVERT_PAIRS, 'pdf').filter((p) => p.target !== 'pdf')
);
const skipExtSync = useRef(true);
const selected = useMemo(
() => pairs.find((p) => p.target === target),
[pairs, target]
);
useEffect(() => {
if (open) {
const baseName = documentName.replace(/\.pdf$/i, '');
const defaultExportName = baseName ? `${baseName}-final.pdf` : 'document-final.pdf';
setFilename(defaultExportName);
setIsExporting(false);
setErrorMessage(null);
}
if (!open) return;
let cancelled = false;
void fetchConvertInfo().then((info) => {
if (cancelled) return;
setPairs(targetsFromSource(info, 'pdf').filter((p) => p.target !== 'pdf'));
});
return () => {
cancelled = true;
};
}, [open]);
useEffect(() => {
if (!open) return;
skipExtSync.current = true;
setTarget('pdf');
setIsExporting(false);
setErrorMessage(null);
const stem = documentName.replace(/\.[^.]+$/i, '') || 'document';
setFilename(`${stem}-final.pdf`);
}, [open, documentName]);
useEffect(() => {
if (!open) return;
if (skipExtSync.current) {
skipExtSync.current = false;
return;
}
setFilename((prev) => replaceFilenameExtension(prev || documentName, target));
}, [target, open, documentName]);
const handleDownload = async () => {
if (isExporting) return;
let trimmed = filename.trim();
if (!trimmed) {
trimmed = 'document-final.pdf';
trimmed = replaceFilenameExtension('document-final', target);
}
if (!trimmed.toLowerCase().endsWith('.pdf')) {
trimmed += '.pdf';
const ext = fileExtensionForTarget(target);
if (!trimmed.toLowerCase().endsWith(`.${ext}`)) {
trimmed = replaceFilenameExtension(trimmed, target);
}
try {
setIsExporting(true);
setErrorMessage(null);
console.log(`[EXPORT_PDF] Export requested. Filename: ${trimmed}`);
await onConfirmExport(trimmed);
console.log('[EXPORT_PDF] Export completed successfully');
await onConfirmExport(trimmed, target);
onClose();
} catch (err: any) {
console.error('[EXPORT_PDF] Export failed:', err);
setErrorMessage(err.message || 'Export failed. Please try again.');
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Export failed. Please try again.';
setErrorMessage(message);
} finally {
setIsExporting(false);
}
@@ -57,14 +98,27 @@ export const ExportPDFModal: React.FC<ExportPDFModalProps> = ({
if (!open) return null;
const formats: { target: string; label: string; hint: string }[] = [
{
target: 'pdf',
label: 'PDF',
hint: 'Edited document as PDF',
},
...pairs.map((p) => ({
target: p.target,
label: pairDisplayName(p),
hint: pairDescription(p),
})),
];
return (
<Modal
open={open}
onClose={() => {
if (!isExporting) onClose();
}}
title="Export PDF"
width={440}
title="Export / Convert"
width={560}
>
<div className="flex flex-col gap-4 text-[13px]">
{errorMessage && (
@@ -92,7 +146,7 @@ export const ExportPDFModal: React.FC<ExportPDFModalProps> = ({
onKeyDown={(e) => {
if (e.key === 'Enter' && !isExporting) {
e.preventDefault();
handleDownload();
void handleDownload();
}
}}
placeholder="document-final.pdf"
@@ -102,14 +156,41 @@ export const ExportPDFModal: React.FC<ExportPDFModalProps> = ({
<div className="flex flex-col gap-1.5">
<label className="text-[12px] font-bold text-text-primary">
Format:
Format (conversion engine):
</label>
<div className="flex h-9 w-full items-center justify-between rounded-lg border border-border-primary bg-bg-secondary px-3 text-[13px] font-semibold text-text-primary">
<span>PDF Document (.pdf)</span>
<span className="rounded bg-brand-primary/10 px-2 py-0.5 text-[11px] font-bold text-brand-primary">
PDF
</span>
<div className="grid max-h-[240px] grid-cols-2 gap-1.5 overflow-auto pr-0.5">
{formats.map((fmt) => {
const meta = FORMAT_META[fmt.target];
const active = target === fmt.target;
return (
<button
key={fmt.target}
type="button"
disabled={isExporting}
onClick={() => setTarget(fmt.target)}
className={`rounded-[10px] border px-3 py-2 text-left transition ${
active
? 'border-brand-primary bg-brand-primary/5'
: 'border-border-primary bg-bg-primary hover:border-brand-primary/50'
} disabled:opacity-50`}
>
<span className="flex items-center justify-between gap-2">
<span className="text-[12px] font-extrabold text-text-primary">{fmt.label}</span>
<span className="rounded bg-brand-primary/10 px-1.5 py-0.5 text-[10px] font-bold uppercase text-brand-primary">
{meta?.extension ?? fmt.target}
</span>
</span>
<span className="mt-0.5 block text-[10px] leading-4 text-text-secondary">{fmt.hint}</span>
</button>
);
})}
</div>
{target !== 'pdf' && selected && (
<p className="text-[11px] leading-5 text-text-secondary">
Sends the current PDF through the document conversion engine
({selected.fidelity} fidelity).
</p>
)}
</div>
<div className="mt-2 flex items-center justify-end gap-2.5 pt-3 border-t border-border-primary">
@@ -122,7 +203,7 @@ export const ExportPDFModal: React.FC<ExportPDFModalProps> = ({
</CustomButton>
<CustomButton
variant="primary"
onClick={handleDownload}
onClick={() => void handleDownload()}
disabled={isExporting}
>
{isExporting ? (
@@ -137,10 +218,12 @@ export const ExportPDFModal: React.FC<ExportPDFModalProps> = ({
<circle cx="12" cy="12" r="10" strokeOpacity="0.25" />
<path d="M12 2a10 10 0 0 1 10 10" />
</svg>
Generating PDF...
{target === 'pdf' ? 'Generating PDF…' : `Converting to ${target.toUpperCase()}…`}
</span>
) : (
) : target === 'pdf' ? (
'Download PDF'
) : (
`Convert to ${FORMAT_META[target]?.label ?? target.toUpperCase()}`
)}
</CustomButton>
</div>
frontend/src/components/ProductHub.tsx
+482 -55
View File
File diff suppressed because one or more lines are too long
frontend/src/components/Sidebar.tsx
+3 -1
View File
@@ -17,6 +17,8 @@ interface SidebarProps {
onOpenSignature: () => void;
onOpenAbout: () => void;
disabledTools?: Set<ToolId>;
isOCRLoading?: boolean;
isRemote?: boolean;
}
interface GroupDef {
@@ -59,7 +61,7 @@ const SidebarButton: React.FC<{
);
export const Sidebar: React.FC<SidebarProps> = ({
activeTool, activeGroup, onToolChange, onGroupChange, hasSignature, onOpenSignature, onOpenAbout, disabledTools
activeTool, activeGroup, onToolChange, onGroupChange, onOpenAbout, disabledTools
}) => {
return (
<nav className="flex h-full shrink-0 flex-col items-center gap-2 overflow-y-auto border-r border-border-primary bg-bg-primary scroll-micro" style={{ width: '92px', paddingTop: '16px', paddingBottom: '16px' }}>
frontend/src/components/Toolbar.tsx
-1
View File
@@ -116,7 +116,6 @@ export const Toolbar: React.FC<ToolbarProps> = ({
pendingRedactionCount = 0, onApplyRedactions, onClearRedactions, onRedactPages, onOpenWatermark,
selectedAnnotation, onUpdateAnnotation, onDeleteAnnotation, onDeselectAnnotation
}) => {
const meta = TOOL_META[activeTool];
const isRedact = activeTool === 'redact';
const toolsInGroup = GROUP_TOOLS[activeGroup] || [];
frontend/src/components/TopBar.tsx
+1 -1
View File
@@ -87,7 +87,7 @@ export const TopBar: React.FC<TopBarProps> = ({
{!isRemote && <MenuItem icon={<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth={2} strokeLinecap="round" strokeLinejoin="round"><path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z" /><polyline points="14 2 14 8 20 8" /><line x1="12" y1="18" x2="12" y2="12" /><line x1="9" y1="15" x2="15" y2="15" /></svg>} onClick={() => onNewBlankPDF?.()}>New Blank PDF…</MenuItem>}
{!isRemote && <MenuItem icon={<UploadIcon size={16} />} onClick={() => fileRef.current?.click()}>Open PDF…</MenuItem>}
<MenuItem icon={<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth={2} strokeLinecap="round" strokeLinejoin="round"><path d="M8 7v10M12 4v16M16 7v10M4 11h16M4 15h16"/></svg>} onClick={() => onMergePDF?.()}>Merge PDFs…</MenuItem>
<MenuItem icon={<DownloadIcon size={16} />} onClick={onExport} disabled={!documentName || !canExport}>Export / Download</MenuItem>
<MenuItem icon={<DownloadIcon size={16} />} onClick={onExport} disabled={!documentName || !canExport}>Export / Convert</MenuItem>
<MenuItem icon={<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth={2} strokeLinecap="round" strokeLinejoin="round"><polyline points="6 9 6 2 18 2 18 9"></polyline><path d="M6 18H4a2 2 0 0 1-2-2v-5a2 2 0 0 1 2-2h16a2 2 0 0 1 2 2v5a2 2 0 0 1-2 2h-2"></path><rect x="6" y="14" width="12" height="8"></rect></svg>} onClick={onPrint} disabled={!documentName || !canPrint}>Print</MenuItem>
<MenuItem icon={<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth={2} strokeLinecap="round" strokeLinejoin="round"><path d="M9 5H7a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V7a2 2 0 0 0-2-2h-2"/><rect x="9" y="3" width="6" height="4" rx="1"/></svg>} onClick={() => onCompare?.()} disabled={!documentName}>Compare PDFs…</MenuItem>
{isEncrypted ? (
frontend/src/lib/convertService.ts
+554
View File
@@ -0,0 +1,554 @@
/**
* DocQube conversion API client (Wave 1+ OSS converters).
*/
export type ConvertFidelity = 'high' | 'medium' | 'lossy';
export type LayoutMode = 'flowing' | 'continuous' | 'nocolumns' | 'exact';
export type OcrPolicy = 'auto' | 'force' | 'never';
export type OcrEngineChoice = 'auto' | 'rapid' | 'mistral';
export type HeaderFooterMode = 'detect' | 'ignore' | 'remove';
export type RecognitionMode = 'enhanced_flow' | 'flow' | 'textbox';
export interface ConvertDeskOptions {
layoutMode?: LayoutMode;
ocrPolicy?: OcrPolicy;
ocrEngine?: OcrEngineChoice;
headerFooterMode?: HeaderFooterMode;
recognitionMode?: RecognitionMode;
detectTables?: boolean;
}
/**
* Reconstruction defaults, kept for internal callers and tests only.
*
* The conversion desk does NOT send these. Omitting every reconstruction
* parameter is what tells the gateway to route the document itself — which
* path suits a clean digital contract is not the same as what suits a
* photographed invoice, and the customer should not have to know that.
*/
export const DEFAULT_CONVERT_OPTIONS: Required<ConvertDeskOptions> = {
layoutMode: 'flowing',
ocrPolicy: 'auto',
ocrEngine: 'auto',
headerFooterMode: 'detect',
recognitionMode: 'enhanced_flow',
detectTables: true,
};
/**
* Query string for a convert call.
*
* With no options this is empty, so the server applies automatic routing.
* Only the keys a caller explicitly sets are sent: filling in the rest from
* the defaults would read as "explicit" to the gateway and silently disable
* the routing.
*/
function convertQuery(opts?: ConvertDeskOptions): string {
if (!opts) return '';
const q = new URLSearchParams();
if (opts.layoutMode) q.set('layout_mode', opts.layoutMode);
if (opts.ocrPolicy) q.set('ocr_policy', opts.ocrPolicy);
if (opts.ocrEngine) q.set('ocr_engine', opts.ocrEngine);
if (opts.headerFooterMode) q.set('header_footer_mode', opts.headerFooterMode);
if (opts.recognitionMode) q.set('recognition_mode', opts.recognitionMode);
if (opts.detectTables !== undefined) q.set('detect_tables', opts.detectTables ? 'true' : 'false');
return q.toString();
}
export interface ConvertJob {
job_id: string;
status: 'queued' | 'running' | 'completed' | 'failed';
source?: string;
target?: string;
fidelity?: ConvertFidelity | null;
warnings?: string[];
filename?: string | null;
size_bytes?: number | null;
error?: string | null;
download_path?: string | null;
meta?: Record<string, unknown>;
}
export interface ConvertResult {
blob: Blob;
fidelity: string;
warnings: string;
filename: string;
/** 0..1 from X-Quality-Score when the gateway provides it */
qualityScore: number | null;
statusCode?: number;
}
/** Sync convert size above which we prefer async job polling (bytes). */
export const ASYNC_THRESHOLD_BYTES = 8 * 1024 * 1024;
function gatewayBaseUrl(): string {
const url = import.meta.env.VITE_GATEWAY_URL as string | undefined;
if (!url) {
throw new Error('VITE_GATEWAY_URL is not configured');
}
return url.replace(/\/$/, '');
}
export function extensionToTarget(extension: string): string {
const ext = extension.toLowerCase().replace(/^\./, '');
if (ext === 'jpg') return 'jpeg';
if (ext === 'data') return 'json';
return ext;
}
/** Infer convert source from filename / MIME. */
export function inferSourceFormat(file: File): string {
const name = (file.name || '').toLowerCase();
const ext = name.includes('.') ? name.split('.').pop() || '' : '';
if (ext === 'jpg' || file.type === 'image/jpeg') return 'jpeg';
if (ext === 'png' || file.type === 'image/png') return 'png';
if (ext === 'tif' || ext === 'tiff' || file.type === 'image/tiff') return 'tiff';
if (ext === 'docx') return 'docx';
if (ext === 'xlsx') return 'xlsx';
if (ext === 'csv' || file.type === 'text/csv') return 'csv';
if (ext === 'pptx') return 'pptx';
if (ext === 'html' || ext === 'htm') return 'html';
if (ext === 'md' || ext === 'markdown') return 'md';
if (ext === 'txt') return 'txt';
if (ext === 'json' || file.type === 'application/json') return 'json';
if (ext === 'pdf' || file.type === 'application/pdf') return 'pdf';
return ext || 'pdf';
}
export interface ConverterInfo {
source: string;
target: string;
fidelity: ConvertFidelity;
description?: string;
gaps?: string[];
}
/**
* Fallback pairs matching gateway `converters.all_plugins()`.
* Used when `/v1/convert/info` is unreachable so the editor still lists
* the engine's Wave-1 formats rather than inventing planned ones.
*/
export const ENGINE_CONVERT_PAIRS: ConverterInfo[] = [
{ source: 'pdf', target: 'pdf', fidelity: 'high', description: 'PDF->PDF OCR Searchable rebuild' },
{ source: 'pdf', target: 'docx', fidelity: 'lossy', description: 'PDF->DOCX via layout IDM' },
{ source: 'docx', target: 'pdf', fidelity: 'medium', description: 'DOCX->PDF via reportlab' },
{ source: 'pdf', target: 'xlsx', fidelity: 'lossy', description: 'PDF->XLSX via layout IDM' },
{ source: 'xlsx', target: 'pdf', fidelity: 'medium', description: 'XLSX->PDF via reportlab' },
{ source: 'pdf', target: 'csv', fidelity: 'lossy', description: 'PDF->CSV via layout IDM' },
{ source: 'csv', target: 'pdf', fidelity: 'high', description: 'CSV->PDF via reportlab' },
{ source: 'pdf', target: 'pptx', fidelity: 'medium', description: 'PDF->PPTX via layout IDM' },
{ source: 'pptx', target: 'pdf', fidelity: 'high', description: 'PPTX->PDF via slide renderer' },
{ source: 'pdf', target: 'txt', fidelity: 'medium', description: 'PDF->TXT via IDM' },
{ source: 'txt', target: 'pdf', fidelity: 'high', description: 'TXT->PDF via reportlab' },
{ source: 'json', target: 'pdf', fidelity: 'high', description: 'JSON->PDF via styled grid or inspector' },
{ source: 'pdf', target: 'md', fidelity: 'medium', description: 'PDF->Markdown via IDM' },
{ source: 'docx', target: 'md', fidelity: 'medium', description: 'DOCX->Markdown' },
{ source: 'xlsx', target: 'md', fidelity: 'medium', description: 'XLSX->Markdown' },
{ source: 'pptx', target: 'md', fidelity: 'medium', description: 'PPTX->Markdown' },
{ source: 'pdf', target: 'json', fidelity: 'medium', description: 'PDF->JSON idm.v1' },
{ source: 'pdf', target: 'html', fidelity: 'medium', description: 'PDF->HTML via IDM' },
{ source: 'pdf', target: 'png', fidelity: 'high', description: 'PDF->PNG (ZIP if multi-page)' },
{ source: 'pdf', target: 'jpeg', fidelity: 'high', description: 'PDF->JPEG (ZIP if multi-page)' },
{ source: 'pdf', target: 'tiff', fidelity: 'high', description: 'PDF->TIFF' },
{ source: 'png', target: 'pdf', fidelity: 'high', description: 'PNG->PDF' },
{ source: 'jpeg', target: 'pdf', fidelity: 'high', description: 'JPEG->PDF' },
{ source: 'tiff', target: 'pdf', fidelity: 'high', description: 'TIFF->PDF' },
{ source: 'html', target: 'pdf', fidelity: 'medium', description: 'HTML->PDF via reportlab' },
{ source: 'md', target: 'html', fidelity: 'high', description: 'Markdown->HTML' },
{ source: 'md', target: 'pdf', fidelity: 'medium', description: 'Markdown->PDF' },
{ source: 'html', target: 'md', fidelity: 'medium', description: 'HTML->Markdown' },
{ source: 'docx', target: 'txt', fidelity: 'high', description: 'DOCX->TXT' },
{ source: 'docx', target: 'json', fidelity: 'medium', description: 'DOCX->JSON' },
{ source: 'docx', target: 'html', fidelity: 'medium', description: 'DOCX->HTML via mammoth' },
{ source: 'html', target: 'docx', fidelity: 'medium', description: 'HTML->DOCX' },
{ source: 'docx', target: 'docx', fidelity: 'high', description: 'DOCX passthrough' },
];
export const FORMAT_META: Record<
string,
{ label: string; extension: string; group: string }
> = {
pdf: { label: 'PDF', extension: 'PDF', group: 'PDF' },
docx: { label: 'Word', extension: 'DOCX', group: 'Office' },
xlsx: { label: 'Excel', extension: 'XLSX', group: 'Office' },
csv: { label: 'CSV', extension: 'CSV', group: 'Office' },
pptx: { label: 'PowerPoint', extension: 'PPTX', group: 'Office' },
html: { label: 'HTML', extension: 'HTML', group: 'Web and text' },
md: { label: 'Markdown', extension: 'MD', group: 'Web and text' },
txt: { label: 'Plain text', extension: 'TXT', group: 'Web and text' },
json: { label: 'JSON', extension: 'JSON', group: 'Web and text' },
png: { label: 'PNG', extension: 'PNG', group: 'Images' },
jpeg: { label: 'JPEG', extension: 'JPG', group: 'Images' },
tiff: { label: 'TIFF', extension: 'TIFF', group: 'Images' },
};
export const CONVERT_GROUP_ORDER = ['Office', 'Web and text', 'Images', 'PDF'];
export function pairId(source: string, target: string): string {
return `${source}->${target}`;
}
export function fileExtensionForTarget(target: string): string {
const t = extensionToTarget(target);
return t === 'jpeg' ? 'jpg' : t;
}
export function replaceFilenameExtension(name: string, target: string): string {
const stem = name.replace(/\.[^.]+$/, '') || 'document';
return `${stem}.${fileExtensionForTarget(target)}`;
}
export function pairDisplayName(pair: ConverterInfo): string {
const tgt = FORMAT_META[pair.target]?.label ?? pair.target.toUpperCase();
if (pair.source === pair.target) return `${tgt} passthrough`;
if (pair.target === 'pdf') return `PDF from ${pair.source.toUpperCase()}`;
if (pair.source === 'pdf') return tgt;
return `${tgt} from ${pair.source.toUpperCase()}`;
}
export function pairDescription(pair: ConverterInfo): string {
const fid = pair.fidelity;
if (pair.description) return `${pair.description} (${fid})`;
return `${pair.source.toUpperCase()} → ${pair.target.toUpperCase()} (${fid})`;
}
let cachedConvertInfo: ConverterInfo[] | null = null;
/** Live converter registry from the document conversion engine. */
export async function fetchConvertInfo(): Promise<ConverterInfo[]> {
if (cachedConvertInfo?.length) return cachedConvertInfo;
try {
const res = await fetch(`${gatewayBaseUrl()}/v1/convert/info`);
if (res.ok) {
const data = (await res.json()) as { converters?: ConverterInfo[] };
if (Array.isArray(data.converters) && data.converters.length > 0) {
cachedConvertInfo = data.converters;
return cachedConvertInfo;
}
}
} catch {
/* fall through to engine fallback */
}
return ENGINE_CONVERT_PAIRS;
}
export function targetsFromSource(pairs: ConverterInfo[], source: string): ConverterInfo[] {
const src = extensionToTarget(source);
return pairs.filter((p) => p.source === src);
}
export function groupedPairs(pairs: ConverterInfo[]): { label: string; formats: ConverterInfo[] }[] {
const buckets = new Map<string, ConverterInfo[]>();
for (const pair of pairs) {
const group = FORMAT_META[pair.target]?.group ?? 'More formats';
const list = buckets.get(group) ?? [];
list.push(pair);
buckets.set(group, list);
}
const ordered = CONVERT_GROUP_ORDER.filter((g) => buckets.has(g)).map((label) => ({
label,
formats: buckets.get(label)!,
}));
for (const [label, formats] of buckets) {
if (!CONVERT_GROUP_ORDER.includes(label)) ordered.push({ label, formats });
}
return ordered;
}
export function uploadAcceptFromPairs(pairs: ConverterInfo[]): string {
const extras: Record<string, string[]> = {
pdf: ['.pdf', 'application/pdf'],
docx: ['.docx'],
xlsx: ['.xlsx'],
pptx: ['.pptx'],
html: ['.html', '.htm', 'text/html'],
md: ['.md', '.markdown'],
png: ['.png', 'image/png'],
jpeg: ['.jpg', '.jpeg', 'image/jpeg'],
tiff: ['.tif', '.tiff', 'image/tiff'],
json: ['.json', 'application/json'],
};
const tokens = new Set<string>();
for (const pair of pairs) {
(extras[pair.source] ?? [`.${pair.source}`]).forEach((t) => tokens.add(t));
}
return [...tokens].join(',');
}
export async function canConvert(source: string, target: string): Promise<boolean> {
const res = await fetch(
`${gatewayBaseUrl()}/v1/convert/canconvert/${source}/to/${target}`
);
if (!res.ok) return false;
const data = (await res.json()) as { can_convert?: boolean };
return Boolean(data.can_convert);
}
function parseErrorDetail(status: number, bodyText: string): string {
const statusHint =
status === 504
? 'Conversion timed out (504). Try a smaller file or raise CONVERT_TIMEOUT_SECONDS.'
: status === 422
? 'This conversion pair is not supported (422).'
: status === 400
? 'Invalid input (400).'
: `Conversion failed (${status}).`;
try {
const body = JSON.parse(bodyText) as { detail?: unknown };
if (typeof body?.detail === 'string' && body.detail.trim()) return body.detail;
if (Array.isArray(body?.detail)) return JSON.stringify(body.detail);
} catch {
if (bodyText.trim()) return bodyText.slice(0, 400);
}
return statusHint;
}
function parseFilenameFromDisposition(disposition: string, fallback: string): string {
// Prefer RFC 5987 filename*=UTF-8''...
const star = /filename\*\s*=\s*UTF-8''([^;]+)/i.exec(disposition);
if (star?.[1]) {
try {
return decodeURIComponent(star[1].trim().replace(/^"+|"+$/g, ''));
} catch {
/* fall through */
}
}
const plain = /filename\s*=\s*"([^"]+)"|filename\s*=\s*([^;]+)/i.exec(disposition);
const raw = (plain?.[1] || plain?.[2] || '').trim();
return raw || fallback;
}
function friendlyNetworkError(err: unknown): Error {
const msg = err instanceof Error ? err.message : String(err);
if (/failed to fetch|networkerror|load failed/i.test(msg)) {
return new Error(
'Could not reach the convert gateway (or download crashed). ' +
'Confirm VITE_GATEWAY_URL points at a running gateway on 8080, then retry.'
);
}
return err instanceof Error ? err : new Error(msg);
}
async function parseConvertResponse(res: Response, target: string): Promise<ConvertResult> {
if (!res.ok) {
const bodyText = await res.text();
const err = new Error(parseErrorDetail(res.status, bodyText));
(err as Error & { statusCode?: number }).statusCode = res.status;
throw err;
}
const blob = await res.blob();
const disposition = res.headers.get('Content-Disposition') || '';
const filename = parseFilenameFromDisposition(
disposition,
`converted.${target === 'jpeg' ? 'jpg' : target}`
);
const qsRaw = res.headers.get('X-Quality-Score');
const qualityScore = qsRaw != null && qsRaw !== '' ? Number.parseFloat(qsRaw) : null;
return {
blob,
fidelity: res.headers.get('X-Fidelity') || 'lossy',
warnings: res.headers.get('X-Warnings') || '',
filename,
qualityScore: Number.isFinite(qualityScore) ? qualityScore : null,
statusCode: res.status,
};
}
export async function convertFile(
file: File,
source: string,
targetExtension: string,
deskOptions?: ConvertDeskOptions
): Promise<ConvertResult> {
const target = extensionToTarget(targetExtension);
const form = new FormData();
form.append('file', file, file.name || `upload.${source}`);
const qs = convertQuery(deskOptions);
let res: Response;
try {
res = await fetch(`${gatewayBaseUrl()}/v1/convert/${source}/to/${target}${qs ? `?${qs}` : ''}`, {
method: 'POST',
body: form,
});
} catch (err) {
throw friendlyNetworkError(err);
}
return parseConvertResponse(res, target);
}
/**
* Client poll budget for async convert.
*
* Must stay at CONVERT_TIMEOUT_MAX_SECONDS (gateway default 1800s). A
* size-scaled wait used to abort a 15 MB bilingual scan at ~13 min while
* the job was still inside the 20 min OCR wall — that is a desk timeout,
* not a convert failure. The PDF editor never hits this path: it only
* renders pages.
*/
export const ASYNC_POLL_CAP_MS = 1_800_000;
export function asyncPollMaxWaitMs(_fileSizeBytes: number): number {
return ASYNC_POLL_CAP_MS;
}
function nextPollDelayMs(elapsedMs: number, fallback: number): number {
if (elapsedMs < 60_000) return fallback;
if (elapsedMs < 300_000) return Math.max(fallback, 3_000);
return Math.max(fallback, 5_000);
}
export type AsyncConvertProgress = {
jobId: string;
status: ConvertJob['status'];
elapsedMs: number;
maxWaitMs: number;
};
async function downloadCompletedJob(
jobId: string,
target: string,
status: ConvertJob
): Promise<ConvertResult> {
let dl: Response;
try {
dl = await fetch(`${gatewayBaseUrl()}/v1/jobs/${jobId}/download`);
} catch (err) {
throw friendlyNetworkError(err);
}
const result = await parseConvertResponse(dl, target);
// Prefer UTF-8 original stem from job metadata when header fallback is ASCII-only
if (status.filename && /[^\x00-\x7F]/.test(status.filename)) {
result.filename = status.filename;
}
const metaScore = status.meta?.quality_score;
if (result.qualityScore == null && typeof metaScore === 'number') {
result.qualityScore = metaScore;
}
if (!result.fidelity && status.fidelity) result.fidelity = status.fidelity;
if (!result.warnings && status.warnings?.length) {
result.warnings = status.warnings.join(' | ');
}
return result;
}
/** Prefer async path for large uploads; poll until completed/failed. */
export async function convertFileMaybeAsync(
file: File,
source: string,
targetExtension: string,
opts?: {
forceAsync?: boolean;
pollMs?: number;
maxWaitMs?: number;
onProgress?: (p: AsyncConvertProgress) => void;
deskOptions?: ConvertDeskOptions;
}
): Promise<ConvertResult> {
const target = extensionToTarget(targetExtension);
const useAsync = opts?.forceAsync || file.size >= ASYNC_THRESHOLD_BYTES;
if (!useAsync) {
return convertFile(file, source, target, opts?.deskOptions);
}
const form = new FormData();
form.append('file', file, file.name || `upload.${source}`);
const qs = convertQuery(opts?.deskOptions);
let start: Response;
try {
start = await fetch(
`${gatewayBaseUrl()}/v1/async/convert/${source}/to/${target}${qs ? `?${qs}` : ''}`,
{
method: 'POST',
body: form,
}
);
} catch (err) {
throw friendlyNetworkError(err);
}
if (!start.ok) {
const bodyText = await start.text();
throw new Error(parseErrorDetail(start.status, bodyText));
}
const job = (await start.json()) as ConvertJob;
const pollMs = opts?.pollMs ?? 1500;
const maxWaitMs = opts?.maxWaitMs ?? asyncPollMaxWaitMs(file.size);
const t0 = Date.now();
const emit = (status: ConvertJob['status']) => {
opts?.onProgress?.({
jobId: job.job_id,
status,
elapsedMs: Date.now() - t0,
maxWaitMs,
});
};
emit(job.status || 'queued');
while (Date.now() - t0 < maxWaitMs) {
await new Promise((r) => setTimeout(r, nextPollDelayMs(Date.now() - t0, pollMs)));
let st: Response;
try {
st = await fetch(`${gatewayBaseUrl()}/v1/jobs/${job.job_id}`);
} catch (err) {
throw friendlyNetworkError(err);
}
if (!st.ok) {
throw new Error(parseErrorDetail(st.status, await st.text()));
}
const status = (await st.json()) as ConvertJob;
emit(status.status);
if (status.status === 'failed') {
throw new Error(status.error || 'Async conversion failed');
}
if (status.status === 'completed') {
return downloadCompletedJob(job.job_id, target, status);
}
}
// Final check: job may have completed in the last poll interval (race).
try {
const last = await fetch(`${gatewayBaseUrl()}/v1/jobs/${job.job_id}`);
if (last.ok) {
const status = (await last.json()) as ConvertJob;
if (status.status === 'completed') {
return downloadCompletedJob(job.job_id, target, status);
}
if (status.status === 'failed') {
throw new Error(status.error || 'Async conversion failed');
}
}
} catch (err) {
if (err instanceof Error && !/failed to fetch|networkerror/i.test(err.message)) {
throw err;
}
}
const mins = Math.round(maxWaitMs / 60_000);
throw new Error(
`Async conversion timed out after ~${mins} min while job ${job.job_id} was still running. ` +
`Keep the gateway running and retry Convert — a finished job can still be downloaded. ` +
`Do not close the tab while Converting is shown.`
);
}
export async function convertPdfFile(
file: File,
targetExtension: string
): Promise<ConvertResult> {
return convertFileMaybeAsync(file, 'pdf', targetExtension);
}
export function downloadBlob(blob: Blob, filename: string): void {
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = filename;
document.body.appendChild(a);
a.click();
a.remove();
URL.revokeObjectURL(url);
}
gateway/Dockerfile
+18 -3
View File
@@ -34,8 +34,11 @@ RUN git clone https://github.com/microsoft/vcpkg.git /opt/vcpkg \
&& /opt/vcpkg/bootstrap-vcpkg.sh -disableMetrics
ENV VCPKG_ROOT=/opt/vcpkg
# Grab uv binary for high-speed package management
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
# Ensure a recent CMake is available (system cmake in slim images can be too old for vcpkg)
RUN python -m pip install --upgrade pip cmake
RUN uv pip install --system cmake
# Cache vcpkg dependencies in a separate layer
COPY vcpkg.json ./
@@ -102,13 +105,25 @@ RUN groupadd --system app \
WORKDIR /home/app
# Copy uv binary for high-speed package installation
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
# Copy gateway Python code
COPY gateway/pyproject.toml gateway/README.md ./
RUN python -m pip install --upgrade pip \
&& pip install --no-cache-dir -e ".[dev]"
RUN uv pip install --system --no-cache -e ".[dev]"
COPY gateway/app ./app
COPY gateway/tests ./tests
# Model files are runtime inputs, not source-only documentation. Omitting
# them makes the image start successfully while silently disabling Arabic OCR
# and layout ML because all model lookups are intentionally fail-open.
COPY gateway/models ./models
COPY gateway/tools/verify_models.py ./tools/verify_models.py
# Fail the image build if a vendored model is missing, truncated, or has drifted
# from the checked-in checksum manifest. Optional capabilities remain optional
# according to MANIFEST.json, but a copied file can never be silently corrupt.
RUN python tools/verify_models.py
RUN chown -R app:app /home/app
USER app
gateway/README.md
+125
View File
@@ -40,7 +40,29 @@ for `import pdfengine` lazily and reports `engine_available: false` via
Requires Python 3.11+ (3.12 works). Build dirs are git-ignored.
### Fast path with `uv` (Recommended — 10x faster)
```powershell
# Windows
cd gateway
uv venv
.venv\Scripts\Activate.ps1
uv pip install -e ".[dev]"
```
```sh
# Linux/macOS
cd gateway
uv venv
source .venv/bin/activate
uv pip install -e ".[dev]"
```
<details>
<summary>Standard <code>pip</code> fallback</summary>
```powershell
# Windows
cd gateway
python -m venv .venv
.venv\Scripts\Activate.ps1
@@ -54,6 +76,7 @@ python -m venv .venv
source .venv/bin/activate
pip install -e ".[dev]"
```
</details>
Run the service:
@@ -77,6 +100,45 @@ uvicorn app.main:app --reload --port 8080
# → http://127.0.0.1:<port>/health
# → http://127.0.0.1:<port>/docs (OpenAPI UI)
## Document conversion API (OSS)
Wave 1+ converters live under `app/routers/convert` and `app/services/convert`.
No ConvertAPI/Aspose/PyMuPDF — `python-docx`, `openpyxl`, `reportlab`, `pypdf`,
and the C++ engine (when present) for text/layout/render.
| Method | Path | Notes |
| --- | --- | --- |
| GET | `/v1/convert/info` | Registered converter pairs + fidelity/gaps |
| GET | `/v1/convert/canconvert/{from}/to/{to}` | Capability probe |
| POST | `/v1/convert/{from}/to/{to}` | Sync convert (multipart `file`) |
| POST | `/v1/async/convert/{from}/to/{to}` | Returns `job_id` |
| GET | `/v1/jobs/{id}` | Job status |
| GET | `/v1/jobs/{id}/download` | Result bytes |
Success responses include `X-Fidelity: high|medium|lossy`, optional `X-Quality-Score` (0–1), and `X-Warnings`.
**Honesty (M2 + prod readiness):** `X-Quality-Score` is output-based; live table score uses source-derived grids. Sync convert uses `asyncio.to_thread`; hard timeouts return **504**. Soft cancel is between-pages only. Wave-1 Office stays `lossy`/`medium` — **shippable**, not ConvertAPI-class. Prod smoke: `pdf/corpus/convert/real/` + `tests/convert/test_prod_smoke.py`. Frontend: `pdf/frontend` Conversion desk.
**Accuracy path (Wave-1 climb):** PDF→Office/text goes through layout → Intermediate Document Model (**idm.v1**) → formatters. Tables **v3** (shared column schema + rulings + X-gap; under-detect preferred), span/font runs into DOCX/XLSX, headers/footers, multi-column reading order, RapidOCR + live Mistral OCR when configured. Corpus CI under `pdf/corpus/convert/` — see [docs/accuracy_climb.md](docs/accuracy_climb.md). No ConvertAPI/Aspose; **no LibreOffice**.
**Office→PDF (Path A — reportlab max):** in-process, concurrent-safe, no LibreOffice.
DOCX: BaseDocTemplate frames (incl. **multi-column**), font alias map, runs/lists/tables/nested
tables/images/**text boxes**/HF. XLSX: number formats, fills, merges, print area/titles,
freeze→repeat rows, column-band split, **chart→reportlab Drawing** (bar/line/pie). Fidelity
stays **medium** (not Word/Excel print clone). SSIM harness is a structural **proxy only** (no
raster SSIM in CI). Timeouts: `CONVERT_TIMEOUT_SECONDS`; OCR: `CONVERT_OCR_PAGE_CAP`.
**Implemented pairs (honest fidelity):** PDF↔DOCX/XLSX (lossy/medium), DOCX→PDF,
PDF→TXT/MD/JSON/HTML/PNG/JPEG/TIFF, HTML→PDF, MD→HTML/PDF, images→PDF. `pdf→json` exports **idm.v1**.
**Rejected in v1:** legacy `.doc`, `.docm`, encrypted PDFs, unknown pairs (422).
**Windows / multi-user note:** DOCX/XLSX→PDF uses **reportlab** in-process so many concurrent converts stay light. Fidelity is medium (not Word/Excel print-identical).
```powershell
pytest tests/convert -v
```
Lint, format, and test (the exact commands CI runs):
@@ -95,6 +157,8 @@ Environment variables are read by `app/config.py` with the prefix
|------------------------------|---------|--------------------------------------------------|
| `PDFENGINE_ENVIRONMENT` | `dev` | `dev` / `staging` / `prod` — echoed in `/health` |
| `PDFENGINE_ENGINE_AVAILABLE` | `false` | Forces the engine-availability flag for testing |
| `PDFENGINE_CORS_ALLOWED_ORIGINS` | `http://localhost:5173,http://127.0.0.1:5173` | Comma-separated browser origins allowed to call the gateway. |
| `PDFENGINE_CORS_ALLOW_CREDENTIALS` | `false` | Enables cookies/credentials only for explicitly listed origins; wildcard is rejected. |
Additionally, the gateway startup scripts and frontend configuration support:
@@ -102,6 +166,67 @@ Additionally, the gateway startup scripts and frontend configuration support:
|--------------------|-------------------------|--------------------------------------------------|
| `PORT` | `8765` | Gateway listening port (used by `start_gateway.ps1`). |
| `VITE_GATEWAY_URL` | `http://127.0.0.1:8765` | URL of the gateway API (used by the frontend). |
| `CONVERT_TIMEOUT_SECONDS` | `120` | Hard convert timeout; exceeded runs return **504**. |
| `CONVERT_OCR_PAGE_CAP` | `200` | Max pages OCR'd per document; capped runs report skipped pages in metadata/warnings. |
| `CONVERT_OCR_IMAGE_MAX_BYTES` | `15000000` | Per-page OCR raster budget; oversized pages are downsampled/compressed, never dropped. |
| `CONVERT_FIGURE_MAX_BYTES` | `4000000` | In-memory figure budget; large figures are compacted while retained. |
| `CONVERT_OCR_PAGE_TIMEOUT` | `30` | Per-page OCR timeout (seconds). |
| `CONVERT_LAYOUT_ML` | `0` | `1` enables optional Apache ONNX layout beside heuristics (fail-open). |
| `CONVERT_LAYOUT_ML_WEIGHTS` | `models/layout/v1/layout.onnx` | ONNX path (relative to gateway cwd). |
| `CONVERT_LAYOUT_ML_DEVICE` | `cpu` | `cpu` or `cuda`. |
| `CONVERT_LAYOUT_ML_DPI` | `150` | Page raster DPI for layout ML. |
| `CONVERT_LAYOUT_ML_PAGE_TIMEOUT` | `15` | Per-page ML soft timeout (seconds). |
| `CONVERT_LAYOUT_ML_MIN_SCORE` | `0.5` | Drop weak detection boxes. |
| `CONVERT_LAYOUT_ML_TABLE_STRUCTURE` | `0` | `1` enables optional table-structure ONNX (column separators only, fail-open). |
| `CONVERT_LAYOUT_ML_TABLE_STRUCTURE_WEIGHTS` | `models/layout/table/structure.onnx` | Structure ONNX path; missing ⇒ heuristics, no error. |
| `CONVERT_LAYOUT_ML_TABLE_STRUCTURE_MIN_SCORE` | `0.5` | Drop weak column/row boxes. |
| `CONVERT_LAYOUT_ML_TABLE_STRUCTURE_TIMEOUT` | `10` | Per-region structure timeout (seconds). |
| `CONVERT_OCR_ARABIC` | auto | Dual-pass Arabic rec; auto = on when `rec.onnx` exists. |
| `CONVERT_OCR_ARABIC_ADAPTIVE` | 0 (startup script: 1) | When enabled, skip the Arabic pass for high-confidence Latin pages and retry it for empty/garbled/Arabic-looking EN output. Set 0/always for forced dual-pass behavior. |
| `CONVERT_OCR_ARABIC_WEIGHTS` | `models/ocr/ar/v5/rec.onnx` | Arabic PP-OCRv5 rec ONNX path. |
| `CONVERT_OCR_ARABIC_DICT` | `models/ocr/ar/v5/arabic_dict.txt` | Arabic charset dict. |
| `CONVERT_OCR_REC_MODEL` | (unset) | Optional Latin/CJK **rec** ONNX. Do not reuse this path for det/cls. |
| `CONVERT_OCR_DET_MODEL` | (unset) | Optional text **det** ONNX. |
| `CONVERT_OCR_CLS_MODEL` | (unset) | Optional angle **cls** ONNX. |
Convert plugins default to **50 MB** max upload (`max_bytes`). The `pdf/frontend` Conversion desk uses async jobs for files ≥ 8 MB.
**Start gateway (recommended on Windows):** use the helper so port 8080 is freed and the **project venv** is used (avoids WinError 10048 from an orphan `python3.12` still listening):
```powershell
cd pdf\gateway
.\start_convert_gateway.ps1
```
If you start manually and see `only one usage of each socket address` / errno 10048:
```powershell
# Find and stop the holder
Get-NetTCPConnection -LocalPort 8080 -State Listen | Select OwningProcess
Stop-Process -Id <PID> -Force
```
Always start with `.\.venv\Scripts\python.exe` from `pdf\gateway` — not Windows Store `python.exe`.
**Layout ML / public eval:** see [`docs/ml_layout_free_stack.md`](docs/ml_layout_free_stack.md). Generate smoke fixtures:
```powershell
cd pdf
.\gateway\.venv\Scripts\python.exe scripts\convert\generate_public_smoke_fixtures.py
.\gateway\.venv\Scripts\python.exe -m pytest gateway\tests\convert -q -k "not scanned_page_count"
```
Enable layout ML later (PowerShell — quote URLs; no `<placeholders>`):
```powershell
cd pdf
.\gateway\.venv\Scripts\python.exe scripts\convert\fetch_layout_onnx.py
# when you have a real Apache ONNX URL:
.\gateway\.venv\Scripts\python.exe scripts\convert\fetch_layout_onnx.py --url "https://YOUR-HOST/layout.onnx" --sha256 "YOUR_SHA256_HEX"
cd gateway
$env:CONVERT_LAYOUT_ML="1"
$env:CONVERT_LAYOUT_ML_WEIGHTS="models/layout/v1/layout.onnx"
```
A `.env` file in `gateway/` is auto-loaded if present (it is git-ignored
via the repo-wide `.venv/` and Python rules — add `.env` to your local
gateway/_tmp_validate_huwiyati.py
+200
View File
@@ -0,0 +1,200 @@
"""One-shot validator for an operator-owned reconverted DOCX. Not a test."""
from __future__ import annotations
import collections
import re
import sys
import zipfile
from pathlib import Path
if sys.platform == "win32":
try:
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
except Exception:
pass
from docx import Document
from docx.oxml.ns import qn
from lxml import etree
AR = re.compile(r"[\u0600-\u06FF]")
GARBAGE = re.compile(
r"Arob|AUTflOR|AUTHORIIY|ldentity|ctltztt|cu5I0M|Jil-riYl|IIOERAI|"
r"IDINTITY|StCURtTy|@\s|Areb |Emlrates|Urted ",
re.I,
)
# Generic OCR salad, not agency names
SALAD = re.compile(r"\b(?:[A-Za-z]*[Il1]{2,}[A-Za-z]*){2,}\b")
def all_text(doc: Document) -> list[str]:
chunks = []
for p in doc.paragraphs:
t = p.text.strip()
if t:
chunks.append(t)
for table in doc.tables:
for row in table.rows:
for cell in row.cells:
t = cell.text.strip()
if t:
chunks.append(t)
for section in doc.sections:
for part in (section.header, section.footer):
for p in part.paragraphs:
t = p.text.strip()
if t:
chunks.append(f"[hf] {t}")
for table in part.tables:
for row in table.rows:
for cell in row.cells:
t = cell.text.strip()
if t:
chunks.append(f"[hf-table] {t}")
return chunks
def count_drawings(xml: bytes) -> int:
root = etree.fromstring(xml)
ns = {
"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
"wp": "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing",
"a": "http://schemas.openxmlformats.org/drawingml/2006/main",
"r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
}
return len(root.xpath(".//w:drawing", namespaces=ns))
def run_font_sizes(doc: Document) -> list[tuple[float, str]]:
out = []
for p in doc.paragraphs:
for run in p.runs:
t = (run.text or "").strip()
if not t or run.font.size is None:
continue
pt = run.font.size.pt
if pt and pt >= 20:
out.append((pt, t[:80]))
return out
def page_breaks(doc: Document) -> int:
n = 0
body = doc.element.body
for el in body.iter():
if el.tag == qn("w:br") and el.get(qn("w:type")) == "page":
n += 1
if el.tag == qn("w:lastRenderedPageBreak"):
n += 1
return n
def sect_page_fields(doc: Document) -> bool:
xml = doc.element.xml
return "PAGE" in xml or "w:instrText" in xml
def main(path: Path) -> None:
data = path.read_bytes()
print(f"file={path.name}")
print(f"bytes={len(data)}")
doc = Document(str(path))
paras = [p.text.strip() for p in doc.paragraphs if p.text.strip()]
tables = doc.tables
print(f"body_paras={len(paras)}")
print(f"tables={len(tables)}")
print(f"sections={len(doc.sections)}")
texts = all_text(doc)
joined = "\n".join(texts)
ar_chars = len(AR.findall(joined))
print(f"arabic_chars={ar_chars}")
print(f"latin_letters={sum(1 for c in joined if c.isascii() and c.isalpha())}")
print(f"total_chars={len(joined)}")
hits = GARBAGE.findall(joined)
print(f"known_garbage_hits={len(hits)} unique={sorted(set(hits))[:20]}")
logo_ph = sum(1 for t in texts if "[Logo]" in t or t.strip() == "Logo")
print(f"logo_placeholders={logo_ph}")
# header/footer
hdr_txt, ftr_txt, hdr_draw, ftr_draw = [], [], 0, 0
page_field = False
for i, section in enumerate(doc.sections):
h, f = section.header, section.footer
hdr_txt.extend(p.text.strip() for p in h.paragraphs if p.text.strip())
ftr_txt.extend(p.text.strip() for p in f.paragraphs if p.text.strip())
for t in h.tables:
for row in t.rows:
for c in row.cells:
if c.text.strip():
hdr_txt.append(c.text.strip())
hdr_draw += count_drawings(h._element.xml.encode() if isinstance(h._element.xml, str) else h._element.xml)
# python-docx returns str xml
hx = etree.tostring(h._element)
fx = etree.tostring(f._element)
hdr_draw = count_drawings(hx) if i == 0 else hdr_draw + count_drawings(hx)
ftr_draw += count_drawings(fx)
fxml = f._element.xml
if "PAGE" in fxml or "instrText" in fxml:
page_field = True
print("header_lines:")
for t in hdr_txt[:15]:
print(f" {t[:120]!r}")
print("footer_lines:")
for t in ftr_txt[:10]:
print(f" {t[:120]!r}")
print(f"header_drawings={hdr_draw} footer_drawings={ftr_draw} page_field={page_field}")
print("table_shapes:")
for i, t in enumerate(tables):
rows, cols = len(t.rows), len(t.columns)
sample = " | ".join((c.text.strip()[:40] for c in t.rows[0].cells))
print(f" t{i}: {rows}x{cols} :: {sample[:140]}")
sizes = run_font_sizes(doc)
print(f"large_runs>={20}pt: {len(sizes)}")
for pt, t in sizes[:12]:
print(f" {pt}pt {t!r}")
# zip media
with zipfile.ZipFile(path) as z:
media = [n for n in z.namelist() if n.startswith("word/media/")]
print(f"media={len(media)} {media}")
for n in media:
info = z.getinfo(n)
print(f" {n} {info.file_size}b")
rels = z.read("word/_rels/document.xml.rels").decode("utf-8", "replace")
print(f"header_rels={'header' in ''.join(z.namelist())}")
headers = [n for n in z.namelist() if "header" in n]
footers = [n for n in z.namelist() if "footer" in n]
print(f"header_parts={headers} footer_parts={footers}")
# first / last body paras
print("first_paras:")
for t in paras[:12]:
print(f" {t[:160]!r}")
print("sample_arabic_paras:")
ar_paras = [t for t in paras if AR.search(t)]
print(f" count={len(ar_paras)}")
for t in ar_paras[:8]:
print(f" {t[:160]!r}")
# garbage paragraphs
bad = [t for t in paras if GARBAGE.search(t)]
print(f"garbage_body_paras={len(bad)}")
for t in bad[:15]:
print(f" {t[:160]!r}")
# @ heading
at = [t for t in paras if t.strip() in ("@",) or t.startswith("@ ")]
print(f"at_paras={at[:5]}")
print(f"lastRendered_or_page_br_approx={page_breaks(doc)}")
if __name__ == "__main__":
p = Path(sys.argv[1])
main(p)
gateway/app/ai/models/base.py
+1 -1
View File
@@ -13,7 +13,7 @@ class BaseModelWrapper(ABC):
self._is_loaded = False
@abstractmethod
def load((self) -> bool:
def load(self) -> bool:
"""Load model weights into memory."""
pass
gateway/app/config.py
+46 -4
View File
@@ -7,7 +7,7 @@ needed for `/health` and the app factory.
from functools import lru_cache
from pydantic import Field
from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -26,18 +26,60 @@ class Settings(BaseSettings):
"Phase 0 default is False — routes that need the engine return 501."
),
)
render_cache_enabled: bool = Field(default=True)
render_cache_max_entries: int = Field(default=256)
render_cache_max_bytes: int = Field(
default=536_870_912,
description="Max total bytes for the tile render cache. Set to 0 to disable byte cap."
description="Max total bytes for the tile render cache. Set to 0 to disable byte cap.",
)
render_cache_max_entry_bytes: int = Field(
default=8_388_608,
description="Max bytes for a single cache entry. Oversized tiles are rendered but not cached."
description="Max bytes for a single cache entry. Oversized tiles are rendered but not cached.",
)
gotenberg_url: str = Field(
default="http://127.0.0.1:3000",
description="URL for the Gotenberg headless microservice for high-fidelity Office->PDF.",
)
# CORS is explicit by default. A comma-separated string keeps deployment
# configuration ergonomic across Docker/PowerShell/systemd, while the
# property below gives the middleware a normalized list.
cors_allowed_origins: str = Field(
default="http://localhost:5173,http://127.0.0.1:5173",
description="Comma-separated browser origins allowed to call the gateway.",
)
cors_allow_credentials: bool = Field(
default=False,
description="Allow browser credentials only with an explicit origin allowlist.",
)
convert_max_concurrent: int = Field(
default=4,
description="Max concurrent conversion jobs executed simultaneously to prevent memory exhaustion.",
)
convert_queue_timeout_seconds: float = Field(
default=60.0,
description="Max seconds a request will wait in the queue before timing out with 504.",
)
@property
def cors_origins(self) -> list[str]:
"""Return normalized CORS origins, preserving ``*`` when explicitly set."""
return [origin.strip() for origin in self.cors_allowed_origins.split(",") if origin.strip()]
@model_validator(mode="after")
def validate_cors(self) -> "Settings":
"""Keep wildcard origins out of credentialed and production deployments."""
if self.cors_allow_credentials and "*" in self.cors_origins:
raise ValueError(
"PDFENGINE_CORS_ALLOWED_ORIGINS cannot contain '*' when "
"PDFENGINE_CORS_ALLOW_CREDENTIALS is enabled"
)
if self.environment.strip().lower() == "prod" and "*" in self.cors_origins:
raise ValueError("PDFENGINE_CORS_ALLOWED_ORIGINS cannot contain '*' in production")
return self
@lru_cache(maxsize=1)
def get_settings() -> Settings:
gateway/app/main.py
+27 -4
View File
@@ -1,13 +1,26 @@
"""FastAPI app factory."""
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app import __version__
from app.routers import compare, documents, edits, health, info, internal, layout, ocr, render
from app.services import ocr as ocr_service
from app.ai.services.font_service import FontRecognitionService
from app.config import get_settings
from app.routers import (
compare,
convert,
documents,
edits,
health,
info,
internal,
layout,
ocr,
render,
)
from app.services import ocr as ocr_service
@asynccontextmanager
@@ -44,6 +57,7 @@ async def lifespan(app: FastAPI):
def create_app() -> FastAPI:
settings = get_settings()
app = FastAPI(
title="PDF Engine Gateway",
version=__version__,
@@ -53,14 +67,23 @@ def create_app() -> FastAPI:
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_origins=settings.cors_origins,
allow_credentials=settings.cors_allow_credentials,
allow_methods=["*"],
allow_headers=["*"],
expose_headers=[
"Content-Disposition",
"X-Fidelity",
"X-Warnings",
"X-Quality-Score",
"X-Convert-Source",
"X-Convert-Target",
],
)
app.include_router(health.router)
app.include_router(info.router)
app.include_router(convert.router)
app.include_router(documents.router)
app.include_router(compare.router)
app.include_router(render.router)
gateway/app/routers/convert/__init__.py
+9
View File
@@ -0,0 +1,9 @@
"""Convert routers package."""
from fastapi import APIRouter
from . import info, jobs
router = APIRouter()
router.include_router(info.router)
router.include_router(jobs.router)
gateway/app/routers/convert/info.py
+28
View File
@@ -0,0 +1,28 @@
"""Convert info endpoints."""
from fastapi import APIRouter
from app.schemas.convert import CanConvertResponse, ConvertInfoResponse
from app.services.convert.registry import registry
router = APIRouter(prefix="/v1/convert", tags=["convert"])
@router.get("/info", response_model=ConvertInfoResponse)
def convert_info() -> ConvertInfoResponse:
return ConvertInfoResponse(converters=registry.list_info())
@router.get("/canconvert/{source}/to/{target}", response_model=CanConvertResponse)
def can_convert(source: str, target: str) -> CanConvertResponse:
plugin = registry.get(source, target)
if not plugin:
return CanConvertResponse(can_convert=False, source=source, target=target)
return CanConvertResponse(
can_convert=True,
source=plugin.source,
target=plugin.target,
fidelity=plugin.fidelity,
engine=plugin.engine,
gaps=list(plugin.gaps),
)
gateway/app/routers/convert/jobs.py
+502
View File
@@ -0,0 +1,502 @@
"""Sync/async conversion job endpoints."""
from __future__ import annotations
import asyncio
import os
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from typing import Annotated
from fastapi import APIRouter, File, HTTPException, Query, UploadFile, status
from fastapi.responses import Response
from app.schemas.convert import ConvertJobResponse, JobStatus
from app.services.convert.http_headers import (
ascii_header_value,
content_disposition_attachment,
)
from app.services.convert.jobs import job_store
from app.services.convert.options import (
ConvertOptions,
HeaderFooterMode,
LayoutMode,
OcrEngineChoice,
OcrPolicy,
RecognitionMode,
)
from app.services.convert.pipeline import ConversionError, run_conversion
from app.services.convert.validate import ValidationError, media_type_for
router = APIRouter(prefix="/v1", tags=["convert"])
_MAX_WORKERS = max(1, int(os.environ.get("CONVERT_MAX_WORKERS", "4")))
_MAX_CONCURRENT = max(1, int(os.environ.get("CONVERT_MAX_CONCURRENT", "4")))
_QUEUE_TIMEOUT = float(os.environ.get("CONVERT_QUEUE_TIMEOUT_SECONDS", "60.0"))
_MAX_QUEUED = max(
0, int(os.environ.get("CONVERT_MAX_QUEUED", str(_MAX_WORKERS * 2)))
)
_executor = ThreadPoolExecutor(max_workers=_MAX_WORKERS, thread_name_prefix="convert-worker")
_semaphore: asyncio.Semaphore | None = None
# The executor itself has an unbounded work queue. This admission gate keeps
# uploaded payloads and job records bounded at ``workers + queued`` jobs.
_async_admission = threading.BoundedSemaphore(_MAX_WORKERS + _MAX_QUEUED)
# Sync requests and async workers share the same actual conversion budget. A
# separate asyncio semaphore cannot coordinate work running in executor
# threads, so this process-wide slot guard is intentionally thread based.
_execution_slots = threading.BoundedSemaphore(_MAX_CONCURRENT)
def _get_semaphore() -> asyncio.Semaphore:
global _semaphore
if _semaphore is None:
_semaphore = asyncio.Semaphore(_MAX_CONCURRENT)
return _semaphore
def _acquire_execution_slot(cancel_check=None) -> bool:
"""Wait briefly at a time so queued async jobs can observe cancellation."""
deadline = time.monotonic() + max(0.0, _QUEUE_TIMEOUT)
while True:
if cancel_check is not None and cancel_check():
return False
remaining = deadline - time.monotonic()
if remaining <= 0:
return False
if _execution_slots.acquire(timeout=min(0.25, remaining)):
return True
def _run_sync_conversion(
data: bytes,
filename: str,
source: str,
target: str,
cancel_check,
options: ConvertOptions | None,
on_resolved,
):
"""Run one synchronous conversion while holding a shared process slot."""
if not _acquire_execution_slot(cancel_check):
raise ConversionError(
"Conversion queue timeout: server is at capacity. Please retry shortly.",
status_code=504,
)
try:
return run_conversion(
data,
filename,
source,
target,
cancel_check,
options,
on_resolved,
)
finally:
_execution_slots.release()
def _options_metadata(
requested: ConvertOptions | None,
effective: ConvertOptions | None = None,
) -> dict[str, object]:
"""Expose the actual knobs while retaining whether routing was automatic."""
if effective is None:
if requested is None:
return {"auto": True}
return requested.as_dict()
values: dict[str, object] = dict(effective.as_dict())
if requested is None:
values["auto"] = True
return values
def _options_from_query(
layout_mode: LayoutMode | None,
ocr_policy: OcrPolicy | None,
ocr_engine: OcrEngineChoice | None,
header_footer_mode: HeaderFooterMode | None,
recognition_mode: RecognitionMode | None,
detect_tables: bool | None,
) -> ConvertOptions | None:
"""Build options from the query, or None when the caller supplied none.
Every reconstruction parameter defaults to ``None`` rather than to its
value, so "omitted" is distinguishable from "explicitly set to the
default". The conversion desk sends none of them and gets automatic
routing; CI and power users pass what they want and it is honoured
exactly, including a value that happens to equal a default.
"""
supplied = (
layout_mode,
ocr_policy,
ocr_engine,
header_footer_mode,
recognition_mode,
detect_tables,
)
if all(v is None for v in supplied):
return None
fallback = ConvertOptions()
return ConvertOptions(
layout_mode=layout_mode if layout_mode is not None else fallback.layout_mode,
ocr_policy=ocr_policy if ocr_policy is not None else fallback.ocr_policy,
ocr_engine=ocr_engine if ocr_engine is not None else fallback.ocr_engine,
header_footer_mode=(
header_footer_mode
if header_footer_mode is not None
else fallback.header_footer_mode
),
recognition_mode=(
recognition_mode if recognition_mode is not None else fallback.recognition_mode
),
detect_tables=detect_tables if detect_tables is not None else fallback.detect_tables,
)
def _run_job(
job_id: str,
data: bytes,
filename: str,
source: str,
target: str,
options: ConvertOptions | None,
) -> None:
slot_acquired = False
resolved: dict[str, ConvertOptions] = {}
try:
if job_store.is_cancelled(job_id):
job_store.update(
job_id,
status=JobStatus.failed.value,
error="cancelled",
result_bytes=None,
)
return
# Executor workers may start before an execution slot is available
# (for example while synchronous requests are using all slots). Poll
# in short intervals so cancellation remains responsive.
slot_acquired = _acquire_execution_slot(
cancel_check=lambda: job_store.is_cancelled(job_id)
)
if not slot_acquired:
if job_store.is_cancelled(job_id):
job_store.update(
job_id,
status=JobStatus.failed.value,
error="cancelled",
result_bytes=None,
)
else:
job_store.update(
job_id,
status=JobStatus.failed.value,
error=(
"Conversion queue timeout: server is at capacity. "
"Please retry shortly."
),
)
return
job_store.update(job_id, status=JobStatus.running.value)
if job_store.is_cancelled(job_id):
job_store.update(
job_id,
status=JobStatus.failed.value,
error="cancelled",
result_bytes=None,
)
return
out, fidelity, warnings, _media, out_name, quality_score = run_conversion(
data,
filename,
source,
target,
cancel_check=lambda: job_store.is_cancelled(job_id),
options=options,
on_resolved=lambda opts: resolved.__setitem__("options", opts),
)
if job_store.is_cancelled(job_id):
job_store.update(
job_id,
status=JobStatus.failed.value,
error="cancelled",
result_bytes=None,
)
return
effective = resolved.get("options") or options
meta: dict = {"convert_options": _options_metadata(options, effective)}
if quality_score is not None:
meta["quality_score"] = round(quality_score, 4)
job_store.update(
job_id,
status=JobStatus.completed.value,
fidelity=fidelity.value,
warnings=warnings,
size_bytes=len(out),
result_bytes=out,
result_filename=out_name,
download_path=f"/v1/jobs/{job_id}/download",
error=None,
meta=meta,
)
except (ValidationError, ConversionError) as exc:
code = getattr(exc, "status_code", 500)
err_code = getattr(exc, "code", None)
meta = {
"status_code": code,
"convert_options": _options_metadata(options, resolved.get("options")),
}
if err_code is not None:
meta["code"] = getattr(err_code, "value", str(err_code))
job_store.update(
job_id,
status=JobStatus.failed.value,
error=str(exc),
meta=meta,
)
except Exception as exc:
job_store.update(job_id, status=JobStatus.failed.value, error=f"Unexpected error: {exc}")
finally:
if slot_acquired:
_execution_slots.release()
def _run_admitted_job(
job_id: str,
data: bytes,
filename: str,
source: str,
target: str,
options: ConvertOptions | None,
) -> None:
"""Run an admitted async job and always return its admission slot."""
try:
_run_job(job_id, data, filename, source, target, options)
finally:
_async_admission.release()
@router.post("/convert/{source}/to/{target}")
async def convert_sync(
source: str,
target: str,
file: UploadFile = File(...),
layout_mode: Annotated[LayoutMode | None, Query()] = None,
ocr_policy: Annotated[OcrPolicy | None, Query()] = None,
ocr_engine: Annotated[OcrEngineChoice | None, Query()] = None,
header_footer_mode: Annotated[HeaderFooterMode | None, Query()] = None,
recognition_mode: Annotated[RecognitionMode | None, Query()] = None,
detect_tables: Annotated[bool | None, Query()] = None,
) -> Response:
data = await file.read()
filename = file.filename or f"upload.{source}"
options = _options_from_query(
layout_mode,
ocr_policy,
ocr_engine,
header_footer_mode,
recognition_mode,
detect_tables,
)
sem = _get_semaphore()
acquired = False
try:
try:
await asyncio.wait_for(sem.acquire(), timeout=_QUEUE_TIMEOUT)
acquired = True
except TimeoutError:
raise HTTPException(
status_code=status.HTTP_504_GATEWAY_TIMEOUT,
detail="Conversion queue timeout: server is at capacity. Please retry shortly.",
)
resolved: dict[str, ConvertOptions] = {}
out, fidelity, warnings, media, out_name, quality_score = await asyncio.to_thread(
_run_sync_conversion,
data,
filename,
source,
target,
None,
options,
lambda opts: resolved.__setitem__("options", opts),
)
effective = resolved.get("options") or options or ConvertOptions()
except ValidationError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.to_detail()) from exc
except ConversionError as exc:
raise HTTPException(status_code=exc.status_code, detail=str(exc)) from exc
except HTTPException:
raise
except Exception as exc:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Convert failed: {type(exc).__name__}: {exc}",
) from exc
finally:
if acquired:
sem.release()
warn_header = ascii_header_value(" | ".join(warnings)) if warnings else ""
headers = {
"Content-Disposition": content_disposition_attachment(out_name),
"X-Fidelity": fidelity.value,
"X-Warnings": warn_header,
"X-Convert-Source": source,
"X-Convert-Target": target,
# Report what the engine actually used, not the literal word "auto":
# a caller debugging an output needs the path that ran.
"X-Layout-Mode": effective.layout_mode.value,
"X-Ocr-Policy": effective.ocr_policy.value,
"X-Reconstruction": "auto" if options is None else "explicit",
}
if quality_score is not None:
headers["X-Quality-Score"] = f"{quality_score:.4f}"
return Response(content=out, media_type=media, headers=headers)
@router.post("/async/convert/{source}/to/{target}", response_model=ConvertJobResponse)
async def convert_async(
source: str,
target: str,
file: UploadFile = File(...),
layout_mode: Annotated[LayoutMode | None, Query()] = None,
ocr_policy: Annotated[OcrPolicy | None, Query()] = None,
ocr_engine: Annotated[OcrEngineChoice | None, Query()] = None,
header_footer_mode: Annotated[HeaderFooterMode | None, Query()] = None,
recognition_mode: Annotated[RecognitionMode | None, Query()] = None,
detect_tables: Annotated[bool | None, Query()] = None,
) -> ConvertJobResponse:
# Acquire before reading the upload so rejected requests do not retain a
# second in-memory copy while waiting behind a saturated executor.
if not _async_admission.acquire(blocking=False):
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=(
"Async conversion queue is full. Please retry shortly."
),
headers={"Retry-After": "1"},
)
submitted = False
try:
data = await file.read()
filename = file.filename or f"upload.{source}"
options = _options_from_query(
layout_mode,
ocr_policy,
ocr_engine,
header_footer_mode,
recognition_mode,
detect_tables,
)
job = job_store.create(source=source, target=target, filename=filename)
try:
_executor.submit(
_run_admitted_job,
job["job_id"],
data,
filename,
source,
target,
options,
)
submitted = True
except Exception as exc:
job_store.update(
job["job_id"],
status=JobStatus.failed.value,
error=f"Unable to queue conversion: {exc}",
)
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Conversion queue is unavailable. Please retry shortly.",
headers={"Retry-After": "1"},
) from exc
return ConvertJobResponse(
job_id=job["job_id"],
status=JobStatus.queued,
source=source,
target=target,
filename=filename,
download_path=None,
meta={"convert_options": _options_metadata(options)},
)
finally:
# The worker owns the slot only after submit succeeds. Every path
# before that point, including upload/read and DB failures, releases it.
if not submitted:
_async_admission.release()
@router.get("/jobs/{job_id}", response_model=ConvertJobResponse)
def get_job(job_id: str) -> ConvertJobResponse:
job = job_store.get(job_id)
if not job:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Job not found")
fidelity = job.get("fidelity")
return ConvertJobResponse(
job_id=job["job_id"],
status=JobStatus(job["status"]),
source=job.get("source"),
target=job.get("target"),
fidelity=fidelity,
warnings=job.get("warnings") or [],
filename=job.get("result_filename") or job.get("filename"),
size_bytes=job.get("size_bytes"),
error=job.get("error"),
download_path=job.get("download_path") if job["status"] == JobStatus.completed.value else None,
meta=job.get("meta") or {},
)
@router.get("/jobs/{job_id}/download")
def download_job(job_id: str) -> Response:
job = job_store.get(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
if job["status"] != JobStatus.completed.value:
raise HTTPException(status_code=409, detail=f"Job is {job['status']}")
data = job_store.get_result_bytes(job_id)
if not data:
raise HTTPException(status_code=404, detail="Result unavailable")
target = job.get("target") or "bin"
filename = job.get("result_filename") or f"converted.{target}"
warn_header = ascii_header_value(" | ".join(job.get("warnings") or []))
headers = {
"Content-Disposition": content_disposition_attachment(filename),
"X-Fidelity": ascii_header_value(str(job.get("fidelity") or "")),
}
if warn_header:
headers["X-Warnings"] = warn_header
meta = job.get("meta") or {}
qs = meta.get("quality_score")
if qs is not None:
headers["X-Quality-Score"] = f"{float(qs):.4f}"
return Response(
content=data,
media_type=media_type_for(target),
headers=headers,
)
@router.delete("/jobs/{job_id}", response_model=ConvertJobResponse)
def delete_job(job_id: str) -> ConvertJobResponse:
job = job_store.get(job_id)
if not job:
raise HTTPException(status_code=404, detail="Job not found")
job_store.request_cancel(job_id)
job_store.update(job_id, status=JobStatus.failed.value, error="cancelled", result_bytes=None)
return ConvertJobResponse(
job_id=job_id,
status=JobStatus.failed,
source=job.get("source"),
target=job.get("target"),
error="cancelled",
)
gateway/app/routers/documents/export.py
+2 -1
View File
@@ -5,6 +5,7 @@ from fastapi import APIRouter, HTTPException, Response, status
from pydantic import BaseModel
from app.services import engine
from app.services.convert.http_headers import content_disposition_attachment
from app.services.store import document_store
router = APIRouter(tags=["documents"])
@@ -44,7 +45,7 @@ def export_document(document_id: str):
content=bytes_data,
media_type="application/pdf",
headers={
"Content-Disposition": f'attachment; filename="{filename}"',
"Content-Disposition": content_disposition_attachment(filename),
"Content-Length": str(len(bytes_data)),
},
)
gateway/app/routers/documents/fonts.py
+4 -4
View File
@@ -54,18 +54,18 @@ def get_font_bytes(document_id: str, internal_font_id: str) -> Response:
)
if not internal_font_id or len(internal_font_id) > 256:
return Response(status_code=status.HTTP_204_NO_CONTENT)
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Font not found")
doc_info = document_store.get_document(document_id)
if not doc_info:
return Response(status_code=status.HTTP_204_NO_CONTENT)
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
try:
data = bytes(doc_info["doc_instance"].get_font_data(internal_font_id))
except Exception:
data = b""
if not data:
return Response(status_code=status.HTTP_204_NO_CONTENT)
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Font not found")
magic = data[:4]
if magic in _SFNT_TTF_MAGIC:
@@ -73,7 +73,7 @@ def get_font_bytes(document_id: str, internal_font_id: str) -> Response:
elif magic == _SFNT_OTF_MAGIC:
media_type = "font/otf"
else:
return Response(status_code=status.HTTP_204_NO_CONTENT)
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Font not found")
etag = '"' + hashlib.sha256(data).hexdigest()[:32] + '"'
return Response(
gateway/app/routers/health.py
+34 -1
View File
@@ -12,6 +12,7 @@ from pydantic import BaseModel
from app import __version__
from app.config import Settings, get_settings
from app.services import engine as engine_service
router = APIRouter(tags=["health"])
@@ -23,6 +24,37 @@ class HealthResponse(BaseModel):
version: str
environment: str
engine_available: bool
# What this process can actually do. Every model path in the converter is
# fail-open, so a deploy that forgot the weights serves happily with
# Arabic recognition off and no page renderer. Without this an operator
# has to read startup logs to find that out — usually after a customer does.
capabilities: dict[str, bool] = {}
def _capabilities() -> dict[str, bool]:
"""Probe each optional capability, never raising: /health must always answer."""
caps: dict[str, bool] = {}
try:
from app.services import raster
caps["page_raster"] = raster.is_available()
except Exception:
caps["page_raster"] = False
try:
from app.services import ocr
caps["ocr"] = ocr.is_ocr_available()
caps["ocr_arabic"] = ocr.is_ocr_arabic_available()
except Exception:
caps["ocr"] = False
caps["ocr_arabic"] = False
try:
from app.services.convert.layout import ml_regions
caps["layout_ml"] = ml_regions.layout_ml_enabled() and ml_regions._weights_path().is_file()
except Exception:
caps["layout_ml"] = False
return caps
@router.get("/health", response_model=HealthResponse)
@@ -31,5 +63,6 @@ def health(settings: SettingsDep) -> HealthResponse:
status="ok",
version=__version__,
environment=settings.environment,
engine_available=settings.engine_available,
engine_available=engine_service.is_available() or settings.engine_available,
capabilities=_capabilities(),
)
gateway/app/schemas/convert.py
+87
View File
@@ -0,0 +1,87 @@
"""Conversion API schemas."""
from __future__ import annotations
from enum import Enum
from typing import Any
from pydantic import BaseModel, Field
class Fidelity(str, Enum):
high = "high"
medium = "medium"
lossy = "lossy"
class JobStatus(str, Enum):
queued = "queued"
running = "running"
completed = "completed"
failed = "failed"
class ConvertFormat(str, Enum):
pdf = "pdf"
docx = "docx"
xlsx = "xlsx"
pptx = "pptx"
html = "html"
md = "md"
txt = "txt"
json = "json"
png = "png"
jpeg = "jpeg"
jpg = "jpg"
tiff = "tiff"
csv = "csv"
xml = "xml"
class ConverterInfo(BaseModel):
source: str
target: str
fidelity: Fidelity
sync_ok: bool = True
max_bytes: int = 50 * 1024 * 1024
description: str = ""
gaps: list[str] = Field(default_factory=list)
class ConvertInfoResponse(BaseModel):
converters: list[ConverterInfo]
class CanConvertResponse(BaseModel):
can_convert: bool
source: str
target: str
fidelity: Fidelity | None = None
engine: str | None = None
gaps: list[str] = Field(default_factory=list)
class ConvertResultMeta(BaseModel):
fidelity: Fidelity
warnings: list[str] = Field(default_factory=list)
source: str
target: str
filename: str
size_bytes: int
page_count: int | None = None
engine: str = "oss"
convert_options: dict[str, Any] | None = None
class ConvertJobResponse(BaseModel):
job_id: str
status: JobStatus
source: str | None = None
target: str | None = None
fidelity: Fidelity | None = None
warnings: list[str] = Field(default_factory=list)
filename: str | None = None
size_bytes: int | None = None
error: str | None = None
download_path: str | None = None
meta: dict[str, Any] = Field(default_factory=dict)
gateway/app/services/convert/__init__.py
+7
View File
@@ -0,0 +1,7 @@
"""Conversion service package."""
from app.services.convert.pipeline import ConversionError, run_conversion
from app.services.convert.registry import registry
from app.services.convert.validate import ValidationError
__all__ = ["ConversionError", "ValidationError", "registry", "run_conversion"]
gateway/app/services/convert/auto_policy.py
+316
View File
@@ -0,0 +1,316 @@
"""Automatic reconstruction policy — the engine picks the path, not the user.
The conversion desk asks for one thing only: the output format. Everything
else — whether to OCR, whether to reflow or position, how to treat running
headers, whether to look for tables — is decided here from what the document
actually is.
Why route first rather than run one universal path: a clean digital contract, a
photographed invoice and a bilingual RFP with a broken ToUnicode map need
different treatment, and applying any one of those treatments to the other two
makes the output worse. Routing costs a page scan; guessing costs the document.
Honest limits, stated the same way in the UI and the docs:
* Word and Excel are **reconstructions**. They stay labelled lossy.
* PNG/JPEG/TIFF of pages are exact rasters of what was uploaded.
* A page that cannot be reconstructed is embedded as a page image rather than
having a layout invented for it.
Every decision is recorded on ``document.meta.convert_policy`` so a support
question ("why did page 7 come out as a picture?") has an answer.
"""
from __future__ import annotations
import os
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, field
from app.services.convert.layout.pdf_router import PdfDocType, PdfRouteResult
from app.services.convert.options import (
ConvertOptions,
HeaderFooterMode,
LayoutMode,
OcrPolicy,
RecognitionMode,
)
# Targets that are reconstructions of the document's *content*.
OFFICE_TARGETS = frozenset({"docx", "xlsx", "csv", "pptx", "html", "md", "txt", "json"})
# Targets that reproduce the page as pixels; nothing here is reconstructed.
RASTER_TARGETS = frozenset({"png", "jpeg", "jpg", "tiff", "tif"})
# --- brochure detection -----------------------------------------------------
# A brochure-like page is mostly artwork with captions placed around it.
# Reflowing one produces a single mashed paragraph, so it is emitted positioned.
BROCHURE_IMAGE_COVERAGE = 0.45
# Below this many words a page is not carrying body prose.
BROCHURE_MAX_WORDS = 120
# Short positioned blocks per paragraph: a poster is many small labels, a report
# is a few long paragraphs.
BROCHURE_BLOCK_RATIO = 2.5
# The share of pages that must look like this before the *document* is treated
# as brochure-like. One art page in a report does not change the whole file.
BROCHURE_PAGE_SHARE = 0.6
# A page with no body prose at all does not need table detection; looking for
# grids in running prose is where soup tables come from.
PROSE_ONLY_MAX_TABLE_HINT = 0.02
def auto_enabled() -> bool:
"""Whether omitted knobs should be filled in by this module."""
raw = (os.environ.get("CONVERT_AUTO_RECONSTRUCT") or "1").strip().lower()
return raw not in ("0", "false", "off", "no")
@dataclass
class DocumentSignals:
"""What we know about the document before choosing a path."""
doc_type: PdfDocType = PdfDocType.text_based
confidence: float = 0.0
page_count: int = 1
pages_needing_ocr: list[int] = field(default_factory=list)
# Mean fraction of page area covered by images.
image_coverage: float = 0.0
# Pages that are one full-page raster and nothing else.
full_page_raster_pages: list[int] = field(default_factory=list)
# Words of usable digital text across the document.
text_words: int = 0
# Positioned blocks vs paragraphs, averaged per page.
block_to_paragraph_ratio: float = 0.0
brochure_pages: list[int] = field(default_factory=list)
@property
def ocr_fraction(self) -> float:
return len(self.pages_needing_ocr) / max(self.page_count, 1)
@property
def brochure_fraction(self) -> float:
return len(self.brochure_pages) / max(self.page_count, 1)
@dataclass
class PolicyDecision:
"""The chosen options plus why, for logging and for X-Warnings."""
options: ConvertOptions
reasons: list[str] = field(default_factory=list)
signals: DocumentSignals = field(default_factory=DocumentSignals)
def as_dict(self) -> dict:
return {
"auto": True,
"options": self.options.as_dict(),
"reasons": list(self.reasons),
"doc_type": self.signals.doc_type.value,
"confidence": round(self.signals.confidence, 3),
"pages_needing_ocr": len(self.signals.pages_needing_ocr),
"page_count": self.signals.page_count,
"image_coverage": round(self.signals.image_coverage, 3),
"brochure_pages": len(self.signals.brochure_pages),
}
def summary(self) -> str:
"""One short line for X-Warnings — no internal jargon."""
opts = self.options
bits = [f"auto: {self.signals.doc_type.value}"]
if opts.layout_mode != LayoutMode.flowing:
bits.append(f"layout={opts.layout_mode.value}")
if opts.ocr_policy != OcrPolicy.auto:
bits.append(f"ocr={opts.ocr_policy.value}")
if not opts.detect_tables:
bits.append("tables=off")
return "; ".join(bits)
def _decide_ocr(signals: DocumentSignals, reasons: list[str]) -> OcrPolicy:
"""OCR the pages that need it — never the whole document out of caution.
``auto`` already means "the rebuild step decides per page from
pages_needing_ocr and the encoding-broken list", which is exactly the
per-page behaviour wanted. It is escalated to ``force`` only when the
document is so thoroughly broken that per-page detection would still leave
corrupt text behind, and dropped to ``never`` when there is nothing to gain.
"""
if signals.doc_type == PdfDocType.text_based and not signals.pages_needing_ocr:
reasons.append("clean digital text layer: no OCR")
return OcrPolicy.never
if signals.ocr_fraction >= 0.95 and signals.page_count > 1:
reasons.append(f"every page needs OCR ({signals.doc_type.value})")
return OcrPolicy.force
if signals.pages_needing_ocr:
reasons.append(f"OCR {len(signals.pages_needing_ocr)}/{signals.page_count} pages")
return OcrPolicy.auto
def _decide_layout(
signals: DocumentSignals, target: str, reasons: list[str]
) -> tuple[LayoutMode, RecognitionMode]:
"""Flowing keeps the output editable; exact keeps a poster looking like one."""
if target != "docx":
# Only DOCX has a positioned emit. Everything else reflows by nature.
return LayoutMode.flowing, RecognitionMode.enhanced_flow
brochure = signals.brochure_fraction >= BROCHURE_PAGE_SHARE
image_doc = signals.doc_type == PdfDocType.image_based
if brochure or (image_doc and signals.text_words < BROCHURE_MAX_WORDS * signals.page_count):
why = "image-led layout" if image_doc else "brochure-like pages"
reasons.append(f"{why}: positioned emit so the design survives")
return LayoutMode.exact, RecognitionMode.textbox
return LayoutMode.flowing, RecognitionMode.enhanced_flow
def _decide_tables(signals: DocumentSignals, target: str, reasons: list[str]) -> bool:
"""Table detection off only where it can do nothing but harm."""
if target in ("xlsx", "csv", "json"):
# A spreadsheet target exists to carry grids.
return True
if signals.doc_type == PdfDocType.image_based and not signals.pages_needing_ocr:
reasons.append("image-only document: table detection off")
return False
return True
def decide(
signals: DocumentSignals, *, target: str, base: ConvertOptions | None = None
) -> PolicyDecision:
"""Choose reconstruction options for this document and target."""
target = (target or "").lower()
reasons: list[str] = []
if target in RASTER_TARGETS:
# Rasters reproduce the page; nothing below applies to them.
options = base or ConvertOptions()
return PolicyDecision(
options=options,
reasons=["raster target: page reproduced exactly, no reconstruction"],
signals=signals,
)
ocr_policy = _decide_ocr(signals, reasons)
layout_mode, recognition_mode = _decide_layout(signals, target, reasons)
detect_tables = _decide_tables(signals, target, reasons)
options = ConvertOptions(
layout_mode=layout_mode,
ocr_policy=ocr_policy,
ocr_engine=(base.ocr_engine if base else ConvertOptions().ocr_engine),
header_footer_mode=HeaderFooterMode.detect,
recognition_mode=recognition_mode,
detect_tables=detect_tables,
)
if not reasons:
reasons.append("digital text: flowing reconstruction")
return PolicyDecision(options=options, reasons=reasons, signals=signals)
def signals_from_pdf(data: bytes, route: PdfRouteResult | None = None) -> DocumentSignals:
"""Gather the document signals the policy needs, reusing cached work.
Deliberately cheap: the router result and the page text are already in the
per-conversion cache by the time this runs, and image placement comes from
the same content-stream walk the figure extractor uses.
"""
from app.services.convert import doc_cache
from app.services.convert.layout.images import image_placements
from app.services.convert.layout.page_classify import estimate_image_coverage
from app.services.convert.layout.pdf_router import route_pdf
if route is None:
try:
route = route_pdf(data)
except Exception: # a routing failure must not fail the conversion
route = None
signals = DocumentSignals()
if route is not None:
signals.doc_type = route.doc_type
signals.confidence = route.confidence
signals.pages_needing_ocr = list(route.pages_needing_ocr)
try:
reader = doc_cache.get_reader(data)
except Exception:
return signals
pages = reader.pages
signals.page_count = len(pages)
coverages: list[float] = []
words = 0
for index, page in enumerate(pages):
try:
width = float(page.mediabox.width)
height = float(page.mediabox.height)
except Exception:
width = height = 0.0
try:
blocks = [
{"x": p.x, "y": p.y, "w": p.w, "h": p.h} for p in image_placements(page)
]
except Exception:
blocks = []
coverage = estimate_image_coverage(blocks, width, height) if blocks else 0.0
coverages.append(coverage)
if coverage >= 0.98:
signals.full_page_raster_pages.append(index)
try:
text = doc_cache.page_text(data, index)
except Exception:
text = ""
page_words = len(text.split())
words += page_words
# Brochure: a lot of artwork and very little running prose.
if coverage >= BROCHURE_IMAGE_COVERAGE and page_words < BROCHURE_MAX_WORDS:
signals.brochure_pages.append(index)
signals.image_coverage = sum(coverages) / max(len(coverages), 1)
signals.text_words = words
return signals
def policy_for(data: bytes, *, target: str, base: ConvertOptions | None = None) -> PolicyDecision:
"""Full auto decision for a PDF source."""
return decide(signals_from_pdf(data), target=target, base=base)
# The decision travels with the conversion so the formatter layer can record it
# on the document without every plugin signature growing a parameter. A context
# variable, not a module global: concurrent jobs must not see each other's.
_active: ContextVar[PolicyDecision | None] = ContextVar("convert_auto_policy", default=None)
def set_active_decision(decision: PolicyDecision | None) -> None:
_active.set(decision)
def get_active_decision() -> PolicyDecision | None:
return _active.get()
@contextmanager
def decision_scope(decision: PolicyDecision | None = None) -> Iterator[None]:
"""Bind the routing decision to one conversion and nothing else.
A ContextVar ``set`` persists in whatever context performed it. Conversions
run on pooled worker threads whose context outlives the job, so a decision
left behind by an automatic conversion was still readable by the next one —
including an explicit conversion that did no routing at all, which would
then report someone else's ``convert_policy`` as its own.
Entering resets the decision to *nothing*, so a job can only ever see a
decision its own run made; leaving restores what was there before.
"""
token = _active.set(decision)
try:
yield
finally:
_active.reset(token)
gateway/app/services/convert/backends/__init__.py
+5
View File
@@ -0,0 +1,5 @@
"""Conversion backends (reportlab Office→PDF exporter)."""
from __future__ import annotations
__all__ = ["pdf_exporter"]
gateway/app/services/convert/backends/pdf_exporter.py
+95
View File
@@ -0,0 +1,95 @@
"""Office → PDF via reportlab only (concurrent-safe, no LibreOffice).
LibreOffice was removed: heavy, poor multi-user concurrency, and out of
primary stack policy. Improve fidelity in pdf_from_docx instead.
"""
from __future__ import annotations
import os
from typing import Callable
import httpx
from app.services.convert.validation import convert_timeout_seconds, run_with_timeout
from app.services.convert.writers import pdf_from_docx as reportlab_pdf
def _gotenberg_url() -> str:
return os.environ.get("GOTENBERG_URL", "http://127.0.0.1:3000").rstrip("/")
def _try_gotenberg_convert(data: bytes, source_ext: str, timeout: float = 30.0) -> bytes | None:
"""Attempt high-fidelity Office->PDF conversion via Gotenberg LibreOffice endpoint."""
url = f"{_gotenberg_url()}/forms/libreoffice/convert"
ext = source_ext.lstrip(".").lower()
filename = f"document.{ext}"
mime_types = {
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"doc": "application/msword",
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"xls": "application/vnd.ms-excel",
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
"ppt": "application/vnd.ms-powerpoint",
}
content_type = mime_types.get(ext, "application/octet-stream")
try:
with httpx.Client(timeout=timeout) as client:
files = {"files": (filename, data, content_type)}
response = client.post(url, files=files)
if response.status_code == 200 and len(response.content) >= 10 and response.content.startswith(b"%PDF"):
return response.content
except Exception:
# Fail-open contract: Gotenberg absence or network glitch must never crash conversion
pass
return None
def export_office_to_pdf(
data: bytes,
*,
source_ext: str,
reportlab_fn: Callable[[bytes], bytes] | None = None,
timeout: float | None = None,
) -> tuple[bytes, list[str]]:
"""
Convert DOCX/XLSX/PPTX bytes to PDF.
First attempts high-fidelity Gotenberg if available; seamlessly falls back
to in-process ReportLab.
Enforces CONVERT_TIMEOUT_SECONDS (or explicit timeout).
"""
limit = timeout if timeout is not None else convert_timeout_seconds(120.0)
ext = source_ext.lstrip(".").lower()
# 1. Try Gotenberg for maximum Office layout fidelity
gotenberg_pdf = _try_gotenberg_convert(data, ext, timeout=min(limit, 30.0))
if gotenberg_pdf is not None:
return gotenberg_pdf, [
"PDF converted via Gotenberg (high-fidelity LibreOffice microservice).",
]
# 2. Seamless fallback to in-process ReportLab
warnings = [
"PDF via reportlab (in-process; concurrent-safe). Complex Word/Excel layout may differ.",
]
def _run() -> tuple[bytes, list[str]]:
if reportlab_fn is not None:
return reportlab_fn(data), []
if ext in ("docx", "doc"):
return reportlab_pdf.docx_to_pdf(data), []
if ext in ("xlsx", "xls"):
out, extra = reportlab_pdf.xlsx_to_pdf(data)
return out, list(extra)
if ext in ("pptx", "ppt"):
from app.services.convert.writers.pdf_from_pptx import pptx_to_pdf
out, extra = pptx_to_pdf(data)
return out, list(extra)
raise ValueError(f"No PDF export path for .{ext}")
try:
out, extra = run_with_timeout(_run, limit, label="Office→PDF reportlab")
except TimeoutError:
raise
return out, warnings + list(extra)
gateway/app/services/convert/cancellation.py
+153
View File
@@ -0,0 +1,153 @@
"""Per-conversion cancellation and deadlines.
Two mechanisms, both scoped to the conversion that owns them:
``CancelToken``
A caller-supplied "should I stop?" predicate, used to honour a job DELETE.
``Deadline``
A wall-clock budget the pipeline checks at page boundaries, so a runaway
document stops doing work rather than merely having its result discarded.
Both travel in :mod:`contextvars` rather than module globals. A module global
is shared by every thread in the process, so with concurrent conversions one
job would overwrite another's callback — and cancelling one job would abort an
unrelated one. Context variables give each conversion its own value whether it
runs in a thread, a task, or nested inside a worker.
"""
from __future__ import annotations
import time
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
class ConversionCancelled(RuntimeError):
"""Raised when the caller cancelled the conversion."""
class ConversionDeadlineExceeded(TimeoutError):
"""Raised when a conversion exhausted its wall-clock budget."""
CancelToken = Callable[[], bool]
_cancel_check: ContextVar[CancelToken | None] = ContextVar("convert_cancel_check", default=None)
_deadline: ContextVar[Deadline | None] = ContextVar("convert_deadline", default=None)
@dataclass
class Deadline:
"""A wall-clock budget for one conversion."""
seconds: float
started: float
@classmethod
def start(cls, seconds: float) -> Deadline:
return cls(seconds=seconds, started=time.monotonic())
@property
def elapsed(self) -> float:
return time.monotonic() - self.started
@property
def remaining(self) -> float:
return self.seconds - self.elapsed
@property
def expired(self) -> bool:
return self.seconds > 0 and self.remaining <= 0
@contextmanager
def conversion_scope(
*, cancel_check: CancelToken | None = None, timeout: float | None = None
) -> Iterator[None]:
"""Bind a cancel token and/or deadline for the duration of one conversion."""
cancel_reset = _cancel_check.set(cancel_check)
deadline_reset = _deadline.set(Deadline.start(timeout) if timeout and timeout > 0 else None)
try:
yield
finally:
_cancel_check.reset(cancel_reset)
_deadline.reset(deadline_reset)
@contextmanager
def deadline_scope(seconds: float | None) -> Iterator[None]:
"""Bind only a wall-clock budget, leaving any cancel token in place."""
reset = _deadline.set(Deadline.start(seconds) if seconds and seconds > 0 else None)
try:
yield
finally:
_deadline.reset(reset)
def set_cancel_check(fn: CancelToken | None) -> None:
"""Bind a cancel token for the current context.
Prefer :func:`conversion_scope`; this exists for callers and tests that
manage the lifetime themselves.
"""
_cancel_check.set(fn)
def get_cancel_check() -> CancelToken | None:
return _cancel_check.get()
def get_deadline() -> Deadline | None:
return _deadline.get()
def remaining_seconds(default: float) -> float:
"""Budget left for a sub-step, never more than the conversion's own budget."""
deadline = _deadline.get()
if deadline is None or deadline.seconds <= 0:
return default
return max(0.0, min(default, deadline.remaining))
def check_cancelled(stage: str = "conversion") -> None:
"""Raise only if the caller cancelled — never on an expired deadline.
For loops that handle an exhausted budget themselves by stopping early and
returning what they have. Using :func:`check` there defeats the point: a
single page overrunning the budget raises out of the loop and discards
every page already recovered, which is exactly the failure the budget
logic exists to prevent.
"""
fn = _cancel_check.get()
if fn is None:
return
try:
cancelled = bool(fn())
except Exception:
cancelled = False
if cancelled:
raise ConversionCancelled(f"{stage} cancelled.")
def check(stage: str = "conversion") -> None:
"""Raise if the conversion was cancelled or has run out of time.
Call at safe points — page boundaries, between OCR pages — so a runaway
document stops working instead of merely having its result thrown away.
"""
fn = _cancel_check.get()
if fn is not None:
try:
cancelled = bool(fn())
except Exception:
cancelled = False
if cancelled:
raise ConversionCancelled("Job cancelled.")
deadline = _deadline.get()
if deadline is not None and deadline.expired:
raise ConversionDeadlineExceeded(
f"{stage} exceeded its {deadline.seconds:.0f}s budget"
)
gateway/app/services/convert/converters.py
+1086
View File
File diff suppressed because it is too large Load Diff
gateway/app/services/convert/doc_cache.py
+249
View File
@@ -0,0 +1,249 @@
"""Per-conversion cache for expensive PDF reads.
``PdfReader.extract_text`` is the dominant cost of a text-heavy conversion, and
three independent stages need the same text: the document router deciding which
pages need OCR, the layout pipeline building the IDM, and the quality scorer
establishing a source baseline. Each stage opened its own reader, so a 14-page
document ran extraction 42 times.
The cache is bound to a conversion scope, so entries are released when the
conversion ends and two concurrent conversions never share state. Outside a
scope every call falls through to a direct read, which keeps callers total and
makes the cache a pure optimisation.
"""
from __future__ import annotations
import hashlib
import io
from collections import OrderedDict
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, field
from typing import Any
from pypdf import PdfReader
# Hashing the whole payload is cheap next to a single page extraction
# (~1ms/MB against ~130ms/page), and keying on content means a converter that
# handles more than one document inside a scope stays correct.
_DIGEST_BYTES = 8
def _key(data: bytes) -> str:
return hashlib.blake2b(data, digest_size=_DIGEST_BYTES).hexdigest()
@dataclass
class DocumentCache:
"""Readers and extracted page text for the documents seen in one scope."""
readers: dict[str, PdfReader] = field(default_factory=dict)
page_text: dict[tuple[str, int], str] = field(default_factory=dict)
# Glyph boxes and vector path operators recovered from the content stream
# when the C++ engine is absent. Tokenising a content stream and resolving
# its fonts costs about as much as extracting the page's text, and the
# layout pipeline, table detector and scorer all ask for the same answer.
geometry: OrderedDict[tuple[str, int], tuple[list, list]] = field(
default_factory=OrderedDict
)
# Rendered page images keyed by (document, page, dpi). A page is rasterised
# once and reused by layout detection, OCR and image export.
rasters: OrderedDict[tuple[str, int, int], bytes] = field(default_factory=OrderedDict)
# The engine document handle, opened at most once per document.
engines: dict[str, object] = field(default_factory=dict)
hits: int = 0
misses: int = 0
raster_hits: int = 0
raster_misses: int = 0
raster_bytes: int = 0
def reader_for(self, data: bytes) -> PdfReader:
key = _key(data)
reader = self.readers.get(key)
if reader is None:
reader = PdfReader(io.BytesIO(data), strict=False)
self.readers[key] = reader
return reader
def text_for(self, data: bytes, index: int) -> str:
key = (_key(data), index)
cached = self.page_text.get(key)
if cached is not None:
self.hits += 1
return cached
self.misses += 1
reader = self.reader_for(data)
try:
text = reader.pages[index].extract_text() or ""
except Exception:
text = ""
self.page_text[key] = text
return text
def geometry_for(self, data: bytes, index: int, extract) -> tuple[list, list]:
"""``(glyphs, path_ops)`` for one page, computed at most once.
Bounded to the most recent pages. Every consumer of a page's glyphs —
the layout pipeline, the table detectors, the scorer — asks for them
while that page is being built and never again, so holding all 126
pages' worth to the end of the conversion buys nothing and costs the
peak. Evicting is always safe: a later ask recomputes.
"""
key = (_key(data), index)
cached = self.geometry.get(key)
if cached is not None:
self.hits += 1
# Refresh recency so a page still being worked on is not evicted.
self.geometry.move_to_end(key)
return cached
self.misses += 1
try:
result = extract()
except Exception:
result = ([], [])
self.geometry[key] = result
while len(self.geometry) > MAX_CACHED_GEOMETRY_PAGES:
self.geometry.popitem(last=False)
return result
def raster_for(self, data: bytes, index: int, dpi: int, render) -> bytes:
"""Return a rendered page, computing it at most once per (page, dpi).
Rasters are the largest thing here by an order of magnitude, and a
page's raster is wanted by at most three consecutive stages — layout
detection, OCR, image export. Once the cache is full the oldest page
is dropped rather than the new one refused, so the pages being worked
on now are the ones held.
"""
key = (_key(data), index, int(dpi))
cached = self.rasters.get(key)
if cached is not None:
self.raster_hits += 1
self.rasters.move_to_end(key)
return cached
self.raster_misses += 1
png = render()
if len(png) <= MAX_RASTER_CACHE_BYTES:
self.rasters[key] = png
self.raster_bytes += len(png)
self.rasters.move_to_end(key)
while self.raster_bytes > MAX_RASTER_CACHE_BYTES and len(self.rasters) > 1:
_evicted, payload = self.rasters.popitem(last=False)
self.raster_bytes -= len(payload)
return png
def engine_for(self, data: bytes, open_document) -> object | None:
"""Open the engine document once per conversion."""
key = _key(data)
if key in self.engines:
return self.engines[key]
doc = open_document()
self.engines[key] = doc
return doc
# Page rasters at 200 dpi run 1-3 MB each. A page's raster is wanted by at
# most three consecutive stages, so a cache large enough to hold a hundred of
# them is a hundred pages of peak bought for nothing. 64 MB covers the working
# set with room to spare and takes a third of a gigabyte off a long scan.
MAX_RASTER_CACHE_BYTES = 64 * 1024 * 1024
# Glyph lists are small next to a raster but not free: a dense page carries a
# few thousand dicts, and a 126-page booklet held every one of them to the end
# of the conversion. Twelve pages is far more than any stage looks back.
MAX_CACHED_GEOMETRY_PAGES = 12
_cache: ContextVar[DocumentCache | None] = ContextVar("convert_doc_cache", default=None)
@contextmanager
def document_cache_scope() -> Iterator[DocumentCache]:
"""Cache PDF reads for the duration of one conversion."""
cache = DocumentCache()
token = _cache.set(cache)
try:
yield cache
finally:
_cache.reset(token)
def current() -> DocumentCache | None:
return _cache.get()
def get_reader(data: bytes) -> PdfReader:
"""A ``PdfReader`` for *data*, reused within the current conversion."""
cache = _cache.get()
if cache is None:
return PdfReader(io.BytesIO(data), strict=False)
return cache.reader_for(data)
def page_text(data: bytes, index: int) -> str:
"""Extracted text for one page, computed at most once per conversion."""
cache = _cache.get()
if cache is None:
try:
reader = PdfReader(io.BytesIO(data), strict=False)
return reader.pages[index].extract_text() or ""
except Exception:
return ""
return cache.text_for(data, index)
def page_geometry(data: bytes, index: int, extract) -> tuple[list, list]:
"""Glyph boxes and path operators for one page, reused within a conversion."""
cache = _cache.get()
if cache is None:
try:
return extract()
except Exception:
return ([], [])
return cache.geometry_for(data, index, extract)
def page_raster(data: bytes, index: int, dpi: int, render) -> bytes:
"""Rendered page image, reused across layout detection, OCR and export."""
cache = _cache.get()
if cache is None:
return render()
return cache.raster_for(data, index, dpi, render)
def engine_document(data: bytes, open_document):
"""Engine handle for *data*, opened at most once per conversion."""
cache = _cache.get()
if cache is None:
return open_document()
return cache.engine_for(data, open_document)
def page_count(data: bytes) -> int:
try:
return len(get_reader(data).pages)
except Exception:
return 0
def all_text(data: bytes) -> str:
"""Whole-document plain text, reusing any per-page text already extracted."""
count = page_count(data)
return "\n".join(page_text(data, i) for i in range(count))
def stats() -> dict[str, Any]:
cache = _cache.get()
if cache is None:
return {}
return {
"documents": len(cache.readers),
"pages_cached": len(cache.page_text),
"geometry_cached": len(cache.geometry),
"hits": cache.hits,
"misses": cache.misses,
"rasters_cached": len(cache.rasters),
"raster_hits": cache.raster_hits,
"raster_misses": cache.raster_misses,
"raster_mb": round(cache.raster_bytes / 1024 / 1024, 1),
}
gateway/app/services/convert/errors.py
+27
View File
@@ -0,0 +1,27 @@
"""Typed convert / validation error codes (Firecrawl anydoc-inspired taxonomy)."""
from __future__ import annotations
from enum import Enum
class ConvertErrorCode(str, Enum):
unsupported = "unsupported"
encrypted = "encrypted"
needs_ocr = "needs_ocr"
malformed = "malformed"
resource_limit = "resource_limit"
missing_part = "missing_part"
io = "io"
# Default HTTP status per code
HTTP_STATUS: dict[ConvertErrorCode, int] = {
ConvertErrorCode.unsupported: 415,
ConvertErrorCode.encrypted: 400,
ConvertErrorCode.needs_ocr: 422,
ConvertErrorCode.malformed: 400,
ConvertErrorCode.resource_limit: 413,
ConvertErrorCode.missing_part: 400,
ConvertErrorCode.io: 400,
}
gateway/app/services/convert/formatters/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Formatters package."""
gateway/app/services/convert/formatters/docx_exact.py
+178
View File
@@ -0,0 +1,178 @@
"""Positioned DOCX emit — Wave-2 ``layout_mode=exact`` / ``recognition_mode=textbox``.
Flowing reconstruction is right for a report: the text reflows, the reader can
edit it, and the result is a document rather than a picture of one. It is wrong
for a poster, a certificate or a brochure page, where the arrangement *is* the
content. Reflowing one of those concatenates every caption into a single
paragraph and destroys the thing the reader wanted.
This module emits each block into a floating text frame anchored at the
block's own rectangle, in the page's own coordinate system. Word treats the
result as a normal document — every frame stays selectable and editable — but
the arrangement survives.
Chosen by :mod:`auto_policy`, never by a customer dropdown. Word output remains
a reconstruction and stays labelled lossy: frame positions come from measured
geometry, not from the PDF's own graphics state.
Blocks without usable geometry fall back to a flowing paragraph individually,
with a warning naming the page, rather than dragging the whole page back to
flowing or inventing a position for them.
"""
from __future__ import annotations
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.oxml import OxmlElement
from docx.oxml.ns import qn
from app.services.convert.idm.model import Block, BlockType, Page
from app.services.convert.text.arabic_logical import is_rtl_dominant
# PDF user space is 72 points per inch; OOXML frames are in twips (1/20 pt).
TWIPS_PER_POINT = 20
# A frame narrower or shorter than this is measurement noise, not a text box.
MIN_FRAME_POINTS = 6.0
# Frames are given a little slack so a glyph that overhangs its measured box
# does not wrap to a second line and push the layout apart.
FRAME_PADDING_POINTS = 4.0
def _twips(points: float) -> int:
return round(points * TWIPS_PER_POINT)
def has_usable_geometry(block: Block) -> bool:
"""Whether this block can be positioned at all."""
bbox = getattr(block, "bbox", None)
if bbox is None:
return False
if getattr(block, "synthetic_geometry", False):
# Fabricated from plain text: the numbers look real but describe
# nothing on the page. Positioning by them would be inventing a layout.
return False
return bbox.w > MIN_FRAME_POINTS and bbox.h > MIN_FRAME_POINTS
def _frame_properties(block: Block, page: Page) -> OxmlElement:
"""``w:framePr`` placing a paragraph at the block's rectangle.
Word anchors frames to the page with y measured downwards from the top,
while PDF measures upwards from the bottom, so the vertical coordinate is
flipped here. Getting this wrong mirrors the page — the first thing to
check if a positioned document comes out upside down.
"""
bbox = block.bbox
page_height = float(page.height or 792.0)
x = max(0.0, float(bbox.x))
top = max(0.0, page_height - (float(bbox.y) + float(bbox.h)))
width = max(MIN_FRAME_POINTS, float(bbox.w) + FRAME_PADDING_POINTS)
height = max(MIN_FRAME_POINTS, float(bbox.h) + FRAME_PADDING_POINTS)
frame = OxmlElement("w:framePr")
frame.set(qn("w:w"), str(_twips(width)))
frame.set(qn("w:h"), str(_twips(height)))
frame.set(qn("w:hRule"), "atLeast")
frame.set(qn("w:x"), str(_twips(x)))
frame.set(qn("w:y"), str(_twips(top)))
frame.set(qn("w:hAnchor"), "page")
frame.set(qn("w:vAnchor"), "page")
# Frames must not push each other around; each one owns its rectangle.
frame.set(qn("w:wrap"), "none")
return frame
def position_paragraph(paragraph, block: Block, page: Page) -> None:
"""Turn an ordinary paragraph into a frame at the block's rectangle."""
pPr = paragraph._p.get_or_add_pPr()
pPr.insert(0, _frame_properties(block, page))
# Inside a frame the paragraph must not inherit body spacing, or the text
# sits below the rectangle it was placed in.
spacing = OxmlElement("w:spacing")
spacing.set(qn("w:before"), "0")
spacing.set(qn("w:after"), "0")
pPr.append(spacing)
if is_rtl_dominant(block.plain_text()):
bidi = OxmlElement("w:bidi")
bidi.set(qn("w:val"), "1")
pPr.append(bidi)
paragraph.alignment = WD_ALIGN_PARAGRAPH.RIGHT
def position_figure(paragraph, block: Block, page: Page) -> None:
"""Anchor a picture's paragraph at the block's own rectangle.
Before this, positioned pages put captions in frames and left the pictures
flowing down the page underneath them — the arrangement half-applied, which
reads worse than either approach on its own. A brochure's picture belongs
where the brochure put it.
"""
position_paragraph(paragraph, block, page)
def position_table(table, block: Block, page: Page) -> bool:
"""Float a table at the block's rectangle. False if it has no usable geometry.
Word floats tables with ``w:tblpPr`` rather than ``w:framePr``. A table that
cannot be placed is left flowing rather than dropped: a grid in the wrong
position is still the customer's data, an absent grid is not.
"""
if not has_usable_geometry(block):
return False
bbox = block.bbox
page_height = float(page.height or 792.0)
top = max(0.0, page_height - (float(bbox.y) + float(bbox.h)))
tblPr = table._tbl.tblPr
pos = OxmlElement("w:tblpPr")
pos.set(qn("w:leftFromText"), "0")
pos.set(qn("w:rightFromText"), "0")
pos.set(qn("w:topFromText"), "0")
pos.set(qn("w:bottomFromText"), "0")
pos.set(qn("w:vertAnchor"), "page")
pos.set(qn("w:horzAnchor"), "page")
pos.set(qn("w:tblpX"), str(_twips(max(0.0, float(bbox.x)))))
pos.set(qn("w:tblpY"), str(_twips(top)))
tblPr.insert(0, pos)
return True
def page_is_positionable(page: Page, *, min_ratio: float = 0.5) -> bool:
"""Whether enough of a page has real geometry for positioning to mean anything.
A page where most blocks would fall back to flowing is better emitted as a
flowing page outright: a handful of floating frames interleaved with body
text reads worse than either approach on its own.
"""
blocks = [
b
for b in page.blocks
if b.type not in (BlockType.header, BlockType.footer) and (b.plain_text().strip() or b.image_png)
]
if not blocks:
return False
usable = sum(1 for b in blocks if has_usable_geometry(b))
return usable / len(blocks) >= min_ratio
def figure_size_points(block: Block, page: Page) -> tuple[float, float]:
"""Width/height for a positioned figure, clamped to the page."""
page_w = float(page.width or 612.0)
page_h = float(page.height or 792.0)
w = float(block.bbox.w or 0) or page_w * 0.4
h = float(block.bbox.h or 0) or page_h * 0.3
return min(w, page_w), min(h, page_h)
__all__ = [
"FRAME_PADDING_POINTS",
"MIN_FRAME_POINTS",
"TWIPS_PER_POINT",
"figure_size_points",
"has_usable_geometry",
"page_is_positionable",
"position_figure",
"position_paragraph",
"position_table",
]
gateway/app/services/convert/formatters/docx_formatter.py
+1036
View File
File diff suppressed because it is too large Load Diff
gateway/app/services/convert/formatters/pptx_formatter.py
+161
View File
@@ -0,0 +1,161 @@
"""Format IDM document into native Microsoft PowerPoint (.pptx)."""
from __future__ import annotations
import io
from pptx import Presentation
from pptx.util import Inches, Pt
from pptx.dml.color import RGBColor
from pptx.enum.text import PP_ALIGN
from pptx.enum.shapes import MSO_SHAPE
from app.services.convert.idm.model import BlockType, Document, Page
def format_pptx(doc: Document) -> bytes:
"""Serialize canonical IDM document into a clean Microsoft PowerPoint presentation."""
prs = Presentation()
# Determine presentation slide dimensions from first page (default 16:9 widescreen: 13.333 x 7.5 in)
if doc.pages:
p0 = doc.pages[0]
w_pt = p0.width or 612.0
h_pt = p0.height or 792.0
# If landscape or wide, set 16:9 widescreen
if w_pt >= h_pt:
prs.slide_width = Inches(13.333)
prs.slide_height = Inches(7.5)
else:
# 4:3 standard
prs.slide_width = Inches(10.0)
prs.slide_height = Inches(7.5)
else:
prs.slide_width = Inches(13.333)
prs.slide_height = Inches(7.5)
blank_layout = prs.slide_layouts[6] # Blank slide
slide_w = prs.slide_width
slide_h = prs.slide_height
for page in doc.pages:
slide = prs.slides.add_slide(blank_layout)
# Scale factor from PDF points to PPTX slide dimensions
pdf_w = page.width or 612.0
pdf_h = page.height or 792.0
sx = float(slide_w) / max(pdf_w, 1.0)
sy = float(slide_h) / max(pdf_h, 1.0)
# Process blocks
for block in page.blocks:
if block.type == BlockType.table and block.cells:
_render_table(slide, block, sx, sy, float(slide_w), float(slide_h), pdf_h)
elif block.type == BlockType.heading:
_render_heading(slide, block, sx, sy, float(slide_w), float(slide_h), pdf_h)
elif block.type in (BlockType.paragraph, BlockType.list_item):
_render_text(slide, block, sx, sy, float(slide_w), float(slide_h), pdf_h)
buf = io.BytesIO()
prs.save(buf)
return buf.getvalue()
def _render_heading(
slide, block, sx: float, sy: float, max_w: float, max_h: float, pdf_h: float
) -> None:
text = block.plain_text().strip()
if not text:
return
# PDF origin is bottom-left, PPTX origin is top-left
top = int((pdf_h - (block.bbox.y + block.bbox.h)) * sy) if block.bbox.h else Inches(0.8)
left = int(block.bbox.x * sx) if block.bbox.w else Inches(0.8)
width = int(block.bbox.w * sx) if block.bbox.w else int(max_w - Inches(1.6))
height = max(int(Inches(0.6)), int(block.bbox.h * sy))
# Clamp coordinates inside slide
top = max(0, min(int(max_h - Inches(0.8)), top))
left = max(0, min(int(max_w - Inches(1.5)), left))
width = max(int(Inches(2.0)), min(int(max_w - left), width))
txBox = slide.shapes.add_textbox(left, top, width, height)
tf = txBox.text_frame
tf.word_wrap = True
p = tf.paragraphs[0]
p.text = text
p.font.size = Pt(22)
p.font.bold = True
p.font.color.rgb = RGBColor(0x1E, 0x3A, 0x8A) # Navy corporate
def _render_text(
slide, block, sx: float, sy: float, max_w: float, max_h: float, pdf_h: float
) -> None:
text = block.plain_text().strip()
if not text:
return
top = int((pdf_h - (block.bbox.y + block.bbox.h)) * sy) if block.bbox.h else Inches(1.6)
left = int(block.bbox.x * sx) if block.bbox.w else Inches(0.8)
width = int(block.bbox.w * sx) if block.bbox.w else int(max_w - Inches(1.6))
height = max(int(Inches(0.4)), int(block.bbox.h * sy))
top = max(0, min(int(max_h - Inches(0.6)), top))
left = max(0, min(int(max_w - Inches(1.5)), left))
width = max(int(Inches(2.0)), min(int(max_w - left), width))
txBox = slide.shapes.add_textbox(left, top, width, height)
tf = txBox.text_frame
tf.word_wrap = True
p = tf.paragraphs[0]
p.text = text
p.font.size = Pt(13)
p.font.color.rgb = RGBColor(0x1F, 0x29, 0x37) # Dark Charcoal
def _render_table(
slide, block, sx: float, sy: float, max_w: float, max_h: float, pdf_h: float
) -> None:
grid = block.cells
if not grid or not grid[0]:
return
rows = len(grid)
cols = max(len(r) for r in grid)
if rows < 1 or cols < 1:
return
top = int((pdf_h - (block.bbox.y + block.bbox.h)) * sy) if block.bbox.h else Inches(2.0)
left = int(block.bbox.x * sx) if block.bbox.w else Inches(0.8)
width = int(block.bbox.w * sx) if block.bbox.w else int(max_w - Inches(1.6))
height = max(int(Inches(1.0)), int(block.bbox.h * sy))
top = max(0, min(int(max_h - Inches(1.5)), top))
left = max(0, min(int(max_w - Inches(2.0)), left))
width = max(int(Inches(3.0)), min(int(max_w - left), width))
table_shape = slide.shapes.add_table(rows, cols, left, top, width, height)
tbl = table_shape.table
for r_idx, row in enumerate(grid):
is_header = r_idx == 0
for c_idx in range(cols):
cell_val = row[c_idx] if c_idx < len(row) else ""
cell = tbl.cell(r_idx, c_idx)
cell.text = str(cell_val).strip()
p = cell.text_frame.paragraphs[0]
p.font.size = Pt(10)
if is_header:
p.font.bold = True
p.font.color.rgb = RGBColor(0xFF, 0xFF, 0xFF)
cell.fill.solid()
cell.fill.fore_color.rgb = RGBColor(0x1E, 0x3A, 0x8A) # Navy
else:
p.font.color.rgb = RGBColor(0x11, 0x18, 0x27)
cell.fill.solid()
if r_idx % 2 == 1:
cell.fill.fore_color.rgb = RGBColor(0xF9, 0xFA, 0xFB)
else:
cell.fill.fore_color.rgb = RGBColor(0xFF, 0xFF, 0xFF)
gateway/app/services/convert/formatters/text_formatters.py
+271
View File
@@ -0,0 +1,271 @@
"""HTML / MD / TXT / JSON formatters from IDM."""
from __future__ import annotations
import json
import os
from app.services.convert.formatters.text_postprocess import (
postprocess_inline,
postprocess_plain_text,
)
from app.services.convert.idm.model import Block, BlockType, Document
from app.services.convert.idm.serialize import document_to_dict
from app.services.convert.text.arabic_logical import contains_arabic, to_logical
def _md_cell(text: str) -> str:
"""Escape a value so it cannot break out of a Markdown table cell."""
return (
(text or "")
.replace("\\", "\\\\")
.replace("|", "\\|")
.replace("\r\n", " ")
.replace("\n", "<br>")
.replace("\r", " ")
.strip()
)
def _md_table(cells: list[list[str]], visual: bool) -> list[str]:
"""Render a grid as a valid Markdown table.
Every row is padded to the widest row so the column count matches the
delimiter row; ragged input otherwise produces a table no parser accepts.
"""
grid = [row for row in cells if row is not None]
if not grid:
return []
width = max(len(row) for row in grid)
if width == 0:
return []
out: list[str] = []
for idx, row in enumerate(grid):
values = [_md_cell(postprocess_inline(to_logical(c, visual=visual))) for c in row]
values += [""] * (width - len(values))
out.append("| " + " | ".join(values) + " |")
if idx == 0:
out.append("| " + " | ".join(["---"] * width) + " |")
out.append("")
return out
def md_table_lines(cells: list[list[str]]) -> list[str]:
"""Public entry to the Markdown table renderer, for non-IDM callers."""
return _md_table(cells, False)
def _escape(s: str) -> str:
return (
s.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replace('"', "&quot;")
)
def _block_visual(block) -> bool:
"""True when a block's text is still in visual order and needs repairing.
Ordering is now resolved at extraction time — ``layout.glyphs`` and the OCR
adapter sort each line by its own reading direction — so a block's source
no longer implies visual order. Only an explicit span flag does, which is
set when a producer genuinely could not determine direction.
"""
return any(getattr(s, "visual_order", False) for s in (getattr(block, "spans", None) or []))
def _norm_text(block: Block) -> str:
return postprocess_inline(to_logical(block.plain_text(), visual=_block_visual(block)))
def format_txt(doc: Document) -> bytes:
parts: list[str] = []
for page in doc.pages:
for block in sorted(page.blocks, key=lambda b: b.reading_order):
t = _norm_text(block).strip()
if t:
parts.append(t)
return postprocess_plain_text("\n".join(parts)).encode("utf-8")
def format_md(doc: Document, *, page_markers: bool | None = None) -> bytes:
"""IDM → Markdown.
``page_markers`` inserts a ``## Page N`` heading per page. It defaults to
off because page headings fragment semantic chunking for search and LLM
ingest; set ``CONVERT_MD_PAGE_MARKERS=1`` or pass True to restore them.
"""
if page_markers is None:
page_markers = os.environ.get("CONVERT_MD_PAGE_MARKERS", "0").strip().lower() in (
"1",
"true",
"yes",
"on",
)
parts: list[str] = []
for page in doc.pages:
if page_markers:
parts.append(f"## Page {page.index + 1}\n")
in_list = False
for block in sorted(page.blocks, key=lambda b: b.reading_order):
if block.type == BlockType.heading:
if in_list:
parts.append("")
in_list = False
level = min(max(block.level, 1), 3)
parts.append("#" * level + f" {_norm_text(block)}\n")
elif block.type == BlockType.table and block.cells:
if in_list:
parts.append("")
in_list = False
parts.extend(_md_table(block.cells, _block_visual(block)))
elif block.type == BlockType.list_item:
in_list = True
marker = "1." if block.list_style == "number" else "-"
parts.append(f"{marker} {_norm_text(block)}")
else:
if in_list:
parts.append("")
in_list = False
t = _norm_text(block).strip()
if t:
parts.append(t)
parts.append("")
if in_list:
parts.append("")
# Document-level passes for cross-block hyphenation / page# / leaders
return postprocess_plain_text("\n".join(parts)).encode("utf-8")
def format_html(doc: Document) -> bytes:
chunks = [
'<!DOCTYPE html><html><head><meta charset="utf-8">'
"<title>Converted PDF</title></head><body>"
]
for page in doc.pages:
chunks.append(f'<section data-page="{page.index + 1}">')
chunks.append(f"<h2>Page {page.index + 1}</h2>")
open_list: str | None = None # "ul" | "ol"
def _close_list() -> None:
nonlocal open_list
if open_list:
chunks.append(f"</{open_list}>")
open_list = None
for block in sorted(page.blocks, key=lambda b: b.reading_order):
if block.type == BlockType.heading:
_close_list()
level = min(max(block.level, 1), 3)
txt = _escape(_norm_text(block))
rtl = ' dir="rtl" lang="ar"' if contains_arabic(block.plain_text()) else ""
chunks.append(f"<h{level}{rtl}>{txt}</h{level}>")
elif block.type == BlockType.table and block.cells:
_close_list()
visual = _block_visual(block)
chunks.append('<table border="1" cellpadding="4">')
for ri, row in enumerate(block.cells):
tag = "th" if ri == 0 else "td"
cells_html = []
for c in row:
cell_txt = _escape(to_logical(c, visual=visual))
rtl = ' dir="rtl"' if contains_arabic(c) else ""
cells_html.append(f"<{tag}{rtl}>{cell_txt}</{tag}>")
chunks.append("<tr>" + "".join(cells_html) + "</tr>")
chunks.append("</table>")
elif block.type == BlockType.list_item:
want = "ol" if block.list_style == "number" else "ul"
if open_list != want:
_close_list()
chunks.append(f"<{want}>")
open_list = want
txt = _escape(_norm_text(block))
rtl = ' dir="rtl"' if contains_arabic(block.plain_text()) else ""
chunks.append(f"<li{rtl}>{txt}</li>")
else:
_close_list()
t = _norm_text(block).strip()
if t:
rtl = ' dir="rtl" lang="ar"' if contains_arabic(block.plain_text()) else ""
chunks.append(f"<p{rtl}>{_escape(t)}</p>")
_close_list()
chunks.append("</section>")
chunks.append("</body></html>")
return "\n".join(chunks).encode("utf-8")
def format_json(doc: Document) -> bytes:
payload = document_to_dict(doc)
# Compatibility fields for consumers expecting page.text
for page_dict, page in zip(payload["pages"], doc.pages, strict=True):
page_dict["text"] = page.all_text()
page_dict["paragraphs"] = [
_norm_text(b)
for b in page.blocks
if b.type in (BlockType.paragraph, BlockType.heading, BlockType.list_item)
]
page_dict["tables"] = [b.cells for b in page.blocks if b.type == BlockType.table]
return json.dumps(payload, indent=2, ensure_ascii=False).encode("utf-8")
def format_csv(doc: Document, *, dialect: str = "excel") -> bytes:
"""IDM Document -> RFC 4180 compliant CSV bytes with UTF-8 BOM.
Extracts all detected tables across document pages.
- Normalizes ragged rows to the maximum column width per table.
- Normalizes Arabic/RTL text to logical order.
- Preserves embedded commas, quotes, and newlines per RFC 4180.
- Emits UTF-8 BOM (\\xef\\xbb\\xbf) for seamless Excel display on Windows.
- If multiple tables exist, separates them with clean section headers.
- If zero tables exist, safely exports non-empty paragraphs as single-column records.
"""
import codecs
import csv
import io
stream = io.StringIO()
writer = csv.writer(stream, dialect=dialect, quoting=csv.QUOTE_MINIMAL, lineterminator="\r\n")
# Collect all tables across pages
tables_found: list[tuple[int, list[list[str]], bool]] = []
for page in doc.pages:
for block in sorted(page.blocks, key=lambda b: b.reading_order):
if block.type == BlockType.table and block.cells:
tables_found.append((page.index + 1, block.cells, _block_visual(block)))
if tables_found:
for idx, (page_num, raw_cells, visual) in enumerate(tables_found, 1):
if idx > 1:
writer.writerow([])
writer.writerow([f"# --- Table {idx} (Page {page_num}) ---"])
elif len(tables_found) > 1:
writer.writerow([f"# --- Table 1 (Page {page_num}) ---"])
grid = [row for row in raw_cells if row is not None]
if not grid:
continue
width = max(len(row) for row in grid)
if width == 0:
continue
for row in grid:
cleaned_row = []
for c in row:
val = postprocess_inline(to_logical(str(c or ""), visual=visual)).strip()
val = val.replace("\r\n", "\n").replace("\r", "\n").replace("<br>", "\n")
cleaned_row.append(val)
if len(cleaned_row) < width:
cleaned_row.extend([""] * (width - len(cleaned_row)))
writer.writerow(cleaned_row)
else:
# Fallback for documents with no detected tables: export paragraphs
for page in doc.pages:
for block in sorted(page.blocks, key=lambda b: b.reading_order):
t = _norm_text(block).strip()
if t:
writer.writerow([t])
csv_text = stream.getvalue()
return codecs.BOM_UTF8 + csv_text.encode("utf-8")
gateway/app/services/convert/formatters/text_postprocess.py
+108
View File
@@ -0,0 +1,108 @@
"""MD/HTML/TXT postprocess — hyphenation, TOC leaders, page numbers, drop-caps.
Every pass here deletes or rewrites extracted content, so each one is scoped as
narrowly as the signal allows. A pass that removes a running page number must
not also remove a standalone figure in a financial table, and a pass that
merges a drop cap must not merge the "A." of a lettered list.
"""
from __future__ import annotations
import re
_HYPHEN_BREAK_RE = re.compile(r"(\w)[\-­‐‑\x02]\n+(\w)")
_DOT_LEADER_RE = re.compile(r"(\S)\s*\.{2,}\s*(\S)")
# A page-number line: an optional "page" word, digits, optional dash decoration.
# Roman numerals are included because front matter uses them.
_PAGE_NUM_LINE_RE = re.compile(
r"^\s*(?:page\s+)?\d{1,4}\s*$"
r"|^\s*[-–—]\s*\d{1,4}\s*[-–—]\s*$"
r"|^\s*(?:page\s+)?\d{1,4}\s*(?:of|/)\s*\d{1,4}\s*$"
r"|^\s*[ivxlcdm]{1,7}\s*$",
re.I,
)
# A drop cap is a lone capital followed by lowercase continuation text. Require
# the continuation to look like prose so "A." / "B." list markers are untouched.
_DROP_CAP_RE = re.compile(r"(?m)^([A-ZÀ-ÖØ-Þ])\n+([a-zà-öø-ÿ][a-zà-öø-ÿ ,;]{8,})")
# Do not treat a lone number as a page number when it carries a decimal point,
# a thousands separator, a sign or a currency symbol — those are data.
_DATA_NUMBER_RE = re.compile(r"[.,%$€£¥₹+\-]")
def rejoin_hyphenated_breaks(text: str) -> str:
"""Join end-of-line hyphenation: ``end-\\nword`` → ``endword``."""
if not text:
return text
prev = None
out = text
while prev != out:
prev = out
out = _HYPHEN_BREAK_RE.sub(r"\1\2", out)
return out
def collapse_dot_leaders(text: str) -> str:
"""Collapse TOC-style dot leaders to ``' ... '``."""
if not text:
return text
return _DOT_LEADER_RE.sub(r"\1 ... \2", text)
def _looks_like_page_number(line: str) -> bool:
stripped = line.strip()
if not _PAGE_NUM_LINE_RE.match(line):
return False
# "1,234" or "12.5" or "-40" are values, not page numbers.
return not _DATA_NUMBER_RE.search(stripped)
def filter_page_number_lines(text: str, *, min_repeats: int = 2) -> str:
"""Drop running page-number lines.
A single standalone number is ambiguous — it may be a page number or a
value from a table that lost its row. The line is removed only when the
document shows the *pattern* repeatedly (at least ``min_repeats`` such
lines), which is what a running folio looks like and what an isolated data
point does not.
"""
if not text:
return text
lines = text.splitlines()
if len(lines) < 2:
return text
candidates = [i for i, ln in enumerate(lines) if _looks_like_page_number(ln)]
if len(candidates) < min_repeats:
return text
drop = set(candidates)
return "\n".join(ln for i, ln in enumerate(lines) if i not in drop)
def merge_drop_caps(text: str) -> str:
"""Merge single-letter drop-cap lines into the following paragraph."""
if not text:
return text
return _DROP_CAP_RE.sub(r"\1\2", text)
def postprocess_inline(text: str) -> str:
"""Safe per-block polish (no page-number stripping)."""
if not text:
return text
out = rejoin_hyphenated_breaks(text)
out = collapse_dot_leaders(out)
return out
def postprocess_plain_text(text: str) -> str:
"""Apply all MD/TXT document-level polish passes."""
if not text:
return text
out = rejoin_hyphenated_breaks(text)
out = collapse_dot_leaders(out)
out = filter_page_number_lines(out)
out = merge_drop_caps(out)
out = re.sub(r"\n{3,}", "\n\n", out)
if text.endswith("\n"):
return out.strip() + "\n"
return out.strip()
gateway/app/services/convert/formatters/xlsx_formatter.py
+276
View File
@@ -0,0 +1,276 @@
"""XLSX formatter from IDM."""
from __future__ import annotations
import io
import re
from datetime import date
from openpyxl import Workbook
from openpyxl.styles import Font
from openpyxl.utils import get_column_letter
from app.services.convert.formatters.xml_sanitize import sanitize_ooxml_text
from app.services.convert.idm.model import BlockType, Document
from app.services.convert.text.arabic_logical import to_logical_cell
from app.services.convert.text.numbers import (
excel_number_format,
parse_iso_date,
parse_number,
)
def _block_visual(block) -> bool:
"""True when a block's text is still in visual order and needs repairing.
Ordering is now resolved at extraction time — ``layout.glyphs`` and the OCR
adapter sort each line by its own reading direction — so a block's source
no longer implies visual order. Only an explicit span flag does, which is
set when a producer genuinely could not determine direction.
"""
return any(getattr(s, "visual_order", False) for s in (getattr(block, "spans", None) or []))
# Excel grid limits (xlsx). Exceeding either raises inside openpyxl.
MAX_XLSX_ROWS = 1_048_576
MAX_XLSX_COLS = 16_384
def _safe_sheet_name(name: str, used: set[str]) -> str:
cleaned = re.sub(r"[\[\]\*\/\\\?\:]", "_", sanitize_ooxml_text(name))[:31] or "Sheet"
base = cleaned
n = 1
while cleaned in used:
suffix = f"_{n}"
cleaned = base[: 31 - len(suffix)] + suffix
n += 1
used.add(cleaned)
return cleaned
def coerce_cell(value: str, *, visual: bool = False) -> tuple[object, str | None]:
"""Return ``(python value, excel number format)`` for a cell string.
Numbers are written as numbers so ``=SUM()`` works — storing them as text
is the most common complaint about PDF-to-Excel output. The display format
travels with the value so "4.50", "1,234" and "12%" still look the same in
Excel while being numeric underneath.
Labels that merely contain digits ("R1", "Q3", "00123") stay text: coercing
them would rewrite row identifiers into values.
"""
raw = sanitize_ooxml_text(to_logical_cell(value, visual=visual)).strip()
if not raw:
return "", None
iso = parse_iso_date(raw)
if iso:
return date(*iso), "yyyy-mm-dd"
parsed = parse_number(raw)
if parsed is None:
return raw, None
fmt = excel_number_format(parsed)
if parsed.is_integral and not parsed.percent and not parsed.currency:
return int(parsed.value), fmt
return parsed.value, fmt
def _coerce_cell(value: str, *, visual: bool = False):
"""Backwards-compatible single-value form used by existing callers/tests."""
return coerce_cell(value, visual=visual)[0]
def _write_prose(ws, blocks, start_row: int = 1) -> int:
r_idx = start_row
for block in sorted(blocks, key=lambda b: b.reading_order):
if block.type in (BlockType.header, BlockType.footer, BlockType.table, BlockType.figure):
continue
text = sanitize_ooxml_text(
to_logical_cell(block.plain_text(), visual=_block_visual(block))
).strip()
if not text:
continue
value, number_format = coerce_cell(text)
cell = ws.cell(row=r_idx, column=1, value=value)
if number_format:
cell.number_format = number_format
if block.type == BlockType.heading or any(s.bold for s in block.spans):
cell.font = Font(bold=True)
r_idx += 1
return r_idx
def format_xlsx(doc: Document) -> tuple[bytes, list[str]]:
warnings: list[str] = []
wb = Workbook()
default = wb.active
any_table = False
used_names: set[str] = set()
first_sheet = True
all_blocks = [b for page in doc.pages for b in page.blocks]
high_conf_tables = [
b
for b in all_blocks
if b.type == BlockType.table and b.cells and b.table_confidence >= 0.65
]
# Always keep a Content sheet with non-table prose (invoice KV, etc.)
content_needed = any(
b.type not in (BlockType.table, BlockType.header, BlockType.footer, BlockType.figure)
and b.plain_text().strip()
for b in all_blocks
)
if len(high_conf_tables) > 1:
for ti, table_block in enumerate(high_conf_tables):
any_table = True
name = _safe_sheet_name(f"Table{ti + 1}", used_names)
if first_sheet:
ws = default
ws.title = name
first_sheet = False
else:
ws = wb.create_sheet(name)
_write_table(
ws,
table_block.cells,
table_block.merges,
freeze=table_block.table_confidence >= 0.75,
warnings=warnings,
visual=_block_visual(table_block),
)
if content_needed:
name = _safe_sheet_name("Content", used_names)
ws = wb.create_sheet(name)
_write_prose(ws, all_blocks)
else:
for page in doc.pages:
name = _safe_sheet_name(f"Page{page.index + 1}", used_names)
if first_sheet:
ws = default
ws.title = name
first_sheet = False
else:
ws = wb.create_sheet(name)
tables = [b for b in page.blocks if b.type == BlockType.table and b.cells]
prose_blocks = [b for b in page.blocks if b.type != BlockType.table]
r_next = 1
if prose_blocks:
r_next = _write_prose(ws, prose_blocks, start_row=1)
if tables:
r_next += 1
if tables:
any_table = True
# Write first table starting at r_next
_write_table_at(
ws,
tables[0].cells,
tables[0].merges,
start_row=r_next,
freeze=tables[0].table_confidence >= 0.75,
warnings=warnings,
visual=_block_visual(tables[0]),
)
col_offset = max(len(r) for r in tables[0].cells) + 2
for extra in tables[1:]:
_write_table(
ws,
extra.cells,
extra.merges,
col_offset=col_offset,
warnings=warnings,
visual=_block_visual(extra),
)
col_offset += max(len(r) for r in extra.cells) + 2
if not any_table:
warnings.append("No table structure detected; exported paragraphs as a single column.")
buf = io.BytesIO()
wb.save(buf)
return buf.getvalue(), warnings
def _write_table(
ws,
cells: list[list[str]],
merges: list[tuple[int, int, int, int]] | None = None,
*,
col_offset: int = 0,
freeze: bool = False,
warnings: list[str] | None = None,
visual: bool = False,
) -> None:
_write_table_at(
ws,
cells,
merges,
start_row=1,
col_offset=col_offset,
freeze=freeze,
warnings=warnings,
visual=visual,
)
def _write_table_at(
ws,
cells: list[list[str]],
merges: list[tuple[int, int, int, int]] | None = None,
*,
start_row: int = 1,
col_offset: int = 0,
freeze: bool = False,
warnings: list[str] | None = None,
visual: bool = False,
) -> None:
for r_idx, row in enumerate(cells):
if start_row + r_idx > MAX_XLSX_ROWS:
if warnings is not None:
warnings.append(
f"Table truncated at {MAX_XLSX_ROWS} rows (Excel sheet limit)."
)
break
for c_idx, cell in enumerate(row):
if col_offset + c_idx + 1 > MAX_XLSX_COLS:
if warnings is not None:
warnings.append(
f"Table truncated at {MAX_XLSX_COLS} columns (Excel sheet limit)."
)
break
number_format: str | None = None
if isinstance(cell, str):
val, number_format = coerce_cell(cell, visual=visual)
elif cell is None:
val = ""
else:
val = cell
xl = ws.cell(row=start_row + r_idx, column=col_offset + c_idx + 1, value=val)
if number_format:
xl.number_format = number_format
if r_idx == 0:
xl.font = Font(bold=True)
# Column width heuristic
try:
letter = get_column_letter(col_offset + c_idx + 1)
cur = ws.column_dimensions[letter].width or 8
ws.column_dimensions[letter].width = max(cur, min(28, len(str(val)) + 2))
except Exception:
pass
for r0, c0, r1, c1 in merges or []:
try:
start = f"{get_column_letter(col_offset + c0 + 1)}{start_row + r0}"
end = f"{get_column_letter(col_offset + c1 + 1)}{start_row + r1}"
ws.merge_cells(f"{start}:{end}")
except Exception as exc:
if warnings is not None:
warnings.append(f"XLSX merge failed ({r0},{c0})-({r1},{c1}): {exc}")
if freeze and start_row >= 1:
try:
ws.freeze_panes = f"A{start_row + 1}"
except Exception:
pass
gateway/app/services/convert/formatters/xml_sanitize.py
+33
View File
@@ -0,0 +1,33 @@
"""Strip characters illegal in XML 1.0 / OOXML text nodes.
PDF extractors sometimes emit C0 controls (e.g. \\x02 soft-break markers)
that lxml (python-docx) and openpyxl reject on write. Also strip unpaired
UTF-16 surrogates which can appear in broken PDF ToUnicode maps.
"""
from __future__ import annotations
import re
# XML 1.0 Char production excludes C0 controls except TAB/LF/CR.
# Also drop soft hyphen (U+00AD), BOM, non-chars, and lone surrogates.
_ILLEGAL_OOXML_RE = re.compile(
r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\u00ad\ufeff\ufffe\uffff"
r"\ud800-\udfff]"
)
def sanitize_ooxml_text(value: object | None) -> str:
"""Return text safe for OOXML / SpreadsheetML string cells."""
if value is None:
return ""
if not isinstance(value, str):
value = str(value)
if not value:
return ""
# Drop unpaired surrogates that PDF extractors may leave in strings
try:
value.encode("utf-8")
except UnicodeEncodeError:
value = value.encode("utf-8", errors="surrogatepass").decode("utf-8", errors="ignore")
return _ILLEGAL_OOXML_RE.sub("", value)
gateway/app/services/convert/http_headers.py
+34
View File
@@ -0,0 +1,34 @@
"""HTTP header helpers for convert responses (latin-1 safe)."""
from __future__ import annotations
from urllib.parse import quote
def ascii_fallback_filename(filename: str) -> str:
"""Strip/replace non-ASCII so Starlette can encode headers as latin-1."""
raw = (filename or "download").replace('"', "_").replace("\r", "").replace("\n", "")
ascii_name = raw.encode("ascii", "replace").decode("ascii")
# '?' from replace looks ugly in downloads — use underscore
ascii_name = ascii_name.replace("?", "_").strip() or "download"
return ascii_name
def content_disposition_attachment(filename: str) -> str:
"""
RFC 6266 / 5987 Content-Disposition.
HTTP header values must be latin-1. Arabic/CJK stems crash Starlette unless
we provide an ASCII ``filename=`` fallback plus UTF-8 ``filename*=``.
"""
name = (filename or "download").replace("\r", "").replace("\n", "").strip() or "download"
ascii_name = ascii_fallback_filename(name)
if ascii_name == name and all(ord(c) < 128 for c in name):
return f'attachment; filename="{name}"'
encoded = quote(name, safe="")
return f"attachment; filename=\"{ascii_name}\"; filename*=UTF-8''{encoded}"
def ascii_header_value(value: str) -> str:
"""Force a header value to ascii (warnings, fidelity extras)."""
return (value or "").encode("ascii", "replace").decode("ascii")
gateway/app/services/convert/idm/__init__.py
+24
View File
@@ -0,0 +1,24 @@
"""IDM package."""
from app.services.convert.idm.model import (
BBox,
Block,
BlockType,
Document,
Page,
PageKind,
TextSpan,
)
from app.services.convert.idm.serialize import dumps_idm, maybe_dump_idm
__all__ = [
"BBox",
"Block",
"BlockType",
"Document",
"Page",
"PageKind",
"TextSpan",
"dumps_idm",
"maybe_dump_idm",
]
gateway/app/services/convert/idm/model.py
+119
View File
@@ -0,0 +1,119 @@
"""Intermediate Document Model (IDM) for conversion."""
from __future__ import annotations
from dataclasses import dataclass, field
from enum import Enum
from typing import Any
class PageKind(str, Enum):
digital = "digital"
scan = "scan"
hybrid = "hybrid"
blank = "blank"
class BlockType(str, Enum):
heading = "heading"
paragraph = "paragraph"
list_item = "list"
table = "table"
figure = "figure"
header = "header"
footer = "footer"
# ``Block.source`` is usually ``digital`` or ``ocr``. This value marks a
# whole source page that was deliberately embedded as a raster in a DOCX. It
# is a fidelity fallback, not an OCR result: the writer must keep the picture
# at page size and must not turn its placeholder text into a visible caption.
PAGE_RASTER_SOURCE = "page-raster"
@dataclass
class BBox:
x: float = 0.0
y: float = 0.0
w: float = 0.0
h: float = 0.0
def as_list(self) -> list[float]:
return [self.x, self.y, self.w, self.h]
@dataclass
class TextSpan:
text: str
font_name: str = ""
font_size: float = 0.0
bold: bool = False
italic: bool = False
x: float | None = None
w: float | None = None
url: str | None = None
# Text colour as uppercase ``RRGGBB``, or None for Word's "automatic".
# None is not the same as "000000": automatic follows the reader's theme,
# which is what an author who never set a colour expects.
color: str | None = None
# True when OCR emitted visual LTR Arabic (needs reverse before OOXML storage)
visual_order: bool = False
@dataclass
class Block:
type: BlockType
text: str = ""
level: int = 0 # heading level 1-3 OR list nest depth
bbox: BBox = field(default_factory=BBox)
spans: list[TextSpan] = field(default_factory=list)
cells: list[list[str]] = field(default_factory=list) # table rows
# Merged cell ranges as (r0, c0, r1, c1) inclusive, 0-based
merges: list[tuple[int, int, int, int]] = field(default_factory=list)
table_confidence: float = 0.0
image_png: bytes | None = None
reading_order: int = 0
align: str = "left" # left|center|right|justify
list_style: str = "" # bullet|number|""
# "ocr" | "digital" | PAGE_RASTER_SOURCE | "" — drives Arabic
# visual→logical handling and the faithful DOCX page-raster fallback.
source: str = ""
# True when this block's bbox was fabricated from plain text rather than
# measured from the page. The numbers look real — evenly spaced lines of
# uniform height — so any consumer that reasons about proximity must check
# this first, or it will read a column of table cells as wrapped prose.
synthetic_geometry: bool = False
def plain_text(self) -> str:
if self.type == BlockType.table:
return "\n".join("\t".join(row) for row in self.cells)
if self.spans:
return "".join(s.text for s in self.spans)
return self.text
@dataclass
class Page:
index: int
width: float
height: float
kind: PageKind = PageKind.digital
blocks: list[Block] = field(default_factory=list)
def all_text(self) -> str:
parts = [b.plain_text() for b in self.blocks if b.plain_text().strip()]
return "\n".join(parts)
@dataclass
class Document:
pages: list[Page] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
meta: dict[str, Any] = field(default_factory=dict)
@property
def page_count(self) -> int:
return len(self.pages)
def all_text(self) -> str:
return "\n\n".join(p.all_text() for p in self.pages)
gateway/app/services/convert/idm/serialize.py
+70
View File
@@ -0,0 +1,70 @@
"""IDM JSON serialization / debug dump."""
from __future__ import annotations
import json
import os
from pathlib import Path
from typing import Any
from app.services.convert.idm.model import Block, Document, Page
def block_to_dict(block: Block) -> dict[str, Any]:
return {
"type": block.type.value,
"text": block.text,
"level": block.level,
"bbox": block.bbox.as_list(),
"spans": [
{
"text": s.text,
"font_name": s.font_name,
"font_size": s.font_size,
"bold": s.bold,
"italic": s.italic,
}
for s in block.spans
],
"cells": block.cells,
"merges": [list(m) for m in block.merges],
"table_confidence": block.table_confidence,
"reading_order": block.reading_order,
"has_image": block.image_png is not None,
}
def page_to_dict(page: Page) -> dict[str, Any]:
return {
"index": page.index,
"width": page.width,
"height": page.height,
"kind": page.kind.value,
"blocks": [block_to_dict(b) for b in page.blocks],
}
def document_to_dict(doc: Document) -> dict[str, Any]:
return {
"version": "idm.v1",
"page_count": doc.page_count,
"warnings": list(doc.warnings),
"meta": dict(doc.meta),
"pages": [page_to_dict(p) for p in doc.pages],
}
def dumps_idm(doc: Document, *, indent: int = 2) -> str:
return json.dumps(document_to_dict(doc), indent=indent, ensure_ascii=False)
def maybe_dump_idm(doc: Document, stem: str = "convert") -> str | None:
"""If CONVERT_DEBUG_IDM=1, write IDM JSON under temp/cwd and return path."""
flag = os.environ.get("CONVERT_DEBUG_IDM", "").strip().lower()
if flag not in ("1", "true", "yes"):
return None
out_dir = Path(os.environ.get("CONVERT_DEBUG_DIR", "."))
out_dir.mkdir(parents=True, exist_ok=True)
path = out_dir / f"{stem}-idm.json"
path.write_text(dumps_idm(doc), encoding="utf-8")
return str(path)
gateway/app/services/convert/jobs.py
+308
View File
@@ -0,0 +1,308 @@
"""Persistent SQLite-backed conversion job store with WAL mode and spooling."""
from __future__ import annotations
import json
import os
import sqlite3
import threading
import time
import uuid
from pathlib import Path
from typing import Any
from app.schemas.convert import JobStatus
from contextlib import contextmanager
_DEFAULT_SPOOL_THRESHOLD = 512 * 1024 # 512 KB
class JobStore:
"""Persistent SQLite job store with WAL mode, disk spooling, and crash recovery."""
def __init__(
self,
ttl_seconds: int = 3 * 60 * 60,
db_path: str | Path | None = None,
spool_dir: str | Path | None = None,
):
self._lock = threading.RLock()
self._ttl = ttl_seconds
base_data = Path(__file__).resolve().parents[3] / "data"
if db_path is None:
db_path = os.environ.get("CONVERT_JOBS_DB", str(base_data / "jobs.sqlite3"))
if spool_dir is None:
spool_dir = os.environ.get("CONVERT_JOBS_SPOOL", str(base_data / "jobs_spool"))
self._db_path = Path(db_path)
self._spool_dir = Path(spool_dir)
self._db_path.parent.mkdir(parents=True, exist_ok=True)
self._spool_dir.mkdir(parents=True, exist_ok=True)
self._init_db()
self._recover_crashed_jobs()
@contextmanager
def _conn(self):
conn = sqlite3.connect(
str(self._db_path),
timeout=30.0,
check_same_thread=False,
)
conn.row_factory = sqlite3.Row
try:
with conn:
yield conn
finally:
try:
conn.close()
except Exception:
pass
def _init_db(self) -> None:
with self._lock, self._conn() as conn:
conn.execute("PRAGMA journal_mode=WAL;")
conn.execute("PRAGMA synchronous=NORMAL;")
conn.execute(
"""
CREATE TABLE IF NOT EXISTS jobs (
job_id TEXT PRIMARY KEY,
status TEXT NOT NULL,
source TEXT NOT NULL,
target TEXT NOT NULL,
filename TEXT NOT NULL,
fidelity TEXT,
warnings TEXT,
size_bytes INTEGER,
error TEXT,
result_path TEXT,
result_blob BLOB,
result_filename TEXT,
download_path TEXT,
created_at REAL NOT NULL,
updated_at REAL NOT NULL,
meta TEXT,
cancel_requested INTEGER DEFAULT 0
);
"""
)
conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_created_at ON jobs (created_at);")
conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs (status);")
def _recover_crashed_jobs(self) -> None:
"""Mark any jobs left queued or running from a crashed server as failed."""
with self._lock, self._conn() as conn:
conn.execute(
"""
UPDATE jobs
SET status = ?, error = ?, updated_at = ?
WHERE status IN (?, ?)
""",
(
JobStatus.failed.value,
"Server process restarted during execution",
time.time(),
JobStatus.queued.value,
JobStatus.running.value,
),
)
def create(self, source: str, target: str, filename: str) -> dict[str, Any]:
self._purge_expired()
job_id = str(uuid.uuid4())
now = time.time()
job = {
"job_id": job_id,
"status": JobStatus.queued.value,
"source": source,
"target": target,
"filename": filename,
"fidelity": None,
"warnings": [],
"size_bytes": None,
"error": None,
"result_bytes": None,
"result_filename": None,
"download_path": None,
"created_at": now,
"updated_at": now,
"meta": {},
"cancel_requested": False,
}
with self._lock, self._conn() as conn:
conn.execute(
"""
INSERT INTO jobs (
job_id, status, source, target, filename, fidelity, warnings,
size_bytes, error, result_path, result_blob, result_filename,
download_path, created_at, updated_at, meta, cancel_requested
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
job_id,
job["status"],
source,
target,
filename,
None,
json.dumps([]),
None,
None,
None,
None,
None,
None,
now,
now,
json.dumps({}),
0,
),
)
return job
def update(self, job_id: str, **fields: Any) -> dict[str, Any] | None:
with self._lock:
existing = self.get(job_id)
if not existing:
return None
now = time.time()
set_clauses = ["updated_at = ?"]
params: list[Any] = [now]
# Spool large result bytes to disk
if "result_bytes" in fields:
rb = fields["result_bytes"]
if rb is not None and len(rb) > _DEFAULT_SPOOL_THRESHOLD:
spool_file = self._spool_dir / f"{job_id}.bin"
spool_file.write_bytes(rb)
set_clauses.extend(["result_path = ?", "result_blob = ?"])
params.extend([str(spool_file), None])
elif rb is not None:
set_clauses.extend(["result_blob = ?", "result_path = ?"])
params.extend([rb, None])
else:
set_clauses.extend(["result_blob = ?", "result_path = ?"])
params.extend([None, None])
for k, v in fields.items():
if k in ("result_bytes", "updated_at"):
continue
if k == "warnings":
set_clauses.append("warnings = ?")
params.append(json.dumps(v if isinstance(v, list) else []))
elif k == "meta":
set_clauses.append("meta = ?")
params.append(json.dumps(v if isinstance(v, dict) else {}))
elif k == "cancel_requested":
set_clauses.append("cancel_requested = ?")
params.append(1 if v else 0)
elif k in (
"status",
"fidelity",
"size_bytes",
"error",
"result_filename",
"download_path",
):
set_clauses.append(f"{k} = ?")
params.append(v)
params.append(job_id)
query = f"UPDATE jobs SET {', '.join(set_clauses)} WHERE job_id = ?"
with self._conn() as conn:
conn.execute(query, params)
return self.get(job_id)
def request_cancel(self, job_id: str) -> None:
"""Soft-cancel: checked between pages when cancel_check is wired."""
with self._lock, self._conn() as conn:
conn.execute(
"UPDATE jobs SET cancel_requested = 1, updated_at = ? WHERE job_id = ?",
(time.time(), job_id),
)
def is_cancelled(self, job_id: str) -> bool:
with self._lock, self._conn() as conn:
cur = conn.execute("SELECT cancel_requested FROM jobs WHERE job_id = ?", (job_id,))
row = cur.fetchone()
return bool(row and row["cancel_requested"])
def get(self, job_id: str) -> dict[str, Any] | None:
self._purge_expired()
with self._lock, self._conn() as conn:
cur = conn.execute("SELECT * FROM jobs WHERE job_id = ?", (job_id,))
row = cur.fetchone()
if not row:
return None
return self._row_to_dict(row)
def get_result_bytes(self, job_id: str) -> bytes | None:
with self._lock, self._conn() as conn:
cur = conn.execute(
"SELECT result_blob, result_path FROM jobs WHERE job_id = ?",
(job_id,),
)
row = cur.fetchone()
if not row:
return None
if row["result_path"] and Path(row["result_path"]).is_file():
try:
return Path(row["result_path"]).read_bytes()
except OSError:
return None
return row["result_blob"]
def _row_to_dict(self, row: sqlite3.Row) -> dict[str, Any]:
warnings = []
if row["warnings"]:
try:
warnings = json.loads(row["warnings"])
except Exception:
warnings = []
meta = {}
if row["meta"]:
try:
meta = json.loads(row["meta"])
except Exception:
meta = {}
return {
"job_id": row["job_id"],
"status": row["status"],
"source": row["source"],
"target": row["target"],
"filename": row["filename"],
"fidelity": row["fidelity"],
"warnings": warnings,
"size_bytes": row["size_bytes"],
"error": row["error"],
"result_filename": row["result_filename"],
"download_path": row["download_path"],
"created_at": row["created_at"],
"updated_at": row["updated_at"],
"meta": meta,
"cancel_requested": bool(row["cancel_requested"]),
}
def _purge_expired(self) -> None:
now = time.time()
cutoff = now - self._ttl
with self._lock, self._conn() as conn:
cur = conn.execute(
"SELECT job_id, result_path FROM jobs WHERE created_at < ?",
(cutoff,),
)
expired = cur.fetchall()
for row in expired:
if row["result_path"]:
Path(row["result_path"]).unlink(missing_ok=True)
conn.execute("DELETE FROM jobs WHERE created_at < ?", (cutoff,))
job_store = JobStore()
gateway/app/services/convert/layout/__init__.py
+5
View File
@@ -0,0 +1,5 @@
"""Layout package."""
from app.services.convert.layout.pipeline import build_idm_from_pdf
__all__ = ["build_idm_from_pdf"]
gateway/app/services/convert/layout/blocks.py
+186
View File
@@ -0,0 +1,186 @@
"""Block typing: heading, list, paragraph."""
from __future__ import annotations
import re
from app.services.convert.idm.model import BBox, Block, BlockType, TextSpan
from app.services.convert.layout.glyphs import Line
from app.services.convert.layout.styles import heading_level, is_caption
_BULLET_RE = re.compile(r"""^(\u2022|\-|\*|•|○|●|◦|·|▪|['"`])\s+""")
_NUMBER_RE = re.compile(r"^(\d+[\.\)]|\(\d+\))\s+")
_LETTER_RE = re.compile(r"^([A-Za-z][\.\)]|\([A-Za-z]\))\s+")
# OCR often turns a bullet into a lone I / l before the sentence
_OCR_BULLET_RE = re.compile(r"^[Il]\s+[A-ZÀ-ÖØ-Þ]")
_URI_RE = re.compile(r"(https?://[^\s<>]+|www\.[^\s<>]+)", re.I)
def _annotate_uris(spans: list[TextSpan]) -> list[TextSpan]:
out: list[TextSpan] = []
for s in spans:
m = _URI_RE.search(s.text or "")
if m and not s.url:
out.append(
TextSpan(
text=s.text,
font_name=s.font_name,
font_size=s.font_size,
bold=s.bold,
italic=s.italic,
x=s.x,
w=s.w,
url=m.group(1),
)
)
else:
out.append(s)
return out
def _line_is_bold(line: Line) -> bool:
"""Whether essentially the whole line is bold, weighted by character count.
A part-bold line is emphasis inside prose -- "**Note:** the rest of the
sentence" -- and must not read as a heading, so the test is share of
characters rather than presence of any bold span.
"""
total = bold = 0
for span in line.spans or []:
n = len((span.text or "").strip())
if not n:
continue
total += n
if span.bold:
bold += n
return total > 0 and (bold / total) >= 0.8
def line_to_block(line: Line, order: int, body_size: float) -> Block:
text = line.text.strip()
list_style = ""
nest = 0
# M2: prefer list detection over heading for bullet/number markers
if _BULLET_RE.match(text) or _OCR_BULLET_RE.match(text):
btype = BlockType.list_item
list_style = "bullet"
level = 0
elif _NUMBER_RE.match(text):
btype = BlockType.list_item
list_style = "number"
level = 0
elif _LETTER_RE.match(text):
btype = BlockType.list_item
list_style = "letter"
level = 0
elif is_caption(text):
btype = BlockType.paragraph
level = 0
else:
level = heading_level(text, line.font_size, body_size, bold=_line_is_bold(line))
if level:
btype = BlockType.heading
else:
btype = BlockType.paragraph
level = 0
# Indent heuristic → shallow nest
if btype == BlockType.list_item and line.x0 > 100:
nest = 1
spans = line.spans or [TextSpan(text=text, font_size=line.font_size, font_name=line.font_name)]
spans = _annotate_uris(spans)
return Block(
type=btype,
text=text,
level=level or nest,
bbox=BBox(x=line.x0, y=line.y, w=max(line.x1 - line.x0, 1), h=line.font_size),
spans=spans,
reading_order=order,
list_style=list_style,
align="left",
synthetic_geometry=bool(getattr(line, "synthetic_geometry", False)),
)
def lines_to_block(
lines: list[Line],
order: int,
body_size: float,
*,
lexical_pairs: set[tuple[str, str]] | None = None,
) -> Block:
"""Build one block from a paragraph run of wrapped lines.
Single-line runs delegate to :func:`line_to_block` so heading, caption and
list detection behave identically to the per-line path. Multi-line runs are
joined with hyphenation resolved, and carry the union bbox plus the
concatenated spans so run-level styling survives into the formatters.
"""
if not lines:
raise ValueError("lines_to_block requires at least one line")
if len(lines) == 1:
return line_to_block(lines[0], order, body_size)
from app.services.convert.layout.paragraphs import join_line_texts
text = join_line_texts([ln.text for ln in lines], lexical_pairs=lexical_pairs)
first = lines[0]
spans: list[TextSpan] = []
for idx, ln in enumerate(lines):
line_spans = ln.spans or [
TextSpan(text=ln.text, font_size=ln.font_size, font_name=ln.font_name)
]
if idx > 0 and spans and line_spans:
# Wrapped lines are separated by a space unless the previous line
# was hyphenated, which join_line_texts has already resolved.
prev_text = spans[-1].text or ""
if prev_text and not prev_text.endswith((" ", "-", "­")):
spans.append(
TextSpan(
text=" ",
font_size=line_spans[0].font_size,
font_name=line_spans[0].font_name,
)
)
elif prev_text.endswith(("-", "­")) and not text.count(prev_text[-1]):
# join_line_texts removed the hyphen; mirror that in the spans.
spans[-1] = TextSpan(
text=prev_text[:-1],
font_name=spans[-1].font_name,
font_size=spans[-1].font_size,
bold=spans[-1].bold,
italic=spans[-1].italic,
x=spans[-1].x,
w=spans[-1].w,
url=spans[-1].url,
color=spans[-1].color,
visual_order=spans[-1].visual_order,
)
spans.extend(line_spans)
spans = _annotate_uris(spans)
x0 = min(ln.x0 for ln in lines)
x1 = max(ln.x1 for ln in lines)
y_top = max(ln.y for ln in lines)
y_bot = min(ln.y for ln in lines)
height = max(y_top - y_bot + (first.font_size or body_size), first.font_size or body_size)
return Block(
type=BlockType.paragraph,
text=text,
level=0,
bbox=BBox(x=x0, y=y_bot, w=max(x1 - x0, 1), h=height),
spans=spans,
reading_order=order,
align="left",
synthetic_geometry=any(
getattr(ln, "synthetic_geometry", False) for ln in lines
),
)
def estimate_body_size(lines: list[Line]) -> float:
if not lines:
return 12.0
sizes = sorted(ln.font_size for ln in lines if ln.font_size > 0)
return sizes[len(sizes) // 2]
gateway/app/services/convert/layout/coords.py
+62
View File
@@ -0,0 +1,62 @@
"""PDF page points ↔ raster pixel coordinate transforms."""
from __future__ import annotations
from dataclasses import dataclass
from app.services.convert.idm.model import BBox
@dataclass(frozen=True)
class PageGeom:
page_width: float
page_height: float
dpi: float
@property
def scale(self) -> float:
return self.dpi / 72.0
@property
def pixel_w(self) -> int:
return max(1, int(round(self.page_width * self.scale)))
@property
def pixel_h(self) -> int:
return max(1, int(round(self.page_height * self.scale)))
def pdf_to_pixel(x: float, y: float, geom: PageGeom) -> tuple[float, float]:
"""PDF origin bottom-left → image origin top-left."""
px = x * geom.scale
py = (geom.page_height - y) * geom.scale
return px, py
def pixel_to_pdf(px: float, py: float, geom: PageGeom) -> tuple[float, float]:
"""Image top-left → PDF bottom-left."""
x = px / geom.scale
y = geom.page_height - (py / geom.scale)
return x, y
def pixel_bbox_to_pdf(x0: float, y0: float, x1: float, y1: float, geom: PageGeom) -> BBox:
"""Axis-aligned pixel box (top-left origin) → PDF BBox (x,y,w,h bottom-left)."""
# corners
corners = [
pixel_to_pdf(x0, y0, geom),
pixel_to_pdf(x1, y0, geom),
pixel_to_pdf(x0, y1, geom),
pixel_to_pdf(x1, y1, geom),
]
xs = [c[0] for c in corners]
ys = [c[1] for c in corners]
x_min, x_max = min(xs), max(xs)
y_min, y_max = min(ys), max(ys)
return BBox(x=x_min, y=y_min, w=max(0.0, x_max - x_min), h=max(0.0, y_max - y_min))
def pdf_bbox_to_pixel(bbox: BBox, geom: PageGeom) -> tuple[float, float, float, float]:
x0, y_top = pdf_to_pixel(bbox.x, bbox.y + bbox.h, geom)
x1, y_bot = pdf_to_pixel(bbox.x + bbox.w, bbox.y, geom)
return min(x0, x1), min(y_top, y_bot), max(x0, x1), max(y_top, y_bot)
gateway/app/services/convert/layout/doc_wide.py
+293
View File
@@ -0,0 +1,293 @@
"""Document-wide analysis (ABBYY ADRT / Adobe Extract patterns, in-house)."""
from __future__ import annotations
from collections import Counter
from app.services.convert.idm.model import Block, BlockType, Document
from app.services.convert.layout.headers_footers import tag_headers_footers
from app.services.convert.layout.styles import (
TERMINAL_PUNCT_RE,
dominant_body_size,
heading_level,
section_number_level,
)
from app.services.convert.options import HeaderFooterMode, get_options
def _norm(text: str) -> str:
return " ".join((text or "").lower().split())
def _untag_headers_footers(document: Document) -> None:
for page in document.pages:
for b in page.blocks:
if b.type in (BlockType.header, BlockType.footer):
b.type = BlockType.paragraph
b.level = 0
def _drop_header_footer_blocks(document: Document) -> None:
for page in document.pages:
page.blocks = [
b for b in page.blocks if b.type not in (BlockType.header, BlockType.footer)
]
for i, b in enumerate(page.blocks):
b.reading_order = i
def _repeating_heading_once(document: Document) -> None:
"""Adobe Extract: repeating headings are reported once — retag rest as header.
A banner promoted to H1 on every page produces an unusable outline.
"""
if document.page_count < 2:
return
heading_texts: list[str] = []
for page in document.pages:
for b in page.blocks:
if b.type == BlockType.heading:
t = _norm(b.plain_text())
if t:
heading_texts.append(t)
if len(heading_texts) < 2:
return
counts = Counter(heading_texts)
repeats = {t for t, n in counts.items() if n >= 2 and len(t) >= 8}
if not repeats:
return
seen: set[str] = set()
for page in document.pages:
for b in page.blocks:
if b.type != BlockType.heading:
continue
key = _norm(b.plain_text())
if key not in repeats:
continue
if key in seen:
b.type = BlockType.header
b.level = 0
else:
seen.add(key)
def _block_is_bold(block: Block) -> bool:
"""Whether essentially the whole block is bold, weighted by character count.
Mirrors ``layout.blocks._line_is_bold`` so the two passes classify the same
text the same way.
"""
total = bold = 0
for span in block.spans or []:
n = len((span.text or "").strip())
if not n:
continue
total += n
if span.bold:
bold += n
return total > 0 and (bold / total) >= 0.8
def _block_color(block: Block) -> str | None:
"""The colour this block is *set* in, weighted by characters, or None.
Weighted rather than "any coloured span" for the same reason
``_block_is_bold`` is: one coloured word inside a black sentence is emphasis,
not a coloured line. A block only counts as coloured when essentially all of
it is, and all of it the same colour.
"""
counts: dict[str, int] = {}
total = 0
for span in block.spans or []:
n = len((span.text or "").strip())
if not n:
continue
total += n
colour = getattr(span, "color", None)
if colour:
counts[str(colour).upper()] = counts.get(str(colour).upper(), 0) + n
if not total or not counts:
return None
colour, n = max(counts.items(), key=lambda kv: kv[1])
return colour if (n / total) >= 0.8 else None
# A coloured line is only heading evidence when it is at least this tall relative
# to body text. Word's default Heading 3 is 12pt on an 11pt body -- 1.09x -- so
# the bar has to sit below that, and equal-sized coloured text is admitted too:
# a coloured run-in heading at body size is a real convention. Size is not doing
# the work here, colour is; this only rejects coloured *small print*.
_COLOUR_HEADING_MIN_RATIO = 0.98
# Longest a coloured line may be to still read as a heading. Deliberately tighter
# than the size-based path's 14: colour is weaker evidence than a size jump, and
# a coloured pull-quote or a warning paragraph is exactly what would otherwise be
# swept up.
_COLOUR_HEADING_MAX_WORDS = 12
def _promote_coloured_headings(document: Document) -> None:
"""Promote blocks whose colour sets them apart from the document's body text.
Word's built-in heading styles are *coloured, not bold*: in the default
template Heading 1 is 365F91 and Headings 2 and 3 are 4F81BD, all on black
body text. Heading 3 is 12pt on an 11pt body -- a ratio of 1.09, below the
smallest size tier -- and it is not bold, so neither the size path nor the
weight path in ``styles.heading_level`` can see it. Colour is the only
evidence the document offers, and until now it was thrown away at extraction.
The effect was that every Heading 3 in a default-styled Word document came
back from PDF as ordinary body text, which silently flattens the outline that
Word's navigation pane and any generated table of contents are built from.
The comparison is against the document's *own* dominant text colour, not
against black. A document set entirely in dark blue has no colour signal at
all, and treating every short line in it as a heading would be far worse than
missing a few: this pass returns immediately in that case. That is also why
the body colour is counted over the whole document rather than per page --
a page that happens to contain nothing but headings would otherwise conclude
that its heading colour *is* the body colour and promote nothing.
"""
body_counts: dict[str | None, int] = {}
for page in document.pages:
for b in page.blocks:
if b.type not in (BlockType.paragraph, BlockType.heading):
continue
n = len((b.plain_text() or "").strip())
if n:
key = _block_color(b)
body_counts[key] = body_counts.get(key, 0) + n
if not body_counts:
return
body_colour = max(body_counts.items(), key=lambda kv: kv[1])[0]
sizes: list[float] = []
for page in document.pages:
for b in page.blocks:
if b.spans:
sizes.extend(s.font_size for s in b.spans if s.font_size and s.font_size > 0)
body_size = dominant_body_size(sizes) if sizes else 12.0
for page in document.pages:
for b in page.blocks:
if b.type != BlockType.paragraph:
continue
colour = _block_color(b)
if not colour or colour == body_colour:
continue
text = (b.plain_text() or "").strip()
words = text.split()
if not words or len(words) > _COLOUR_HEADING_MAX_WORDS:
continue
# A line that ends a sentence is prose, however it is coloured --
# the same test the weight-based path uses, and for the same reason.
if TERMINAL_PUNCT_RE.search(text):
continue
if _heading_size(b) < body_size * _COLOUR_HEADING_MIN_RATIO:
continue
b.type = BlockType.heading
# Provisional; ``_rank_heading_levels`` assigns the real depth below
# once every heading size in the document is known.
b.level = 3
def _style_catalog_demote(document: Document) -> None:
"""DetectStyles: body-sized 'headings' are body once the doc median is known."""
sizes: list[float] = []
for page in document.pages:
for b in page.blocks:
if b.type in (BlockType.paragraph, BlockType.heading, BlockType.list_item):
if b.spans:
sizes.extend(s.font_size for s in b.spans if s.font_size and s.font_size > 0)
body = dominant_body_size(sizes) if sizes else 12.0
for page in document.pages:
for b in page.blocks:
if b.type != BlockType.heading:
continue
text = b.plain_text().strip()
size = 12.0
if b.spans:
ss = [s.font_size for s in b.spans if s.font_size and s.font_size > 0]
if ss:
size = sum(ss) / len(ss)
# Trust numbered sections; demote size-only H1 that matches body.
# The bold signal must be passed here too: this pass re-runs the same
# classifier, so without it a heading recognised on weight would be
# demoted straight back to body text and the fix upstream would have
# no visible effect.
if heading_level(text, size, body, bold=_block_is_bold(b)) == 0:
b.type = BlockType.paragraph
b.level = 0
def _heading_size(block: Block) -> float:
if block.spans:
ss = [s.font_size for s in block.spans if s.font_size and s.font_size > 0]
if ss:
return sum(ss) / len(ss)
return 12.0
# Heading sizes within this many points of each other are one tier. Half a point
# of drift is normal between a heading and its own continuation line.
_TIER_SNAP_PT = 0.75
def _rank_heading_levels(document: Document) -> None:
"""Assign heading levels by rank among the sizes this document actually uses.
``styles.heading_level`` scores each line in isolation against fixed multiples
of body size, needing 1.55x for level 1. Word's own default template sets
Heading 1/2/3 at 16/13/12pt on an 11pt body -- ratios of 1.45, 1.18 and 1.09 --
so a PDF printed from a default-styled Word document came back with its H1
demoted to Heading 2, its H2 to Heading 3 and its H3 dropped to body text.
The document then has no level-1 heading at all, which empties Word's
navigation pane and any generated table of contents. Documents set with
dramatic contrast scored correctly, which is why the fixed tiers survived:
they encode one house style rather than a general rule.
Rank is the general rule. Whatever contrast a document chose, its largest
heading size is its top level and each smaller size is one level down -- which
is how a reader infers an outline, and it needs no assumption about how much
larger a heading "should" be. Levels from explicit section numbers are left
alone: ``1.2.3`` states its own depth, and that beats any inference from size.
"""
ranked: list[tuple[Block, float]] = []
for page in document.pages:
for b in page.blocks:
if b.type != BlockType.heading:
continue
if section_number_level(b.plain_text().strip()):
continue # authoritative depth, do not override
ranked.append((b, round(_heading_size(b) * 2) / 2.0))
if not ranked:
return
tiers: list[float] = []
for size in sorted({s for _b, s in ranked}, reverse=True):
if not tiers or (tiers[-1] - size) > _TIER_SNAP_PT:
tiers.append(size)
for block, size in ranked:
rank = min(range(len(tiers)), key=lambda i: abs(tiers[i] - size))
block.level = min(rank + 1, 3)
def apply_document_wide(document: Document) -> None:
"""Cluster HF, style catalog, repeating-heading-once. Honour header_footer_mode."""
opts = get_options()
mode = opts.header_footer_mode
if mode == HeaderFooterMode.ignore:
_untag_headers_footers(document)
else:
tag_headers_footers(document)
if mode == HeaderFooterMode.remove:
_drop_header_footer_blocks(document)
_style_catalog_demote(document)
# After the demote, not before: the demote pass re-runs the size/weight
# classifier and would take a colour-only heading straight back out again.
# Before the ranking, so the promoted blocks are ranked with the rest.
_promote_coloured_headings(document)
_rank_heading_levels(document)
_repeating_heading_once(document)
if mode == HeaderFooterMode.remove:
_drop_header_footer_blocks(document)
gateway/app/services/convert/layout/glyphs.py
+604
View File
@@ -0,0 +1,604 @@
"""Glyph / run clustering into lines."""
from __future__ import annotations
from dataclasses import dataclass, field
from app.services.convert.idm.model import TextSpan
from app.services.convert.layout.styles import infer_font_flags, style_key
from app.services.convert.text.arabic_logical import is_rtl_dominant
# How many *consecutive* text rows a band must stay blank down before it counts
# as a gutter. Five is enough to rule out a stretched space in justified text
# and short enough that a full-width figure dropped between two columns still
# leaves a usable run of rows above and below itself.
MIN_GUTTER_ROWS = 5
# A PDF text walker can place a continuation glyph a fraction of a point on
# the far side of a detected gutter. This is common when a zero-width space
# marker and its following character are emitted with separate text matrices:
# the marker/character pair is one word, not a new page column. Keep the
# tolerance below ordinary inter-column spacing so genuine columns remain
# isolated.
GUTTER_EDGE_CONTINUATION_PT = 2.0
@dataclass
class Line:
y: float
x0: float
x1: float
text: str
font_size: float = 12.0
font_name: str = ""
spans: list[TextSpan] = field(default_factory=list)
# True when x/y were fabricated by lines_from_plain_text rather than measured
# from glyph geometry. Consumers must not draw geometric conclusions
# (columns, indents, line width) from a line carrying this flag.
synthetic_geometry: bool = False
def _span_from_glyph(g: dict, default_size: float) -> TextSpan | None:
t = str(g.get("text", ""))
if not t:
return None
w = float(g.get("w", 0) or 0)
# PDF producers commonly encode a word space as a zero-width glyph whose
# advance is carried by the following text matrix. Dropping that marker
# forces the line builder to infer every boundary from tiny x gaps; in
# tightly set fonts those gaps are indistinguishable from kerning and
# ordinary words get welded together ("Tracebased" / "Justintime").
# Keep the marker with zero width so the explicit source signal wins.
fname = str(g.get("fontName") or g.get("font_name") or "")
h = float(g.get("h", 0) or 0)
raw_size = float(g.get("fontSize") or 0)
if raw_size <= 2.0 and h > 2.0:
fsize = h
elif raw_size > 0:
fsize = raw_size
elif h > 0:
fsize = h
else:
fsize = default_size
bold = bool(g.get("bold")) if "bold" in g else False
italic = bool(g.get("italic")) if "italic" in g else False
if not bold and not italic:
ib, ii = infer_font_flags(fname)
bold = bold or ib
italic = italic or ii
return TextSpan(
text=t,
font_name=fname,
font_size=fsize,
bold=bold,
italic=italic,
x=float(g.get("x", 0)),
w=w or len(t) * 6,
color=(str(g["color"]) if g.get("color") else None),
)
def _runs_on(cur: TextSpan, nxt: TextSpan) -> bool:
"""Whether *nxt* continues *cur* without an intervening gap.
A merged span keeps the first span's ``x`` and the *sum* of the widths, so
merging across a gap yields a span that claims to start where the first one
did and to be far narrower than the ground it covers. Every consumer that
asks "which column does this text sit in?" is then told the wrong answer —
which is how a table row's six cells collapsed into its first one.
"""
if cur.x is None or nxt.x is None:
return True
gap = float(nxt.x) - (float(cur.x) + float(cur.w or 0.0))
if gap <= 0:
return True
# An inter-word space runs about a third of an em. Anything appreciably
# wider is column spacing, a tab stop or a dot leader.
return gap <= max(2.0, float(cur.font_size or 12.0) * 0.8)
def coalesce_spans(spans: list[TextSpan]) -> list[TextSpan]:
"""Merge adjacent spans that share a style *and* run on without a gap."""
if not spans:
return []
out: list[TextSpan] = []
cur = TextSpan(
text=spans[0].text,
font_name=spans[0].font_name,
font_size=spans[0].font_size,
bold=spans[0].bold,
italic=spans[0].italic,
x=spans[0].x,
w=spans[0].w,
color=spans[0].color,
)
cur_right = float(cur.x or 0) + float(cur.w or 0)
for s in spans[1:]:
sx = float(s.x or 0)
sw = float(s.w or 0)
gap = sx - cur_right
same_style = (
(
style_key(cur.font_name, cur.font_size, cur.bold, cur.italic)
== style_key(s.font_name, s.font_size, s.bold, s.italic)
or (
cur.font_name.lower().strip() == s.font_name.lower().strip()
and cur.bold == s.bold
and cur.italic == s.italic
and abs(float(cur.font_size or 0) - float(s.font_size or 0)) <= 2.0
)
)
and cur.color == s.color
)
max_gap = max(2.0, float(cur.font_size or 12.0) * 0.8)
if same_style and (gap <= max_gap or gap <= 0):
cur.text += s.text
cur_right = max(cur_right, sx + sw)
if cur.x is not None:
cur.w = cur_right - float(cur.x)
elif s.w is not None:
cur.w = s.w
else:
out.append(cur)
cur = TextSpan(
text=s.text,
font_name=s.font_name,
font_size=s.font_size,
bold=s.bold,
italic=s.italic,
x=s.x,
w=s.w,
color=s.color,
)
cur_right = sx + sw
out.append(cur)
return out
def page_gutters(
glyphs: list[dict], *, min_width: float = 12.0, row_height: float = 12.0
) -> list[tuple[float, float, float]]:
"""Vertical bands almost no row of the page writes in — its column gutters.
A projection profile over the whole page, not a per-line gap test. The
gutter of a two-column paper is often only 20pt wide, so a fixed
"gap > 36pt means two columns" rule welds both columns of every academic
paper into single lines; and a *per-line* threshold cannot tell a stretched
space in justified text from a real gutter, because on one line they look
identical. Across the page they do not: a gutter is empty on nearly every
row, a stretched space is not.
The profile is *local*, computed over a sliding window of neighbouring
rows rather than the whole page. A page is rarely one thing throughout: a
paper puts a full-width title and author block above two columns of
abstract, and a full-width figure in the middle of them. A page-wide
profile finds no gutter on such a page at all, because the title crosses
every candidate band — and then both columns of the abstract come out
interleaved line by line.
Returns ``(x, y_low, y_high)`` for each gutter band, so a caller can apply
a gutter only to the rows it actually separates.
"""
if len(glyphs) < 12:
return []
xs0 = [float(g.get("x", 0.0)) for g in glyphs]
xs1 = [x + float(g.get("w", 0.0) or 0.0) for x, g in zip(xs0, glyphs, strict=False)]
left, right = min(xs0), max(xs1)
span = right - left
if span <= 0:
return []
step = max(row_height, 1.0)
bin_pt = 2.0
bins = max(8, min(2048, int(span / bin_pt) + 1))
per_bin = span / bins
need = max(2, int(min_width / per_bin))
# Cluster glyphs into baseline rows so descenders (p, g, y) do not fall
# into a separate row and create false gutter detections.
ordered_y = sorted(glyphs, key=lambda g: -float(g.get("y", 0.0)))
row_clusters: list[list[dict]] = []
row_ys: list[float] = []
cluster_tol = max(step * 0.7, 4.0)
for g in ordered_y:
gy = float(g.get("y", 0.0))
placed = False
for idx in range(len(row_clusters) - 1, -1, -1):
if abs(gy - row_ys[idx]) <= cluster_tol:
row_clusters[idx].append(g)
placed = True
break
if row_ys[idx] - gy > cluster_tol:
break
if not placed:
row_clusters.append([g])
row_ys.append(gy)
# One occupancy bitmap per text row, top of page first.
occ: dict[int, int] = {}
for cl, ry in zip(row_clusters, row_ys, strict=False):
row_key = int(round(ry / step))
bits = 0
for g in cl:
gx0 = float(g.get("x", 0.0))
gw = float(g.get("w", 0.0) or 0.0)
i = max(0, min(bins - 1, int((gx0 - left) / span * bins)))
j = max(0, min(bins - 1, int((gx0 + gw - left) / span * bins)))
bits |= (((1 << (j - i + 1)) - 1) << i)
occ[row_key] = occ.get(row_key, 0) | bits
order = sorted(occ, reverse=True)
if len(order) < MIN_GUTTER_ROWS:
return []
full = (1 << bins) - 1
# Bins each row leaves blank, top of page first.
blank = [full ^ occ[r] for r in order]
# A column must be substantially wider than the gutter beside it. Without
# this, any persistently blank band qualifies -- and a form is full of them:
# the numbered stub down the left of an IRS 1040 leaves a real, unbroken gap
# between the item number and its label, so "24 Add lines 22 and 23" was
# split into "24" and "Add lines 22 and 23", detaching every line number
# from the line it numbers. A 10pt-wide column of two-digit numbers is a
# stub, not a text column.
min_side = max(min_width * 3.0, span * 0.12)
out: list[tuple[float, float, float]] = []
# A gutter is a band that stays blank down a *run* of consecutive rows.
#
# This used to be a window centred on each row that required unanimity,
# which inverted the intent described above: the three rows below a
# full-width title, author block or figure caption still had that full-width
# row inside their window, so the gutter was suppressed for exactly the
# first rows of the column region -- the rows where the columns begin.
# Measured on the TraceMonkey paper, the real x=305 gutter was found only
# from y=421 down while the two-column abstract starts at y=445, so its
# first rows were never split, both columns welded into single lines, and
# "...more difficult to com-" was joined to "applications such as Google
# Mail..." to yield "Zimbra Colpile". Recall cannot see that -- every word is
# still present, in the wrong order and partly re-spelled -- which is how it
# survived.
#
# Runs carry the same evidence without the boundary artefact: a band is a
# gutter at every row of any run of MIN_GUTTER_ROWS consecutive rows it
# stays blank down. A stretched space in justified text is blank on one or
# two rows and still cannot reach the run length, so the false positive the
# projection profile exists to avoid is still avoided; and a full-width
# figure between two columns now ends one run and starts another instead of
# erasing three rows on each side of itself.
for s in range(len(order) - MIN_GUTTER_ROWS + 1):
rows = order[s : s + MIN_GUTTER_ROWS]
empty = full
union = 0
for i in range(s, s + MIN_GUTTER_ROWS):
empty &= blank[i]
union |= occ[order[i]]
if not empty or not union:
continue
bits = bin(empty)[2:].zfill(bins)[::-1]
# The ground the run's own rows cover. Judged over the run rather than a
# neighbourhood, because the question is what *this* band separates: a
# row that writes straight through the band is not one of its columns.
used = bin(union)[2:].zfill(bins)[::-1]
first, last = used.find("1"), used.rfind("1")
run = 0
for k in range(bins + 1):
if k < bins and bits[k] == "1":
run += 1
continue
if run >= need:
start = k - run
# Interior only: a gutter has content on both sides of it, and
# enough of it on each side to be a column.
lo_end, hi_start = used.rfind("1", 0, start), used.find("1", k)
if lo_end >= 0 and hi_start >= 0:
left_w = (lo_end - first + 1) * per_bin
right_w = (last - hi_start + 1) * per_bin
if left_w >= min_side and right_w >= min_side:
x = left + (start + k) / 2.0 * per_bin
out.extend((x, r * step, r * step) for r in rows)
run = 0
return _merge_gutter_spans(out, step)
def _merge_gutter_spans(
spans: list[tuple[float, float, float]], step: float
) -> list[tuple[float, float, float]]:
"""Collapse per-row gutter hits into ``(x, y_low, y_high)`` bands."""
if not spans:
return []
merged: list[list[float]] = []
for x, y0, y1 in sorted(spans, key=lambda s: (round(s[0], 0), s[1])):
for band in merged:
if abs(band[0] - x) <= 4.0 and y0 <= band[2] + step * 2.5 and y1 >= band[1] - step * 2.5:
band[0] = (band[0] + x) / 2.0
band[1] = min(band[1], y0)
band[2] = max(band[2], y1)
break
else:
merged.append([x, y0, y1])
return [(b[0], b[1], b[2]) for b in merged]
def _split_group_at(
group: list[dict], gutters: list[tuple[float, float, float]], *, slack: float = 6.0
) -> list[list[dict]]:
"""Split one y-group wherever it straddles a gutter that spans its rows."""
if not gutters or len(group) < 2:
return [group]
ys = [float(g.get("y", 0.0)) for g in group]
y = sum(ys) / len(ys)
here = sorted(x for x, lo, hi in gutters if lo - slack <= y <= hi + slack)
if not here:
return [group]
group = sorted(group, key=lambda g: float(g.get("x", 0)))
parts: list[list[dict]] = []
current: list[dict] = []
idx = 0
for g in group:
x = float(g.get("x", 0))
cur_right = max(
(float(c.get("x", 0)) + float(c.get("w", 0) or 0) for c in current),
default=0.0,
)
while idx < len(here) and x >= here[idx]:
# Only split if there is an actual gap separating columns across the gutter
if current and (x - cur_right >= 4.0) and (cur_right <= here[idx] + 2.0):
parts.append(current)
current = []
idx += 1
current.append(g)
if current:
parts.append(current)
return parts or [group]
def _split_group_on_gutter(group: list[dict], *, min_gap: float = 36.0) -> list[list[dict]]:
"""Recursively split a y-clustered glyph group on large horizontal gutters (2+ columns)."""
if len(group) < 2:
return [group]
group = sorted(group, key=lambda g: float(g.get("x", 0)))
best_i = -1
best_gap = 0.0
for i in range(len(group) - 1):
x0 = float(group[i].get("x", 0))
w0 = float(group[i].get("w", 0) or 0)
x1 = float(group[i + 1].get("x", 0))
gap = x1 - (x0 + w0)
if gap > best_gap:
best_gap = gap
best_i = i
if best_i < 0 or best_gap < min_gap:
return [group]
left, right = group[: best_i + 1], group[best_i + 1 :]
if not left or not right:
return [group]
out: list[list[dict]] = []
out.extend(_split_group_on_gutter(left, min_gap=min_gap))
out.extend(_split_group_on_gutter(right, min_gap=min_gap))
return out
def _group_is_rtl(group: list[dict], *, threshold: float = 0.5) -> bool:
"""True when a line's letters are predominantly right-to-left."""
text = "".join(str(g.get("text", "")) for g in group)
return is_rtl_dominant(text, threshold=threshold)
def _sort_group_by_reading_order(group: list[dict]) -> list[dict]:
"""Order a line's glyphs in logical reading order for its dominant script."""
if _group_is_rtl(group):
# Right-to-left: the logical first token sits furthest right. Sort by
# the right edge descending so a wide glyph does not overtake a narrow
# neighbour that starts further right.
return sorted(
group,
key=lambda g: -(float(g.get("x", 0)) + float(g.get("w", 0) or 0)),
)
return sorted(group, key=lambda g: float(g.get("x", 0)))
def _column_band(
g: dict,
gutters: list[tuple[float, float, float]],
*,
slack: float = 6.0,
at_y: float | None = None,
) -> tuple[int, int]:
"""Which column *g* sits in: whether gutters are active, and its column side index."""
if not gutters:
return (0, 0)
y = at_y if at_y is not None else float(g.get("y", 0.0))
active = tuple(sorted(gx for gx, lo, hi in gutters if lo - slack <= y <= hi + slack))
if not active:
return (0, 0)
x = float(g.get("x", 0.0))
return (
1,
sum(1 for gx in active if x >= gx + GUTTER_EDGE_CONTINUATION_PT),
)
def lines_from_glyphs(glyphs: list[dict]) -> list[Line]:
"""Cluster glyph dicts {text,x,y,w,h,fontSize?,fontName?} into reading lines."""
# Keep zero-width whitespace glyphs: they are real word-boundary markers
# in PDFs that encode spacing through text-matrix advances.
clean_glyphs = [g for g in glyphs if str(g.get("text", ""))]
if not clean_glyphs:
return []
sizes = [float(g.get("fontSize") or g.get("h") or 12.0) for g in clean_glyphs]
median = sorted(sizes)[len(sizes) // 2] if sizes else 12.0
# Generous tol so descenders (g,p,y) stay on the same line as the baseline
tol = max(median * 0.7, 4.0)
# 12pt is wider than the gutter of the most densely set two-column pages --
# the Federal Register CFR pages in the corpus leave only 9pt -- so those
# still weld their columns together. Lowering the floor to about one em does
# fix them, and the two tests in ``page_gutters`` are strong enough to keep
# an inter-word space from passing at that width. It is not landed because it
# is not yet *safe*: splitting the IRS 1040 address block into more lines
# makes something downstream of table reconstruction drop about fifteen
# words, both "Last name" labels among them, and losing a label from a tax
# form is worse than leaving a known welding case unfixed. Raise this only
# together with a fix for that.
gutters = page_gutters(
clean_glyphs, min_width=max(12.0, median * 1.4), row_height=max(median * 0.9, 4.0)
)
# Baselines, descending. Two costs used to hide here: the mean of every
# bucket was recomputed for every glyph, and every bucket was tried even
# though the input is sorted. Carrying a running total and stopping once a
# bucket's baseline is out of reach turns a quadratic scan — 1.6M
# arithmetic operations on four dense pages — into a linear one.
ordered = sorted(clean_glyphs, key=lambda g: -float(g.get("y", 0)))
buckets: list[list[dict]] = []
totals: list[float] = []
# One band per bucket, so a bucket's running baseline is never an average of
# two columns. See ``_column_band`` for what goes wrong without this.
bands: list[tuple[int, int]] = []
for g in ordered:
y = float(g.get("y", 0))
placed = False
for idx in range(len(buckets) - 1, -1, -1):
by = totals[idx] / len(buckets[idx])
if abs(y - by) <= tol:
band = _column_band(g, gutters, at_y=by)
if bands[idx] != band:
# Another column's row at this height. Keep looking for one
# of *this* column's rows; do not join and do not stop, or a
# glyph would be pulled into the wrong column.
continue
buckets[idx].append(g)
totals[idx] += y
placed = True
break
if by - y > tol:
# Buckets before this one sit higher still; none can match.
break
if not placed:
buckets.append([g])
totals.append(y)
bands.append(_column_band(g, gutters, at_y=y))
lines: list[Line] = []
for group in buckets:
for band in _split_group_at(group, gutters):
for subgroup in _split_group_on_gutter(band):
# Reading order within a line follows the script, not the page.
# Sorting a right-to-left line by ascending x yields *visual* order,
# which reverses the words: "مشروع منصة هويتي" comes out as
# "هويتي منصة مشروع". Ordering by descending x recovers logical
# order at the source, which is far more reliable than trying to
# detect and undo the reversal downstream.
rtl_line = _group_is_rtl(subgroup)
subgroup = _sort_group_by_reading_order(subgroup)
spans: list[TextSpan] = []
parts: list[str] = []
prev_x1: float | None = None
for g in subgroup:
span = _span_from_glyph(g, median)
if span is None:
continue
x0 = float(g.get("x", 0))
# Insert inter-word space when engine omits space glyphs.
# Conservative: avoid splitting glued tokens like Apple10 / LeftCol1A.
if (
prev_x1 is not None
and parts
and not parts[-1].endswith((" ", "\t"))
and not (span.text or "").startswith((" ", "\t"))
):
# In a right-to-left line the next glyph lies to the
# *left*, so the inter-glyph gap runs the other way.
gap = (prev_x1 - (x0 + float(g.get("w", 0) or 0))) if rtl_line else (x0 - prev_x1)
prev_ch = parts[-1][-1]
next_ch = (span.text or " ")[0]
glued_alnum = prev_ch.isalnum() and next_ch.isalnum()
# The gap is font-relative. Where the extractor reports
# the font's own space advance for this text state, use
# it: typeset text positions words with TJ offsets
# rather than space glyphs, and a fraction-of-median
# rule welds them together — "Trace-based Just-in-Time"
# arrived as one token at 9pt. Fall back to the median
# heuristic when no space width is reported.
# A reported space width is trusted only inside the
# band real fonts occupy — roughly 0.15em to 0.7em.
# Outside it the number is in the wrong unit (text
# space rather than page space) or the font lies, and
# believing it either welds words together or, worse,
# splits every letter of a word onto its own token.
size = float(g.get("fontSize") or g.get("h") or median or 12.0)
space_w = float(g.get("spaceWidth") or 0.0)
lo, hi = size * 0.15, size * 0.70
if lo <= space_w <= hi:
min_word_gap = space_w * 0.55
else:
min_word_gap = max(3.0, median * 0.40)
if glued_alnum:
min_word_gap = max(min_word_gap, median * 0.55)
if gap > min_word_gap:
parts.append(" ")
spans.append(
TextSpan(
text=" ",
font_size=span.font_size,
font_name=span.font_name,
x=prev_x1,
)
)
parts.append(span.text)
spans.append(span)
w = float(g.get("w", 0) or 0)
# Prefer reported width; floor so under-reported w does not inflate next gap
advance = w if w > 0.5 else max(len(span.text) * median * 0.45, median * 0.35)
# The "trailing edge" in reading order is the left edge for RTL.
prev_x1 = x0 if rtl_line else x0 + advance
text = "".join(parts).strip()
if not text:
continue
spans = coalesce_spans(spans)
xs = [float(g.get("x", 0)) for g in subgroup]
ws = [float(g.get("w", 0)) for g in subgroup]
ys = [float(g.get("y", 0)) for g in subgroup]
first = spans[0] if spans else None
lines.append(
Line(
y=sum(ys) / len(ys),
x0=min(xs),
x1=max(x + w for x, w in zip(xs, ws, strict=False)),
text=text,
font_size=float(first.font_size if first else median),
font_name=str(first.font_name if first else ""),
spans=spans,
)
)
return lines
def lines_from_plain_text(text: str, page_height: float = 792.0) -> list[Line]:
"""Fallback when only plain extract_text is available (no glyph X geometry).
The x/y values here are fabricated to keep downstream code total; they carry
no information about the page. Lines are flagged ``synthetic_geometry`` so
consumers can choose text-based heuristics over geometric ones.
"""
lines: list[Line] = []
y = page_height - 72
for raw in text.splitlines():
t = raw.strip()
if not t:
y -= 14
continue
lines.append(
Line(
y=y,
x0=72,
x1=72 + len(t) * 6,
text=t,
spans=[TextSpan(text=t)],
synthetic_geometry=True,
)
)
y -= 14
return lines
gateway/app/services/convert/layout/headers_footers.py
+876
View File
@@ -0,0 +1,876 @@
"""Header/footer band detection across pages."""
from __future__ import annotations
import re
import threading
from difflib import SequenceMatcher
from functools import lru_cache
from app.services.convert.idm.model import BlockType, Document, Page, PageKind, TextSpan
# Punctuation, whitespace and digits are exactly what varies between repeats of
# the same running header, so they are removed before comparing.
_SIGNATURE_STRIP_RE = re.compile(r"[^a-z\u0600-\u06ff]+")
# Two signatures this similar are treated as the same banner.
SIGNATURE_SIMILARITY = 0.88
# Two texts sharing this share of their words are the same banner even when
# the word order differs. Set high enough that a body sentence quoting the
# agency name does not match the banner it mentions.
TOKEN_OVERLAP_SIMILARITY = 0.72
# Threshold after confusable folding. Higher than the raw one, because folding
# has already removed the differences that were only glyph shape — what is
# left must genuinely be the same line.
FOLDED_SIMILARITY = 0.86
# Containment only applies between texts this short — a running header is a
# line, not a sentence — so body prose quoting the agency name cannot match.
BANNER_MAX_WORDS = 12
# ...and only for a signature this long, so a common short phrase inside a
# longer banner is not treated as the banner itself.
MIN_CONTAINMENT_CHARS = 16
# Below this length a signature is too generic to identify a header.
MIN_SIGNATURE_CHARS = 4
# Members of a banner family kept as comparison probes for single linkage.
MAX_LINKAGE_PROBES = 12
def _norm(text: str) -> str:
return " ".join(text.lower().split())
@lru_cache(maxsize=8192)
def _signature(text: str) -> str:
"""Comparison key robust to OCR noise and per-page variation.
Running headers rarely repeat byte-for-byte across a scanned document:
OCR renders the same banner as "FEDERAL AUTHORITY FOR IDENTITY," on one
page and "FEDERALAUTHORITY FOR IDENTITY." on the next, and footers carry a
page number that changes by design. Folding away punctuation, spacing and
digits leaves the part that actually repeats.
"""
lowered = (text or "").lower()
return _SIGNATURE_STRIP_RE.sub("", lowered)
# Glyph shapes a recogniser confuses, folded to one representative before any
# comparison. These are shape collisions, not language facts, so the same table
# works for a Latin letterhead, a Cyrillic one or a serial number.
_CONFUSABLE_MAP = str.maketrans(
{
"0": "o", "1": "l", "5": "s", "8": "b", "6": "b", "9": "g", "2": "z",
"i": "l", "j": "l", "t": "l",
"u": "v", "n": "m", "r": "m",
"e": "o", "a": "o", "c": "o", "q": "o", "d": "o",
"f": "t", "y": "v", "w": "v",
# Arabic shape collisions the recogniser makes on the same printed line.
"ص": "س", "ض": "د", "ط": "ت", "ظ": "ز",
"ة": "ه", "ى": "ي", "أ": "ا", "إ": "ا", "آ": "ا", "ؤ": "و", "ئ": "ي",
}
)
@lru_cache(maxsize=8192)
def _fold_confusables(signature: str) -> str:
"""Collapse visually confusable glyphs so OCR drift compares as itself.
``Urted Areb Emlrates`` and ``United Arab Emirates`` are the same printed
line read twice. Character-for-character they are only 80% alike, which is
below any threshold safe enough to use on raw text — but the differences
are all shape collisions a recogniser makes, and folding those first lets
the pair be recognised without loosening the threshold for genuinely
different lines.
"""
return signature.translate(_CONFUSABLE_MAP)
# Vowels, and the glyphs OCR emits when it fails. A candidate spelling with
# more of the first and fewer of the second is the better reading.
_VOWELS = frozenset("aeiou")
_JUNK_GLYPHS = frozenset("0123456789|\\/~^`{}[]<>@#*_")
def _reading_quality(text: str) -> float:
"""How *undamaged* a string looks. A tiebreak, not a spell-checker.
This ranks a reading with digits and symbols stuck through it below a
reading made of letters — ``U1t3d Ar@b Em1r@t3s`` below ``United Arab
Emirates``. It deliberately does not try to choose between two plausible
letter strings: without a dictionary, ``Urted Areb Emlrates`` and the
correct spelling are equally word-shaped, and any character statistic
claiming otherwise is inventing a signal.
Choosing between plausible readings is the majority vote's job in
:func:`_canonical_variant`; OCR errors vary from page to page while the
correct reading repeats, so the mode is the real evidence. This only breaks
ties the vote leaves open.
"""
stripped = (text or "").strip()
if not stripped:
return 0.0
letters = [ch for ch in stripped.lower() if ch.isalpha()]
if not letters:
return 0.0
junk_share = sum(1 for ch in stripped if ch in _JUNK_GLYPHS) / len(stripped)
alpha_share = len(letters) / len(stripped)
# A run of letters with no vowel at all is a failed reading in every
# alphabet that has vowels; the check is skipped where there are none.
has_vowel = any(ch in _VOWELS for ch in letters)
vowel_floor = 1.0 if (has_vowel or not letters) else 0.0
return round(0.45 * alpha_share + 0.35 * (1.0 - junk_share) + 0.20 * vowel_floor, 4)
# One matcher per right-hand string, per thread. ``SequenceMatcher`` indexes
# its second sequence on construction, and banner matching compares hundreds of
# candidate lines against the same handful of variants — so that index was
# rebuilt for every comparison. The cache is thread-local because a matcher
# carries the current left-hand sequence as state and conversions run
# concurrently in a thread pool.
_MATCHERS = threading.local()
MAX_CACHED_MATCHERS = 512
def _matcher_for(b: str) -> SequenceMatcher:
cache = getattr(_MATCHERS, "by_seq2", None)
if cache is None:
cache = {}
_MATCHERS.by_seq2 = cache
matcher = cache.get(b)
if matcher is None:
if len(cache) >= MAX_CACHED_MATCHERS:
cache.clear()
matcher = SequenceMatcher(None, "", b)
cache[b] = matcher
return matcher
def _similar(a: str, b: str) -> float:
if not a or not b:
return 0.0
return SequenceMatcher(None, a, b).ratio()
def _similar_at_least(a: str, b: str, floor: float) -> bool:
"""``_similar(a, b) >= floor``, without paying for the full comparison.
Banner grouping compares every candidate line against every variant
collected so far, and ``SequenceMatcher.ratio`` is quadratic in the string
length. On a 126-page form that came to 67,000 comparisons and 83% of the
conversion's wall-clock.
``real_quick_ratio`` and ``quick_ratio`` are difflib's own *upper bounds*
on ``ratio`` — one from the lengths alone, one from the multiset of
characters — so rejecting on them cannot change any verdict. Most pairs
are nothing alike and die on the first, which costs two calls to ``len``.
"""
if not a or not b:
return False
matcher = _matcher_for(b)
matcher.set_seq1(a)
if matcher.real_quick_ratio() < floor:
return False
if matcher.quick_ratio() < floor:
return False
return matcher.ratio() >= floor
def _tokens(text: str) -> frozenset[str]:
"""Word set of a banner, folded the same way as its signature."""
folded = (_SIGNATURE_STRIP_RE.sub("", w) for w in (text or "").lower().split())
return frozenset(t for t in folded if len(t) >= 3)
def _token_overlap(a: str, b: str) -> float:
"""Order-independent similarity between two banner texts.
A bilingual running header comes back from OCR with its halves in
different orders on different pages — "FEDERAL AUTHORITY FOR IDENTITY,
CITIZENSHIP…" on one and "CITIZENSHIP, CUSTOMS & PORT SECURITY FEDERAL
AUTHORITY FOR IDENTITY," on the next. Those share every word and almost no
character *sequence*, so ``SequenceMatcher`` scores them far apart and the
same banner splits into variants that each look too rare to be furniture.
"""
ta, tb = _tokens(a), _tokens(b)
if not ta or not tb:
return 0.0
return len(ta & tb) / len(ta | tb)
# Short letterhead stamps (a country name, a three-word agency line) drift
# further than a long banner: almost every glyph can be a near-miss, so the
# long-line folded threshold would split each page into its own family.
SHORT_BANNER_WORDS = 5
SHORT_FOLDED_SIMILARITY = 0.64
# A stamp glued into a longer cell is recognised when a window of the cell
# matches the stamp this closely after folding.
STAMP_WINDOW_SIMILARITY = 0.78
_DIGITS_RE = re.compile(r"\d+")
# An identifier is a token this many letters long or more. Below it the token
# is a folio marker — "p", "no", "pg" — not content.
IDENTIFIER_LETTERS = 4
def _varies_by_an_identifier(a: str, b: str) -> bool:
"""Whether two lines differ only in digits that belong to an identifier.
A running header may legitimately change from page to page — "Page 1 of
126" — and the signature drops digits so those pages group together. But a
*per-page identifier* also survives that stripping: "ScanTokenpack1P000"
and "ScanTokenpack1P001" have identical signatures, so a scanned document
stamped with a document, case or invoice number had all its numbers read as
one running banner, hoisted into the Word header, and every page's but the
first deleted from the body.
The difference is where the digits sit. A folio is a bare number, or a
number after a short marker word. Digits embedded in a long alphanumeric
token are an identifier, and an identifier is content.
"""
if a == b:
return False
if _DIGITS_RE.sub("", a) != _DIGITS_RE.sub("", b):
return False
ta, tb = a.split(), b.split()
if len(ta) != len(tb):
return False
for left, right in zip(ta, tb, strict=False):
if left == right:
continue
letters = sum(1 for ch in left if ch.isalpha())
if letters >= IDENTIFIER_LETTERS:
return True
return False
def _same_banner(a: str, b: str, *, allow_containment: bool = True) -> bool:
"""Whether two texts are the same running banner.
Three tests, because OCR breaks the banner in three different ways across
a scanned document: it misspells it (sequence similarity), it reorders the
halves of a bilingual line (token overlap), and it splits the banner over
two blocks on one page while gluing its words together on another
(containment).
The containment test compares signatures, which have spacing stripped, so
``FEDERALAUTHORITY FOR IDENTITY,`` is recognised inside the full
``FEDERAL AUTHORITY FOR IDENTITY, CITIZENSHIP, CUSTOMS & PORT SECURITY``.
It is applied only between two banner-length texts, so a body sentence
that quotes the agency is never swallowed by it.
"""
if _varies_by_an_identifier(a, b):
return False
sig_a, sig_b = _signature(a), _signature(b)
if _similar_at_least(sig_a, sig_b, SIGNATURE_SIMILARITY):
return True
# Heavy drift: fold the glyph shapes a recogniser confuses and compare
# again. "Urted Areb Emlrates" is 0.80 against "United Arab Emirates" raw
# — below any threshold that is safe on unfolded text — and comfortably
# above it once o/0, l/1/i and the rest stop counting as differences.
fold_a, fold_b = _fold_confusables(sig_a), _fold_confusables(sig_b)
if _similar_at_least(fold_a, fold_b, FOLDED_SIMILARITY):
return True
wa, wb = len(a.split()), len(b.split())
if (
2 <= wa <= SHORT_BANNER_WORDS
and 2 <= wb <= SHORT_BANNER_WORDS
and abs(wa - wb) <= 1
and _similar_at_least(fold_a, fold_b, SHORT_FOLDED_SIMILARITY)
):
return True
if _token_overlap(a, b) >= TOKEN_OVERLAP_SIMILARITY:
return True
if not allow_containment:
# Containment is the one relation that legitimately joins *different*
# banner lines: a page that read the whole two-line banner as one block
# contains each line. Chaining families through it would merge the
# lines into a single family and lose one of them, so grouping asks for
# the strict relations only.
return False
if len(a.split()) > BANNER_MAX_WORDS or len(b.split()) > BANNER_MAX_WORDS:
return False
short, long_ = sorted((sig_a, sig_b), key=len)
if not (len(short) >= MIN_CONTAINMENT_CHARS and short in long_):
return False
# Containment is for a banner split vs glued, not for a journal title that
# sits inside a longer issue line ("Perspectives" inside "Number 4, Winter…").
if _containment_has_distinct_payload(a, b):
return False
return True
_ISSUE_LINE_RE = re.compile(
r"\b(?:vol(?:ume)?|no\.?|number|issue|pp\.?)\b|\d{2,}",
re.IGNORECASE,
)
def _containment_has_distinct_payload(a: str, b: str) -> bool:
"""True when the longer string carries a different line, not a fuller OCR."""
wa, wb = a.split(), b.split()
if abs(len(wa) - len(wb)) <= 2:
return False
short_w, long_w = (wa, wb) if len(wa) <= len(wb) else (wb, wa)
short_set = {w.lower() for w in short_w}
extra = [w for w in long_w if w.lower() not in short_set]
extra_joined = " ".join(extra)
return bool(_ISSUE_LINE_RE.search(extra_joined))
# A signature must appear on at least this share of pages to count as running
# furniture. Two pages is never enough on a long document, and requiring every
# page is too strict for scans where OCR loses the banner occasionally.
MIN_BAND_PAGE_SHARE = 0.25
# The band is never shallower than this many text lines, so a two- or
# three-line banner is not cut in half on a sparse page.
BAND_MIN_LINES = 3.0
def _page_bands(page, body_blocks: list, band_ratio: float) -> tuple[list, list]:
"""Top and bottom band blocks, in whatever coordinate system the page uses.
Bands are measured against the *observed* extent of the page's own blocks
rather than ``page.height``. OCR-rebuilt pages carry pixel-derived
coordinates (negative y, hundreds of units tall) that share no origin or
scale with the PDF point height, so comparing them against ``page.height``
classified every block on every scanned page as "bottom band" and never
found the banner at all.
"""
tops = [b.bbox.y + b.bbox.h for b in body_blocks]
bots = [b.bbox.y for b in body_blocks]
hi, lo = max(tops), min(bots)
span = hi - lo
if span <= 0:
# Degenerate geometry — fall back to reading order.
ordered = sorted(body_blocks, key=lambda b: b.reading_order)
return ordered[:1], ordered[-1:]
# A running header is a couple of lines, so the band is at least that deep
# however short the page is. Using the span fraction alone made the band
# narrower than the banner itself on sparse pages — a title page with six
# blocks — and caught only the banner's first line.
heights = sorted(b.bbox.h for b in body_blocks if b.bbox.h > 0)
line_h = heights[len(heights) // 2] if heights else 0.0
depth = max(span * band_ratio, line_h * BAND_MIN_LINES)
top_band = [b for b in body_blocks if b.bbox.y + b.bbox.h >= hi - depth]
bot_band = [b for b in body_blocks if b.bbox.y <= lo + depth]
return top_band, bot_band
def _repeating_band_variants(per_page: list[list[str]], page_count: int) -> list[str]:
"""Representative texts of every banner that repeats across enough pages.
Counted once per page, so a banner split into two blocks on one page does
not out-vote a banner that appears on twenty. The return value is a list of
*representatives*, not an exhaustive set of exact strings: callers compare
against them with :func:`_same_banner`, so a page whose spelling drifted
beyond anything already collected is still recognised.
"""
pages_with_text = [texts for texts in per_page if texts]
if len(pages_with_text) < 2:
return []
groups: list[dict] = []
# A running header usually repeats byte-for-byte. Resolving each distinct
# string once and remembering where it landed turns 126 pages of
# comparisons into one, which is most of the cost of this pass on a long
# document.
resolved: dict[str, dict] = {}
for page_i, texts in enumerate(pages_with_text):
for text in dict.fromkeys(texts):
if len(_signature(text)) < MIN_SIGNATURE_CHARS:
continue
known = resolved.get(text)
if known is not None:
known["pages"].add(page_i)
continue
# Single linkage: a variant joins a family if it matches *any*
# member, not just the first one seen. OCR drift forms a chain —
# a badly damaged reading can be far from the clean spelling while
# sitting close to a middling one — and comparing only against a
# fixed representative split one banner into several families, each
# too rare on its own to clear the page-share threshold.
group = next(
(
g
for g in groups
if any(
_same_banner(v, text, allow_containment=False)
for v in g["probes"]
)
or _same_banner(g["text"], text)
),
None,
)
if group is None:
group = {
"text": text,
"pages": {page_i},
"variants": [text],
"probes": [text],
}
groups.append(group)
else:
group["pages"].add(page_i)
group["variants"].append(text)
# Chaining needs several members to link through, but not all
# of them: past a handful the extra probes are near-duplicates
# of ones the family already holds and only cost comparisons.
if len(group["probes"]) < MAX_LINKAGE_PROBES:
group["probes"].append(text)
resolved[text] = group
threshold = max(2, round(len(pages_with_text) * MIN_BAND_PAGE_SHARE))
return [
_canonical_variant(g["variants"]) for g in groups if len(g["pages"]) >= threshold
]
def _canonical_variant(variants: list[str]) -> str:
"""The cleanest spelling of a banner that OCR read differently each page.
A recogniser reading the same printed line on twenty pages returns twenty
near-misses — a zero for an O, an l for an I, a stray full stop. Taking
whichever one happened to come first put page one's mistakes into the Word
header of the whole document. Language-like spellings beat letter-salad
even when the salad repeats; among equally clean readings the majority
(then the shorter running title) wins.
"""
if not variants:
return ""
counts: dict[str, int] = {}
for text in variants:
counts[text] = counts.get(text, 0) + 1
from app.services.convert.layout.text_quality import ocr_text_quality_score
return max(
counts,
key=lambda t: (
ocr_text_quality_score([t]),
counts[t],
-len(t.split()),
-len(t),
),
)
# A bare number, "Page 4", "4 of 12", "- 4 -". Page numbers differ on every
# page by definition, so they never group as a repeating banner and were left
# behind in the body as orphans, one stray digit per page.
_PAGE_NUMBER_RE = re.compile(
r"^(?:[-–—(\[]?\s*)"
r"(?:(?:page|pg|p\.?|seite|página|صفحة)\s*)?"
r"\d{1,4}"
r"(?:\s*(?:/|of|de|von|من)\s*\d{1,4})?"
r"(?:\s*[-–—)\]]?)$",
re.I,
)
# Longer than this and it is a sentence that happens to start with a number.
PAGE_NUMBER_MAX_CHARS = 24
def looks_like_page_number(text: str) -> bool:
"""Whether a line is a page-number stamp rather than content.
Deliberately shape-based: a short line that is essentially a number, with
an optional label and an optional "of N". The few label words listed are a
convenience, not the test — a bare "7" is recognised in any language.
"""
stripped = (text or "").strip()
if not stripped or len(stripped) > PAGE_NUMBER_MAX_CHARS:
return False
if not any(ch.isdigit() for ch in stripped):
return False
return bool(_PAGE_NUMBER_RE.match(stripped))
def tag_headers_footers(
document: Document,
*,
band_ratio: float = 0.12,
header_bboxes: list | None = None,
footer_bboxes: list | None = None,
) -> None:
"""
Detect repeating top/bottom text across ≥2 pages and retag those blocks
as header/footer. Prefer high/low bbox Y when available.
Optional ML header/footer bboxes boost single-page retag.
"""
# ML hints: retag blocks whose centers fall in hint boxes (works on 1-page docs too)
if header_bboxes or footer_bboxes:
for page in document.pages:
for b in page.blocks:
if not b.plain_text().strip() or b.type in (BlockType.table, BlockType.figure):
continue
cx = b.bbox.x + b.bbox.w / 2
cy = b.bbox.y + b.bbox.h / 2
for hb in header_bboxes or []:
if hb.x <= cx <= hb.x + hb.w and hb.y <= cy <= hb.y + hb.h:
b.type = BlockType.header
break
for fb in footer_bboxes or []:
if fb.x <= cx <= fb.x + fb.w and fb.y <= cy <= fb.y + fb.h:
if b.type != BlockType.header:
b.type = BlockType.footer
break
if document.page_count < 2:
return
top_texts: list[list[str]] = []
bottom_texts: list[list[str]] = []
bands: dict[int, tuple[list, list]] = {}
for page in document.pages:
body_blocks = [b for b in page.blocks if b.plain_text().strip()]
if not body_blocks:
continue
top_band, bot_band = _page_bands(page, body_blocks, band_ratio)
bands[page.index] = (top_band, bot_band)
# Every block in the band is a candidate, not just the outermost one.
# A running banner is a *region*: this document repeats an agency line
# and "United Arab Emirates" beneath it, and taking one block per page
# left the second line in the body on all 26 pages.
top_texts.append([_norm(b.plain_text()) for b in top_band])
bottom_texts.append([_norm(b.plain_text()) for b in bot_band])
header_variants = _repeating_band_variants(top_texts, document.page_count)
footer_variants = _repeating_band_variants(bottom_texts, document.page_count)
# A doc can repeat the same line top and bottom; prefer header.
footer_variants = [
f for f in footer_variants if not any(_same_banner(f, h) for h in header_variants)
]
numbered = False
for page in document.pages:
top_band, bot_band = bands.get(page.index, ([], []))
_retag_page(page, header_variants, footer_variants, top_band, bot_band)
numbered |= _tag_page_numbers(bot_band)
# Once a family is established, strip it wherever it appears — the cover
# page, a mid-document divider, a cell. A variant that shows up on only two
# pages never clears the page-share threshold on its own, but it is still
# the same letterhead, and leaving it in the body is how a country line
# ended up as the document's opening paragraph.
_retag_stragglers(document, header_variants, footer_variants)
furniture = header_variants + footer_variants
# Stamp trimming searches every cell and paragraph for a fuzzy substring of
# the banner. It exists because a recogniser reads the letterhead and the
# first clause as one line; a digital text layer never does that — the
# banner is its own text object at its own position, and retagging has
# already moved it. Running the search anyway cost 4s of the 8s spent on
# four pages of a tax table, comparing "1,234" against the running head
# thousands of times.
if _has_recognised_text(document):
_strip_banners_from_tables(document, furniture)
_strip_stamps_from_prose(document, furniture)
if numbered:
# The writer emits one PAGE field instead of the literal numbers, which
# is both correct in Word and the only way a per-page stamp can live in
# a single header definition.
meta = document.meta if document.meta is not None else {}
meta["page_number_footer"] = True
document.meta = meta
# A straggler is only retagged if it is short enough to be furniture. Body
# prose that quotes the organisation is longer than a banner line.
STRAGGLER_MAX_WORDS = 12
# ...and it must sit within this share of the page's own vertical extent,
# measured from whichever end the family belongs to.
STRAGGLER_BAND_SHARE = 0.30
def _retag_stragglers(
document: Document, header_variants: list[str], footer_variants: list[str]
) -> None:
"""Retag banner-family lines that fell outside a detected band.
The band vote answers "does this repeat enough to be furniture". Once it
has answered yes, every other appearance of the same family is furniture
too, wherever it sits: a cover page puts the letterhead lower than a body
page does, and a divider page puts it alone in the middle. Those copies
never joined the vote and so stayed in the body — the first paragraph of
the document being a misread country line is exactly this.
"""
if not header_variants and not footer_variants:
return
for page in document.pages:
candidates = [
b
for b in page.blocks
if b.plain_text().strip()
and b.type not in (BlockType.table, BlockType.figure)
and not b.image_png
]
if not candidates:
continue
tops = [b.bbox.y + b.bbox.h for b in candidates]
lo, hi = min(b.bbox.y for b in candidates), max(tops)
span = max(hi - lo, 1.0)
for block in candidates:
if block.type in (BlockType.header, BlockType.footer):
continue
text = block.plain_text().strip()
if len(text.split()) > STRAGGLER_MAX_WORDS:
continue
# Position decides as much as wording. A line that repeats the
# banner but sits in the middle of the page is a mention — the
# document referring to its own author — while the same words at
# the top of a cover page are the letterhead that the band vote
# simply did not see enough times to count.
near_top = (block.bbox.y + block.bbox.h - lo) / span >= 1.0 - STRAGGLER_BAND_SHARE
near_bottom = (block.bbox.y - lo) / span <= STRAGGLER_BAND_SHARE
# OCR pages often have unusable Y, so a letterhead line in the
# "middle" of the reconstructed page is still furniture.
scanish = page.kind in (PageKind.scan, PageKind.hybrid) or any(
getattr(b, "source", "") == "ocr" for b in page.blocks
)
if (scanish or near_top) and any(_same_banner(text, h) for h in header_variants):
block.type = BlockType.header
block.level = 0
elif near_bottom and any(_same_banner(text, f) for f in footer_variants):
block.type = BlockType.footer
block.level = 0
def _tag_page_numbers(bot_band: list) -> bool:
"""Move page-number stamps out of the body and into the footer."""
found = False
for block in bot_band or []:
if block.type in (BlockType.table, BlockType.figure, BlockType.header):
continue
if looks_like_page_number(block.plain_text()):
block.type = BlockType.footer
block.level = 0
found = True
return found
def _shared_character_budget(needle: str, haystack: str) -> int:
"""Size of the multiset intersection of two strings.
No window of *haystack* can match *needle* more closely than the share of
needle's characters that appear in haystack at all — a bound that costs one
pass over each string, against the hundreds of full comparisons the sliding
search would otherwise run.
"""
counts: dict[str, int] = {}
for ch in needle:
counts[ch] = counts.get(ch, 0) + 1
shared = 0
for ch in haystack:
left = counts.get(ch, 0)
if left:
counts[ch] = left - 1
shared += 1
return shared
@lru_cache(maxsize=4096)
def _folded_window_match(needle: str, haystack: str) -> bool:
"""Whether ``needle`` appears inside ``haystack`` after OCR-shape folding.
Used to find a short running stamp that was concatenated onto a real cell
("Capability Area" + a misspelt country line). Whole-string equality fails
because of the extra words; an un-folded substring fails because of the
misspelling.
The sliding search is the most expensive comparison in the package — six
window widths across every offset, each a fresh ``SequenceMatcher`` — so it
is entered only after a character-budget check rules out the overwhelming
majority of pairs, and the matcher's index over the needle is built once
and reused across every window.
"""
n = _fold_confusables(_signature(needle))
h = _fold_confusables(_signature(haystack))
if len(n) < 10 or not h:
return False
if n in h:
return True
if _shared_character_budget(n, h) < STAMP_WINDOW_SIMILARITY * len(n):
return False
lo = max(8, len(n) - 2)
hi = min(len(h), len(n) + 3)
matcher = SequenceMatcher(None, "", n)
for width in range(lo, hi + 1):
if width > len(h):
break
for i in range(0, len(h) - width + 1):
matcher.set_seq1(h[i : i + width])
if matcher.real_quick_ratio() < STAMP_WINDOW_SIMILARITY:
continue
if matcher.quick_ratio() < STAMP_WINDOW_SIMILARITY:
continue
if matcher.ratio() >= STAMP_WINDOW_SIMILARITY:
return True
return False
def _trim_stamp_from_text(text: str, variants: list[str]) -> str:
"""Drop a running stamp glued onto a cell or a body line; keep the rest."""
raw = (text or "").strip()
if not raw:
return raw
words = raw.split()
if len(words) <= SHORT_BANNER_WORDS and any(_same_banner(raw, v) for v in variants):
return ""
bullets = {"·", "•", "-", "—", "*", "|"}
for v in variants:
vn = len(v.split())
if vn < 2:
continue
for n in (vn, vn + 1, vn - 1):
if n < 2 or n > len(words):
continue
for i in range(0, len(words) - n + 1):
window = " ".join(words[i : i + n])
if not (_same_banner(window, v) or _folded_window_match(v, window)):
continue
at_edge = i == 0 or i + n == len(words)
next_to_mark = (i > 0 and words[i - 1] in bullets) or (
i + n < len(words) and words[i + n] in bullets
)
if not (at_edge or next_to_mark):
continue
kept = words[:i] + words[i + n :]
return " ".join(kept).strip()
return raw
def _has_recognised_text(document: Document) -> bool:
"""Whether any of this document's text came from OCR rather than the PDF."""
for page in document.pages:
if page.kind in (PageKind.scan, PageKind.hybrid):
return True
for block in page.blocks:
if getattr(block, "source", "") == "ocr":
return True
return False
def _strip_banners_from_tables(document: Document, variants: list[str]) -> None:
"""Remove running-banner rows that were swallowed into a grid.
On a scan the banner and the top of a table sit close together, and grid
assembly sometimes takes the banner in as a first row. Retagging skips
tables, so the banner then appeared in the Word header *and* as the table's
header row — the same text twice, one of them wrong.
"""
if not variants:
return
banner_only: list = []
for page in document.pages:
for block in page.blocks:
if block.type != BlockType.table or not block.cells:
continue
rows = []
for row in block.cells:
cleaned = [_trim_stamp_from_text(c or "", variants) for c in row]
joined = " ".join(c for c in cleaned if (c or "").strip()).strip()
if not joined:
continue
if any(_same_banner(joined, v) for v in variants):
continue
rows.append(cleaned)
if rows == block.cells:
continue
if rows:
block.cells = rows
continue
# Every row was banner. This is not a table with a banner in it —
# it is the banner, mistaken for a table, and emitting it as a
# one-row grid puts the letterhead in the document twice, once as
# furniture and once as a box around nothing.
block.cells = []
block.type = BlockType.paragraph
block.text = ""
block.spans = []
banner_only.append(block)
for page in document.pages:
page.blocks = [b for b in page.blocks if id(b) not in {id(x) for x in banner_only}]
def _strip_stamps_from_prose(document: Document, variants: list[str]) -> None:
"""Trim a running stamp that was concatenated onto a body paragraph.
Tables are handled separately. Requirement lines often carry the letterhead
in the same OCR line as the clause; retagging cannot move that line into
the header without losing the clause.
"""
if not variants:
return
for page in document.pages:
for block in page.blocks:
if block.type in (BlockType.table, BlockType.figure, BlockType.header, BlockType.footer):
continue
raw = block.plain_text().strip()
if not raw:
continue
cleaned = _trim_stamp_from_text(raw, variants)
if cleaned == raw:
continue
block.text = cleaned
if block.spans:
sample = block.spans[0]
block.spans = [
TextSpan(
text=cleaned,
font_name=sample.font_name,
font_size=sample.font_size,
bold=sample.bold,
italic=sample.italic,
)
]
def _retag_page(
page: Page,
header_variants: list[str],
footer_variants: list[str],
top_band: list | None = None,
bot_band: list | None = None,
) -> None:
"""Retag banner blocks inside the page's top/bottom band.
Matching within the band rather than walking a leading run matters for two
real cases in scanned documents:
* a masthead logo sits above the banner, and a run-walk stops dead at the
figure — leaving every banner line in the body of page one;
* on a dense page the banner is not the first block in reading order, so a
run starting at index 0 never reaches it.
The band already confines the search to the top or bottom of the page, and
only text matching a document-wide banner is retagged, so body prose that
merely mentions the agency keeps its place.
"""
if top_band is None or bot_band is None:
ordered = sorted(
[b for b in page.blocks if b.plain_text().strip()],
key=lambda b: b.reading_order,
)
top_band = top_band if top_band is not None else ordered[:1]
bot_band = bot_band if bot_band is not None else ordered[-1:]
skip = (BlockType.table, BlockType.figure)
tagged: set[int] = set()
for block in top_band:
if block.type in skip or not header_variants:
continue
if any(_same_banner(block.plain_text(), h) for h in header_variants):
block.type = BlockType.header
block.level = 0
tagged.add(id(block))
for block in bot_band:
if block.type in skip or id(block) in tagged or not footer_variants:
continue
if any(_same_banner(block.plain_text(), f) for f in footer_variants):
block.type = BlockType.footer
block.level = 0
gateway/app/services/convert/layout/idm_optimize.py
+982
View File
@@ -0,0 +1,982 @@
"""Post-IDM optimize: ConvertAPI/Aspose reconstruction modes (in-house)."""
from __future__ import annotations
import re
from dataclasses import replace
from app.services.convert.idm.model import (
Block,
BlockType,
Document,
Page,
PageKind,
TextSpan,
)
from app.services.convert.layout.paragraphs import SHORT_LINE_RATIO, SIZE_TOLERANCE
from app.services.convert.layout.styles import (
dominant_body_size,
split_glued_section_number,
)
from app.services.convert.options import (
LayoutMode,
RecognitionMode,
get_options,
)
# If more than this fraction of body blocks are headings, demote body-sized ones.
HEADING_DENSITY_CAP = 0.35
# A table block carrying at least this confidence was assembled from measured
# column alignment, so it is a detected region and is never soup-flattened.
DETECTED_TABLE_CONFIDENCE = 0.7
# Continuous mode: merge adjacent paragraphs whose font sizes are this close.
# Only ever used when the caller explicitly asks for continuous layout; the
# default flowing mode uses ``paragraphs.SIZE_TOLERANCE`` so that this pass and
# ``breaks_paragraph`` agree on what counts as the same size. At 0.22 this value
# treated 11pt body and a 12pt heading as one size and welded them.
CONTINUOUS_SIZE_TOL = 0.22
# Blocks whose left edges fall within this many points are treated as one
# column when measuring where that column's right margin is.
_WRAP_COLUMN_SNAP = 24.0
# How far right of ``prev`` a continuation block's left edge may sit and still be
# the next line of the same paragraph. Wrapped lines share a left edge; this is
# only slack for glyph-extent rounding.
_WRAP_LEFT_TOL = 3.0
def _block_size(block: Block) -> float:
"""The size this block is *set* in: its dominant size, weighted by characters.
A plain mean over spans lets one large run misreport the whole block. The
fixture's character-formatting line is 11pt prose containing a single 22pt
word, which averaged to 12.6pt -- close enough to the 12pt heading beneath it
that the two were judged the same size and welded together. Weighting by
character count answers the question actually being asked, which is what size
the reader sees this block as.
"""
if block.spans:
weights: dict[float, int] = {}
for s in block.spans:
size = s.font_size
if not size or size <= 0:
continue
n = len((s.text or "").strip())
if not n:
continue
weights[round(size * 2) / 2.0] = weights.get(round(size * 2) / 2.0, 0) + n
if weights:
return max(weights.items(), key=lambda kv: (kv[1], kv[0]))[0]
sizes = [s.font_size for s in block.spans if s.font_size and s.font_size > 0]
if sizes:
return sum(sizes) / len(sizes)
return 12.0
def _block_font(block: Block) -> str:
"""Dominant font family of a block, normalised for comparison.
Weight and slant suffixes are stripped so ``Calibri`` and ``Calibri-Bold``
read as one family: a bold phrase inside a sentence must not split the
paragraph. ``Cambria`` and ``Calibri`` stay distinct. Normalisation matches
``layout.paragraphs._style_changed`` so the two passes cannot disagree.
"""
counts: dict[str, int] = {}
for span in block.spans or []:
text = span.text or ""
if not text.strip():
continue
name = (span.font_name or "").split("+")[-1].split("-")[0].strip().lower()
if name:
counts[name] = counts.get(name, 0) + len(text)
return max(counts, key=lambda key: counts[key]) if counts else ""
def _is_bodyish(block: Block) -> bool:
return block.type in (
BlockType.paragraph,
BlockType.heading,
BlockType.list_item,
) and bool(block.plain_text().strip())
def _latin_label_row(row: list[str]) -> bool:
texts = [c.strip() for c in row if c and str(c).strip()]
if not (2 <= len(texts) <= 4):
return False
if any(any(ch.isdigit() for ch in t) for t in texts):
return False
if any(len(t.split()) > 4 or len(t) >= 48 for t in texts):
return False
if any("\u0600" <= ch <= "\u06FF" for t in texts for ch in t):
return False
return True
def _row_is_empty_or_arabic_masthead(row: list[str]) -> bool:
texts = [c.strip() for c in row if c and str(c).strip()]
if not texts:
return True
ar = sum(1 for t in texts for ch in t if "\u0600" <= ch <= "\u06FF")
latin = sum(1 for t in texts for ch in t if ch.isascii() and ch.isalpha())
return ar >= 8 and ar >= latin
def _table_is_mid_sentence_fragment(cells: list[list[str]] | None) -> bool:
"""OCR leftover of a larger grid: starts mid-phrase or with a hyphen break."""
rows = cells or []
if not (1 <= len(rows) <= 4):
return False
ncols = max((len(r) for r in rows), default=0)
if ncols > 4 or ncols < 2:
return False
first_row = [c.strip() for c in rows[0] if c and str(c).strip()]
if not first_row:
return False
# Real tables start with short title-case headers, not "in structured 2-".
if all(
1 <= len(t.split()) <= 4
and t[:1].isupper()
and not t.endswith("-")
for t in first_row
):
return False
lead = first_row[0]
if lead[:1].islower() and lead[:1].isascii():
return True
return any(t.endswith("-") for t in first_row)
def _table_is_stamp_remnant(cells: list[list[str]] | None) -> bool:
"""Tiny grids that are only leftover header stamps, not data tables."""
rows = cells or []
if not (1 <= len(rows) <= 2):
return False
ncols = max((len(r) for r in rows), default=0)
if ncols > 4:
return False
if len(rows) == 2 and ncols > 3:
return False
if (
len(rows) == 2
and _latin_label_row(rows[0])
and _row_is_empty_or_arabic_masthead(rows[1])
):
return True
texts = [c.strip() for row in rows for c in row if c and str(c).strip()]
if not texts:
return True
if any(ch.isdigit() for t in texts for ch in t):
return False
unique = {t.lower() for t in texts}
max_words = max(len(t.split()) for t in texts)
if not (
len(unique) <= 4
and max_words <= 4
and all(len(t) < 48 for t in texts)
):
return False
# Stamp leftover: the same labels repeat. A 2×N of distinct cells
# (two-column body, Name/Qty/Cost + data) must stay.
if len(rows) == 2:
r0 = [str(c).strip().lower() for c in rows[0] if c and str(c).strip()]
r1 = [str(c).strip().lower() for c in rows[1] if c and str(c).strip()]
if r0 and r0 == r1:
return True
return len(unique) <= max(1, len(texts) // 2)
lows = [t.lower().replace(" ", "") for t in texts]
smashed = any(
i != j and b in a and a != b and len(b) >= 6
for i, a in enumerate(lows)
for j, b in enumerate(lows)
)
return len(unique) <= 3 or smashed
_VALUE_LIKE = re.compile(r"^[\s(\[]*[-+±]?[$€£¥₹]?\s*\d[\d,. :/–—-]*\s*[%°]?\s*[)\]]?$")
def _is_value_like(text: str) -> bool:
"""A number, amount, date, range or percentage — a cell, not a word.
``ocr_text_quality_score`` measures how much a string reads like language.
A tax table's "5,600" reads like nothing at all and scores near zero, so a
grid of numbers looked exactly like recogniser salad and was deleted:
64 rows of the earned-income tables, every financial statement, every
price list. Values have to be recognised as legitimate before that
judgement is made.
"""
stripped = (text or "").strip()
if not stripped:
return False
return bool(_VALUE_LIKE.match(stripped)) and any(ch.isdigit() for ch in stripped)
def _table_cells_are_ocr_junk(cells: list[list[str]] | None) -> bool:
"""A grid whose cells are almost all cmap-salad is not a table to keep.
Flattening it would dump the same salad into body paragraphs. Drop it.
"""
from app.services.convert.layout.text_quality import (
is_cmap_garbled_line,
is_ocr_noise_line,
ocr_text_quality_score,
)
texts = [c.strip() for row in (cells or []) for c in row if c and str(c).strip()]
if not texts:
return True
bad = 0
for t in texts:
if _is_value_like(t):
continue
if is_ocr_noise_line(t) or is_cmap_garbled_line(t) or ocr_text_quality_score([t]) <= 0.15:
bad += 1
return (bad / len(texts)) >= 0.75
def flatten_table_to_paragraphs(block: Block) -> list[Block]:
"""Turn a rejected/soup table into flowing paragraphs (EnhancedFlow)."""
out: list[Block] = []
order = block.reading_order
for row in block.cells or []:
text = " ".join(c.strip() for c in row if c and c.strip()).strip()
if not text:
continue
out.append(
Block(
type=BlockType.paragraph,
text=text,
spans=[TextSpan(text=text)],
bbox=block.bbox,
reading_order=order,
source=block.source,
)
)
order += 1
return out or [
Block(
type=BlockType.paragraph,
text=block.plain_text(),
spans=block.spans or [TextSpan(text=block.plain_text())],
bbox=block.bbox,
reading_order=block.reading_order,
source=block.source,
)
]
def _flatten_soup_tables(document: Document) -> None:
"""OCR word-soup grids → paragraphs. Digital real grids stay tables."""
from app.services.convert.layout.tables import (
is_fragmented_prose_grid,
is_logo_or_wordmark_grid,
)
for page in document.pages:
rebuilt: list[Block] = []
for block in page.blocks:
if block.type != BlockType.table or not block.cells:
rebuilt.append(block)
continue
soup = is_fragmented_prose_grid(block.cells)
logo = is_logo_or_wordmark_grid(block.cells)
ocr_only = (block.source or "") == "ocr"
# A grid the detector assembled from aligned column positions is a
# *detected table region*, not word soup, whatever its cells read
# like. Word-soup flattening exists for the opposite case — prose
# that only looked like a grid — and must never undo a real
# detection: a two-column scanned table of short entries is
# textually indistinguishable from soup and is not soup.
detected = float(getattr(block, "table_confidence", 0.0) or 0.0) >= DETECTED_TABLE_CONFIDENCE
if _table_cells_are_ocr_junk(block.cells):
continue
if logo or (soup and ocr_only and not detected):
rebuilt.extend(flatten_table_to_paragraphs(block))
else:
rebuilt.append(block)
for i, b in enumerate(rebuilt):
b.reading_order = i
page.blocks = rebuilt
def _drop_all_tables(document: Document) -> None:
for page in document.pages:
rebuilt: list[Block] = []
for block in page.blocks:
if block.type == BlockType.table and block.cells:
rebuilt.extend(flatten_table_to_paragraphs(block))
else:
rebuilt.append(block)
for i, b in enumerate(rebuilt):
b.reading_order = i
page.blocks = rebuilt
_SENTENCE_END = ".!?۔؟:;،"
def _reads_as_wrapped(prev_text: str, next_text: str) -> bool:
"""Whether two lines are one sentence split across a line break.
The only merge evidence available when a page has no geometry. A wrapped
line leaves its sentence unfinished and the next line continues it in
lower case (or in a script with no case, continues without a bullet or
number). Table cells and list items fail both halves.
"""
prev_text, next_text = prev_text.strip(), next_text.strip()
if not prev_text or not next_text:
return False
if prev_text[-1] in _SENTENCE_END:
return False
if prev_text.endswith("-"):
return True
head = next_text[0]
if head.isdigit() or head in "•-*–—([":
return False
# Cased scripts: a capital starts a new line, not a continuation.
if head.isupper():
return False
# A single bare token on either side is a cell or label, not prose.
return len(prev_text.split()) > 1 and len(next_text.split()) > 1
def _reached_wrap_margin(blocks: list[Block], prev: Block) -> bool:
"""Whether ``prev`` runs to the right margin of the column it sits in.
A line that wrapped ends because it ran out of room, so it reaches the
margin; a line the author ended stops short of it. That difference is the
only evidence that two consecutive one-line blocks are really one paragraph,
and it is the same signal ``paragraphs.SHORT_LINE_RATIO`` uses -- shared
deliberately, so this pass and ``breaks_paragraph`` cannot reach opposite
conclusions about the same pair of lines.
The margin is measured from blocks sharing ``prev``'s left edge rather than
from the whole page. Measuring page-wide would put every left-column line of
a two-column layout far short of the right column's margin and block every
legitimate merge there.
When too few peers exist to measure, the answer is ``False`` -- do not merge.
Being alone at a left edge is not missing information, it is information: a
block that no other block lines up with is not part of a column of prose, so
it is a centred line, a right-aligned line or a one-off inset, and none of
those wrap into the block below. The opposite default let a centred line
swallow the right-aligned line beneath it, since neither had peers.
``prev`` is excluded from the vote. A line cannot be evidence for its own
margin: when it was the widest of its peers it set the margin it was then
measured against, so the test returned "reached the margin" unconditionally.
That is how a right-aligned line -- necessarily the widest thing at its own
left edge -- absorbed the paragraph beneath it.
"""
x0 = prev.bbox.x if prev.bbox else None
if x0 is None:
return False
peers = [
b
for b in blocks
if b is not prev
and b.type == BlockType.paragraph
and b.bbox
and b.bbox.w
and not b.synthetic_geometry
and abs((b.bbox.x or 0.0) - x0) <= _WRAP_COLUMN_SNAP
]
if len(peers) < 2:
return False
rights = sorted((b.bbox.x or 0.0) + (b.bbox.w or 0.0) for b in peers)
col_right = rights[min(len(rights) - 1, int(len(rights) * 0.90))]
col_width = max(col_right - x0, 1.0)
prev_right = (prev.bbox.x or 0.0) + (prev.bbox.w or 0.0)
return (col_right - prev_right) <= col_width * SHORT_LINE_RATIO
def _coalesce_short_line_paragraphs(page: Page, *, aggressive: bool) -> None:
blocks = sorted(page.blocks, key=lambda b: b.reading_order)
if len(blocks) < 2:
return
out: list[Block] = []
i = 0
while i < len(blocks):
cur = blocks[i]
if cur.type != BlockType.paragraph:
out.append(cur)
i += 1
continue
run = [cur]
i += 1
while i < len(blocks):
nxt = blocks[i]
if nxt.type != BlockType.paragraph:
break
prev = run[-1]
prev_t = prev.plain_text().strip()
nxt_t = nxt.plain_text().strip()
if not prev_t or not nxt_t:
break
# Don't glue a new sentence that already looks complete+complete
# unless continuous mode (harder merge).
size_ok = abs(_block_size(prev) - _block_size(nxt)) / max(
_block_size(prev), 1.0
) <= (CONTINUOUS_SIZE_TOL if aggressive else SIZE_TOLERANCE)
# A change of typeface is a change of role. Without this the pass
# undid the geometry path's own decision: ``breaks_paragraph``
# splits on a font change, and this loop then welded the halves
# back together, so a bold Cambria heading and the Calibri line
# under it reached Word as a single run-on paragraph. Size alone
# cannot catch it -- a 14pt heading above 16pt body is within
# CONTINUOUS_SIZE_TOL, and the heading is the *smaller* of the two.
prev_font, nxt_font = _block_font(prev), _block_font(nxt)
font_ok = not (prev_font and nxt_font and prev_font != nxt_font)
short_prev = len(prev_t.split()) <= (28 if aggressive else 14)
short_nxt = len(nxt_t.split()) <= (28 if aggressive else 16)
has_geometry = bool(
prev.bbox
and nxt.bbox
and prev.bbox.h
and nxt.bbox.h
and not prev.synthetic_geometry
and not nxt.synthetic_geometry
)
if has_geometry:
dy = abs(prev.bbox.y - nxt.bbox.y)
y_close = dy < max(prev.bbox.h, nxt.bbox.h, 12) * (3.2 if aggressive else 2.2)
# Wrapped lines of one paragraph share a left edge. The old
# tolerance was max(80pt, prev_width * 1.4) -- on a 310pt-wide
# block that permitted a 434pt jump, so a centred line and the
# right-aligned line under it were welded despite starting 94pt
# apart. A first-line indent moves the *second* line left, never
# right, so a rightward jump beyond a couple of points means the
# two blocks are not one wrapping paragraph.
nxt_x = nxt.bbox.x or 0.0
prev_x = prev.bbox.x or 0.0
if nxt_x - prev_x > _WRAP_LEFT_TOL or prev_x - nxt_x > max(
80.0, (prev.bbox.w or 40) * 1.4
):
break
# Proximity alone is not wrap evidence, and treating it as such
# is what made this pass overturn the geometry path wholesale.
# ``breaks_paragraph`` separates a line that stops short of its
# column margin, because that is what the *last* line of a
# paragraph looks like; this loop then rejoined it to whatever
# followed within ~2 line heights, which on any document written
# one line per paragraph is everything. A 7-line test page came
# back as 3 run-on blobs grouped only by point size, and the
# heading among them was absorbed and lost. Requiring the line
# to have actually reached its margin is the same wrap signal
# ``paragraphs.SHORT_LINE_RATIO`` encodes, so the two passes now
# agree instead of cancelling out.
if not aggressive and not _reached_wrap_margin(blocks, prev):
break
else:
# No geometry at all — the page came through the plain-text
# fallback, where every extracted token lands on its own
# fabricated line. Absence of geometry is not evidence that two
# lines sit close together, and treating it as such merged a
# whole ruled table ("Name / Qty / Cost / Alpha / 3 / 9") into
# one paragraph. With nothing to measure, only a textual wrap
# signal justifies joining: the previous line must not finish a
# sentence and the next must continue one.
y_close = _reads_as_wrapped(prev_t, nxt_t)
if not (size_ok and font_ok and short_prev and short_nxt and y_close):
break
if (not aggressive) and prev_t[-1:] in ".!?۔؟" and nxt_t[:1].isupper():
break
run.append(nxt)
i += 1
if len(run) == 1:
out.append(run[0])
continue
texts = [b.plain_text().strip() for b in run]
joined = " ".join(texts)
spans: list[TextSpan] = []
for j, b in enumerate(run):
if j and spans:
spans.append(TextSpan(text=" "))
if b.spans:
spans.extend(b.spans)
else:
spans.append(TextSpan(text=b.plain_text()))
first = run[0]
out.append(
Block(
type=BlockType.paragraph,
text=joined,
spans=spans,
bbox=first.bbox,
reading_order=first.reading_order,
source=first.source,
align=first.align,
)
)
for i, b in enumerate(out):
b.reading_order = i
page.blocks = out
def _demote_heading_density(page: Page, body_size: float) -> None:
bodyish = [b for b in page.blocks if _is_bodyish(b)]
headings = [b for b in bodyish if b.type == BlockType.heading]
if not bodyish or not headings:
return
if len(headings) / len(bodyish) <= HEADING_DENSITY_CAP and len(headings) < 8:
return
# Demote headings whose size is at/under body * 1.15 (banner / OCR bbox).
for b in headings:
if b.level >= 1 and _block_size(b) <= body_size * 1.18:
b.type = BlockType.paragraph
b.level = 0
def _single_column_order(page: Page) -> None:
"""nocolumns: reading order is top-to-bottom, then left-to-right (or RTL)."""
from app.services.convert.text.arabic_logical import contains_arabic
def key(b: Block) -> tuple:
y = -(b.bbox.y if b.bbox else 0.0)
x = b.bbox.x if b.bbox else 0.0
rtl = contains_arabic(b.plain_text())
return (round(y / 8.0), -x if rtl else x)
page.blocks.sort(key=key)
for i, b in enumerate(page.blocks):
b.reading_order = i
def _unglue_section_numbers(document: Document) -> None:
"""Restore the lost space in ``4.4.1.3United Arab Emirates``.
The number and the title are separate text runs in the source, and a run
boundary carries no space, so a numbered clause heading arrives glued. It
then fails section-number detection, is not recognised as a heading, and
reads wrong in every output format.
The block's own text and its first span are both repaired: formatters emit
spans when a block has them and fall back to ``text`` when it does not, so
fixing only one leaves the other stale.
"""
for page in document.pages:
for block in page.blocks:
if block.type not in (BlockType.heading, BlockType.paragraph, BlockType.list_item):
continue
spans = getattr(block, "spans", None) or []
if spans and spans[0].text:
fixed = split_glued_section_number(spans[0].text)
if fixed != spans[0].text:
spans[0].text = fixed
if block.text:
block.text = split_glued_section_number(block.text)
# A run of this many consecutive body-size lines opening with a bare integer is
# a numbered list or a table column, not a section hierarchy.
MIN_NUMBERED_RUN = 3
# Sizes within this fraction of each other count as "the same size".
NUMBERED_RUN_SIZE_TOL = 0.12
_BARE_NUMBER_RE = re.compile(r"^(\d{1,3})[.)]?\s+\S")
def _bare_row_number(text: str) -> int | None:
"""The leading integer of ``1 Mooring inspection``, or None.
Only a flat integer counts. Hierarchical numbering (``4.2``, ``4.2.1``) is
genuine section numbering and is left alone.
"""
m = _BARE_NUMBER_RE.match((text or "").strip())
if not m:
return None
return int(m.group(1))
def _demote_numbered_row_runs(document: Document, body: float) -> None:
"""Stop a numbered column of table rows becoming a column of headings.
Almost every scanned table has a narrow leading number column — S.No, line
number, item, ref — and each row then reads exactly like a numbered
section: ``1 Mooring inspection``. The text rule that recognises
``1 Introduction`` as a heading cannot tell them apart from one line alone,
and on a scan there is no font change to separate them either, because the
whole page is one type size.
What does separate them is that headings do not arrive three-in-a-row at
body size with consecutive numbers. When they do, they are rows.
"""
for page in document.pages:
blocks = sorted(page.blocks, key=lambda b: b.reading_order)
run: list[Block] = []
def _flush(run: list[Block]) -> None:
if len(run) < MIN_NUMBERED_RUN:
return
for b in run:
b.type = BlockType.paragraph
b.level = 0
for block in blocks:
number = (
_bare_row_number(block.plain_text())
if block.type == BlockType.heading
else None
)
size = _block_size(block)
body_sized = abs(size - body) / max(body, 1.0) <= NUMBERED_RUN_SIZE_TOL
previous = _bare_row_number(run[-1].plain_text()) if run else None
if number is None or not body_sized:
_flush(run)
run = []
continue
if previous is not None and number != previous + 1:
_flush(run)
run = [block]
continue
run.append(block)
_flush(run)
def _collapsed_span(spans: list[TextSpan], text: str) -> TextSpan:
"""``text`` as one span, keeping the run style it is replacing.
The longest contributing span wins: after an echo collapse the surviving
text is the content that span carried, so its font is the honest answer.
``w`` is dropped because the old measured width describes the doubled text.
"""
lead = max(spans, key=lambda s: len(s.text or ""))
return replace(lead, text=text, w=None)
def _collapse_echoed_text(document: Document) -> None:
from app.services.convert.layout.text_quality import collapse_repeated_span
for page in document.pages:
for block in page.blocks:
if block.cells:
block.cells = [
[collapse_repeated_span(c) for c in row] for row in block.cells
]
if block.text:
block.text = collapse_repeated_span(block.text)
if block.spans:
original = "".join(s.text for s in block.spans)
joined = collapse_repeated_span(original)
# Compare stripped: ``collapse_repeated_span`` also trims, and
# glyph runs recovered from a content stream almost always end
# in a trailing space. Comparing raw made this branch fire on
# essentially every paragraph of every PDF, and the replacement
# span below carried no font -- so all font, size, bold and
# italic information was discarded document-wide. It then read
# as size 0.0 to _coalesce_short_line_paragraphs, whose
# same-size test compares 0.0 against 0.0, so unrelated
# neighbours were also welded into one paragraph: a 14pt
# heading and the 16pt line beneath it came out as one run.
if joined.strip() == original.strip():
continue
block.spans = [_collapsed_span(block.spans, joined)]
block.text = joined
def _drop_ocr_noise_blocks(document: Document) -> None:
"""Remove recogniser specks that survived as their own paragraphs."""
from app.services.convert.layout.text_quality import is_ocr_noise_line
for page in document.pages:
kept: list[Block] = []
for b in page.blocks:
t = b.plain_text().strip()
noisy = is_ocr_noise_line(t)
if b.type == BlockType.table:
kept.append(b)
continue
if b.type == BlockType.figure and b.image_png:
# A picture with *no* caption is a picture, not a speck. The
# OCR rebuild gives a recovered letterhead emblem an empty
# caption deliberately — printing "[Logo]" would put a word in
# the document nobody wrote — and ``is_ocr_noise_line("")`` is
# True, so the emblem was deleted right here while the
# warnings still reported it as placed in the Word header.
# Only a figure whose caption is *itself* recogniser noise
# ("@", ".r-\\^") is dropped.
if t and noisy:
continue
kept.append(b)
continue
if noisy:
continue
kept.append(b)
page.blocks = kept
for i, b in enumerate(page.blocks):
b.reading_order = i
_ROW_KEY_RE = re.compile(
r"^(section\s+\d+|\d{1,2}([.)]|[\s:])|total\b)",
re.I,
)
def _is_table_row_key(text: str) -> bool:
t = (text or "").strip()
if not t:
return False
return bool(_ROW_KEY_RE.match(t))
def _merge_wrapped_table_rows(cells: list[list[str]] | None) -> list[list[str]]:
"""Join OCR line-wraps inside a grid (Section title split across rows).
The third test below used to be "the row starts with a lowercase letter",
which is true of any table whose first column holds units, ids, or plain
lowercase labels — a four-row grid of them collapsed into a single row of
four run-together cells. A wrapped line is not merely lowercase: it is
*incomplete*, and the visible sign of that is that it fills fewer cells
than the row it continues. A row carrying a value in every column is a
data row whatever letter it starts with.
"""
rows = [list(r) for r in (cells or [])]
if len(rows) < 2:
return rows
width = max(len(r) for r in rows)
norm = [r + [""] * (width - len(r)) for r in rows]
def _populated(row: list[str]) -> int:
return sum(1 for c in row if (c or "").strip())
out = [norm[0]]
for row in norm[1:]:
left = (row[0] or "").strip()
prev = out[-1]
prev_left = (prev[0] or "").strip()
wrap = (
(not left)
or (_is_table_row_key(prev_left) and not _is_table_row_key(left))
or (
left[:1].islower()
and left[:1].isascii()
and prev_left
and _populated(row) < _populated(prev)
)
)
if wrap:
for i in range(width):
a, b = (prev[i] or "").strip(), (row[i] or "").strip()
prev[i] = (a + " " + b).strip() if b else a
else:
out.append(row)
return out
def _repair_wrapped_tables(document: Document) -> None:
for page in document.pages:
for block in page.blocks:
if block.type == BlockType.table and block.cells and len(block.cells) >= 3:
block.cells = _merge_wrapped_table_rows(block.cells)
def _drop_repeated_short_stamps(document: Document) -> None:
"""Drop leftover stamp/banner lines that repeat as body paragraphs."""
from collections import Counter
texts: list[str] = []
for page in document.pages:
for b in page.blocks:
if b.type in (BlockType.table, BlockType.figure, BlockType.header, BlockType.footer):
continue
t = b.plain_text().strip()
if t:
texts.append(t)
drop = set()
for t, n in Counter(texts).items():
words = t.split()
if n < 3 or not (2 <= len(words) <= 6):
continue
if any(ch.isdigit() for ch in t):
continue
if all(w[:1].isupper() or ("\u0600" <= w[:1] <= "\u06FF") for w in words):
drop.add(t)
if not drop:
return
for page in document.pages:
page.blocks = [
b
for b in page.blocks
if b.type in (BlockType.table, BlockType.figure, BlockType.header, BlockType.footer)
or b.plain_text().strip() not in drop
]
for i, b in enumerate(page.blocks):
b.reading_order = i
def _drop_junk_ocr_tables(document: Document) -> None:
"""Remove grids that are recogniser debris; demote the ones that are prose.
Three shapes used to be deleted outright. Two of them should be: cells that
are pure recogniser junk, and a stamp remnant whose text is already in the
document's header. The third — a grid whose first cell starts mid-phrase —
is not debris at all. It is a *paragraph* the table detector cut into
cells, and deleting it took real sentences out of the document before the
enhanced-flow pass downstream could turn them back into prose. It is
demoted to paragraphs here instead, so the words survive in every
recognition mode.
"""
for page in document.pages:
recognised = page.kind in (PageKind.scan, PageKind.hybrid)
rebuilt: list[Block] = []
for b in page.blocks:
if b.type != BlockType.table:
rebuilt.append(b)
continue
# Junk detection is about recogniser output. On a digital page
# read from the text layer there is no recogniser, so its verdict
# is a guess about content it was never designed to judge.
from_ocr = (b.source or "") == "ocr" or recognised
if from_ocr and (
_table_cells_are_ocr_junk(b.cells) or _table_is_stamp_remnant(b.cells)
):
continue
if _table_is_mid_sentence_fragment(b.cells):
rebuilt.extend(flatten_table_to_paragraphs(b))
continue
rebuilt.append(b)
page.blocks = rebuilt
for i, b in enumerate(page.blocks):
b.reading_order = i
def optimize_document(document: Document) -> None:
"""Apply reconstruction / recognition modes after per-page IDM + OCR."""
opts = get_options()
# ``exact`` is emitted as real OOXML frames by formatters.docx_exact. It is
# no longer downgraded to flowing here, and the optimisations that assume a
# reflowing document are skipped for it: on a positioned page every block
# owns a rectangle, so merging neighbouring blocks into one paragraph
# destroys the arrangement that was the whole reason to position them.
exact = opts.layout_mode == LayoutMode.exact
meta = document.meta if document.meta is not None else {}
meta["convert_options"] = opts.as_dict()
document.meta = meta
_unglue_section_numbers(document)
_collapse_echoed_text(document)
sizes = [_block_size(b) for p in document.pages for b in p.blocks if _is_bodyish(b)]
body = dominant_body_size(sizes) if sizes else 12.0
for page in document.pages:
if opts.layout_mode == LayoutMode.nocolumns:
_single_column_order(page)
if not exact:
_coalesce_short_line_paragraphs(
page, aggressive=opts.layout_mode == LayoutMode.continuous
)
_demote_heading_density(page, body)
_demote_numbered_row_runs(document, body)
_drop_ocr_noise_blocks(document)
_drop_repeated_short_stamps(document)
_drop_junk_ocr_tables(document)
_repair_wrapped_tables(document)
if not opts.detect_tables:
_drop_all_tables(document)
elif opts.recognition_mode == RecognitionMode.enhanced_flow:
# Word-soup grids from OCR become paragraphs. This must not run on a
# positioned page: a poster's panel grid is deliberate arrangement, not
# a misread paragraph, and flattening it is how a brochure turns to mush.
_flatten_soup_tables(document)
# textbox / flow: keep grids as-is (no soup flatten)
if opts.detect_tables:
_demote_implausible_tables(document)
def _table_has_figure_caption(page: Page, table: Block) -> bool:
"""Whether a visually table-like panel is explicitly captioned as a figure.
Layout models regularly label a legend, state diagram, or annotated code
panel as a table because the labels form clean rows and columns. The
document author provides stronger semantic evidence when a nearby caption
explicitly says "Figure N". Preserve the panel's text as paragraphs in
that case instead of exporting a misleading editable data table.
This deliberately does *not* match "Table N" captions, generic prose, or
distant page text. The caption must sit directly below the candidate and
overlap its horizontal footprint.
"""
box = table.bbox
if box is None or box.w <= 0 or box.h <= 0:
return False
table_right = box.x + box.w
minimum_overlap = min(72.0, box.w * 0.40)
for candidate in page.blocks:
if candidate is table or candidate.type is BlockType.table or candidate.bbox is None:
continue
caption = " ".join(candidate.plain_text().split())
folded = caption.casefold()
if folded.startswith("figure "):
rest = folded[len("figure ") :]
elif folded.startswith("fig. "):
rest = folded[len("fig. ") :]
elif folded.startswith("fig "):
rest = folded[len("fig ") :]
else:
continue
# Requiring an identifier avoids treating ordinary prose such as
# "Figure out..." as a caption.
if not rest or not (rest[0].isdigit() or rest[0].isalpha()):
continue
caption_box = candidate.bbox
caption_top = caption_box.y + caption_box.h
gap = box.y - caption_top
if not -6.0 <= gap <= 54.0:
continue
overlap = min(table_right, caption_box.x + caption_box.w) - max(box.x, caption_box.x)
if overlap >= minimum_overlap:
return True
return False
def _demote_implausible_tables(document: Document) -> None:
"""Last word on every grid in the document, however it got here.
Each detector checks its own output, but the passes that run afterwards
build tables the detectors never saw: a table continued across a page
boundary is merged with its continuation, a wrapped one is repaired, an
ML region is forced into a grid. Those transforms leave shapes no detector
would have accepted — a single row left behind by a merge, a worksheet
paragraph gathered into one 485-character cell — and they reached Word
looking like tables because nothing asked again.
Demotion, not deletion: the text becomes paragraphs, so a wrong answer
here costs formatting rather than content.
"""
from app.services.convert.layout import table_plausibility
for page in document.pages:
rebuilt: list[Block] = []
changed = False
figure_caption_demotions = 0
for block in page.blocks:
if block.type is not BlockType.table or not block.cells:
rebuilt.append(block)
continue
figure_caption = _table_has_figure_caption(page, block)
if not figure_caption and table_plausibility.assess(block.cells):
rebuilt.append(block)
continue
changed = True
figure_caption_demotions += int(figure_caption)
rebuilt.extend(flatten_table_to_paragraphs(block))
if changed:
page.blocks = rebuilt
for i, b in enumerate(page.blocks):
b.reading_order = i
if figure_caption_demotions:
message = (
f"Page {page.index + 1}: {figure_caption_demotions} table-like figure panel(s) "
"were kept as text because of a Figure caption."
)
if message not in document.warnings:
document.warnings.append(message)
gateway/app/services/convert/layout/images.py
+503
View File
@@ -0,0 +1,503 @@
"""Page image / display-list helpers for classification and table rulings."""
from __future__ import annotations
import io
from contextlib import suppress
from dataclasses import dataclass
from typing import Any
# Identity transform, in PDF's [a b c d e f] form.
_IDENTITY = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)
# Below this on-page size (points) an image is furniture — a rule, a bullet, a
# spacer — not a figure worth carrying into the output.
MIN_FIGURE_POINTS = 12.0
def _mat_mul(m: tuple, n: tuple) -> tuple:
"""PDF matrix product m x n (row-vector convention)."""
a1, b1, c1, d1, e1, f1 = m
a2, b2, c2, d2, e2, f2 = n
return (
a1 * a2 + b1 * c2,
a1 * b2 + b1 * d2,
c1 * a2 + d1 * c2,
c1 * b2 + d1 * d2,
e1 * a2 + f1 * c2 + e2,
e1 * b2 + f1 * d2 + f2,
)
def _apply(m: tuple, x: float, y: float) -> tuple[float, float]:
a, b, c, d, e, f = m
return (a * x + c * y + e, b * x + d * y + f)
@dataclass
class ImagePlacement:
"""Where an image XObject is actually painted on the page.
``name`` is the XObject resource name, so the placement can be matched back
to the image data. Coordinates are PDF user space (origin bottom-left).
"""
name: str
x: float
y: float
w: float
h: float
px_w: int = 0
px_h: int = 0
@property
def area(self) -> float:
return max(self.w, 0.0) * max(self.h, 0.0)
def image_placements(page) -> list[ImagePlacement]:
"""True on-page rectangles for every image drawn by this page.
Memoised on the page object: the coverage estimate and the figure
extractor both need it, and walking the content stream twice per page is
pure waste.
An image XObject is painted into the unit square, so the CTM in effect at
its ``Do`` operator *is* its placement. Walking ``q``/``Q``/``cm``/``Do``
recovers that exactly.
This replaces guessing placement from pixel dimensions, which produced a
fabricated rectangle: a 1200x1200 logo and a full-page scan have similar
pixel counts but completely different footprints, and treating the logo as
a full-page raster silently dropped it from the output.
"""
cached = getattr(page, "_dq_image_placements", None)
if cached is not None:
return cached
out: list[ImagePlacement] = []
try:
from pypdf.generic import ContentStream
resources = page.get("/Resources")
xobjects = {}
if resources is not None:
xo = resources.get_object().get("/XObject")
if xo is not None:
xobjects = xo.get_object()
if not xobjects:
return out
contents = page.get_contents()
if contents is None:
return out
stream = ContentStream(contents, getattr(page, "pdf", None))
ctm = _IDENTITY
stack: list[tuple] = []
for operands, operator in stream.operations:
if operator == b"q":
stack.append(ctm)
elif operator == b"Q":
ctm = stack.pop() if stack else _IDENTITY
elif operator == b"cm" and len(operands) >= 6:
try:
ctm = _mat_mul(tuple(float(v) for v in operands[:6]), ctm)
except (TypeError, ValueError):
continue
elif operator == b"Do" and operands:
name = str(operands[0])
obj = xobjects.get(name)
if obj is None:
continue
try:
obj = obj.get_object()
except Exception:
continue
if obj.get("/Subtype") != "/Image":
continue
corners = [_apply(ctm, cx, cy) for cx, cy in ((0, 0), (1, 0), (1, 1), (0, 1))]
xs = [c[0] for c in corners]
ys = [c[1] for c in corners]
out.append(
ImagePlacement(
name=name,
x=min(xs),
y=min(ys),
w=max(xs) - min(xs),
h=max(ys) - min(ys),
px_w=int(obj.get("/Width", 0) or 0),
px_h=int(obj.get("/Height", 0) or 0),
)
)
except Exception:
return out
with suppress(Exception):
page._dq_image_placements = out
return out
def image_blocks_from_pypdf_page(page) -> list[dict]:
"""Image footprints on a pypdf page, in PDF user-space units.
Uses the real content-stream placement, falling back to the old pixel-ratio
approximation only when the content stream cannot be walked.
"""
placements = image_placements(page)
if placements:
return [
{"x": p.x, "y": p.y, "w": p.w, "h": p.h, "width": p.w, "height": p.h, "name": p.name}
for p in placements
]
blocks: list[dict] = []
try:
if "/Resources" not in page or "/XObject" not in page["/Resources"]:
return blocks
xobjects = page["/Resources"]["/XObject"].get_object()
mediabox = page.mediabox
page_w = float(mediabox.width)
page_h = float(mediabox.height)
for _name in xobjects:
obj = xobjects[_name]
if obj.get("/Subtype") != "/Image":
continue
w = float(obj.get("/Width", page_w))
h = float(obj.get("/Height", page_h))
# Without CTM we approximate: large images cover most of the page
scale = min(page_w / max(w, 1), page_h / max(h, 1))
blocks.append({"w": w * scale, "h": h * scale, "width": w * scale, "height": h * scale})
except Exception:
return blocks
return blocks
def image_blocks_from_engine_page(page: Any) -> list[dict]:
blocks: list[dict] = []
for meth in ("extract_images", "list_images", "get_images"):
if not hasattr(page, meth):
continue
try:
raw = getattr(page, meth)()
except Exception:
continue
if not raw:
continue
for item in raw:
if isinstance(item, dict):
blocks.append(item)
else:
blocks.append(
{
"w": float(getattr(item, "w", getattr(item, "width", 0)) or 0),
"h": float(getattr(item, "h", getattr(item, "height", 0)) or 0),
}
)
if blocks:
return blocks
return blocks
def display_list_ops_from_engine_page(page: Any) -> list[dict]:
if not hasattr(page, "extract_display_list"):
return []
try:
raw = page.extract_display_list()
except Exception:
return []
if isinstance(raw, str):
raw_str = raw.strip()
if raw_str.startswith(("[", "{")):
try:
import json
raw = json.loads(raw_str)
except Exception:
pass
if isinstance(raw, list):
ops: list[dict] = []
for x in raw:
if isinstance(x, dict):
op_dict = dict(x)
if "args" in op_dict and isinstance(op_dict["args"], (list, tuple)):
args = op_dict["args"]
if len(args) >= 4 and str(op_dict.get("op") or "").lower() in ("re", "rect"):
try:
rx, ry, rw, rh = float(args[0]), float(args[1]), float(args[2]), float(args[3])
op_dict.setdefault("x", rx)
op_dict.setdefault("y", ry)
op_dict.setdefault("w", rw)
op_dict.setdefault("h", rh)
op_dict.setdefault("x0", rx)
op_dict.setdefault("y0", ry)
op_dict.setdefault("x1", rx + rw)
op_dict.setdefault("y1", ry + rh)
except (ValueError, TypeError):
pass
ops.append(op_dict)
else:
ops.append({"type": "line"})
return ops
if isinstance(raw, str):
# Best-effort parse of simple "line x0 y0 x1 y1" lines
ops: list[dict] = []
for line in raw.splitlines():
parts = line.strip().split()
if len(parts) >= 5 and parts[0].lower() in ("line", "vline"):
try:
ops.append(
{
"type": parts[0].lower(),
"x0": float(parts[1]),
"y0": float(parts[2]),
"x1": float(parts[3]),
"y1": float(parts[4]),
}
)
except ValueError:
continue
return ops
return []
def _colorspace_mode(cs) -> str | None:
"""Map PDF ColorSpace to Pillow mode when possible."""
if cs is None:
return "RGB"
name = str(cs)
if "DeviceGray" in name or name.endswith("/DeviceGray"):
return "L"
if "DeviceRGB" in name or name.endswith("/DeviceRGB"):
return "RGB"
if "DeviceCMYK" in name or name.endswith("/DeviceCMYK"):
return "CMYK"
# ICCBased / Indexed: try RGB bytes if Length matches later
if "ICCBased" in name:
return "RGB"
return None
def _decode_xobject_png(obj) -> bytes | None:
"""One image XObject as PNG bytes, or None when it cannot be decoded."""
from PIL import Image
from pypdf.generic import NameObject
width = int(obj.get("/Width", 0))
height = int(obj.get("/Height", 0))
try:
data = obj.get_data()
except Exception:
return None
filt = obj.get("/Filter")
img = None
try:
if filt == "/DCTDecode" or (isinstance(filt, list) and NameObject("/DCTDecode") in filt):
img = Image.open(io.BytesIO(data))
elif filt == "/FlateDecode" or str(filt) == "/FlateDecode" or (
isinstance(filt, list) and any("Flate" in str(f) for f in filt)
):
mode = _colorspace_mode(obj.get("/ColorSpace")) or "RGB"
expected = width * height * (1 if mode == "L" else 4 if mode == "CMYK" else 3)
if len(data) < expected:
# Sometimes get_data already expands; try open as encoded
try:
img = Image.open(io.BytesIO(data))
except Exception:
return None
else:
img = Image.frombytes(mode, (width, height), data[:expected])
else:
img = Image.open(io.BytesIO(data))
except Exception:
return None
if img is None:
return None
buf = io.BytesIO()
img.convert("RGB").save(buf, format="PNG")
return buf.getvalue()
def decode_page_images(
page, *, warnings: list[str] | None = None, only: set[str] | None = None
) -> dict[str, bytes]:
"""Decodable image XObjects on the page, keyed by resource name.
Keyed by name so a placement recovered from the content stream can be
matched to its pixels: a page with a chart and a photo needs both, and the
old "largest image only" extraction returned one of them.
``only`` restricts decoding to the named XObjects. Decoding is the
expensive part, so a caller that already knows which images it will keep
should say so rather than decoding a full-page raster to discard it.
"""
out: dict[str, bytes] = {}
try:
if "/Resources" not in page or "/XObject" not in page["/Resources"]:
return out
xobjects = page["/Resources"]["/XObject"].get_object()
image_count = 0
for name in xobjects:
if only is not None and str(name) not in only:
continue
obj = xobjects[name]
if obj.get("/Subtype") != "/Image":
continue
image_count += 1
png = _decode_xobject_png(obj)
if png:
out[str(name)] = png
if not out and image_count and warnings is not None:
warnings.append(
f"page has {image_count} image XObject(s) but decode failed "
f"({image_count} failure(s))."
)
except Exception as exc:
if warnings is not None:
warnings.append(f"embedded image extract failed: {exc}")
return out
def embedded_page_image_png(page, *, warnings: list[str] | None = None) -> bytes | None:
"""Largest embedded image on a pypdf page as PNG bytes, if any.
"Largest" is by pixel area, as the OCR path needs the highest-resolution
raster — not by encoded size, which favours noisy images over big ones.
"""
images = decode_page_images(page, warnings=warnings)
if not images:
return None
def _area(png: bytes) -> int:
try:
from PIL import Image
with Image.open(io.BytesIO(png)) as im:
return im.width * im.height
except Exception:
return len(png)
return max(images.values(), key=_area)
def figure_blocks_from_pypdf_page(
page,
*,
reading_order: int = 0,
warnings: list[str] | None = None,
max_page_coverage: float = 0.38,
max_bytes: int = 450_000,
) -> list:
"""Extract embedded images as IDM figure blocks (digital path).
Every image on the page is emitted, at the rectangle the content stream
actually paints it into. Two things this fixes:
*multiple figures per page*
The previous implementation extracted only the single largest image, so
a page with a chart and a photograph lost one of them outright.
*real placement instead of a fabricated box*
Position and size came from a hard-coded guess, which destroyed both
reading order relative to the text and the aspect ratio. Coverage was
likewise inferred from a pixel-count ladder, so a high-resolution logo
was mistaken for a full-page scan and dropped.
Near-full-page rasters are still skipped: those belong to OCR/scan handling
and would otherwise embed a multi-MB page image per page.
"""
from app.services.convert.idm.model import BBox, Block, BlockType
out: list = []
try:
mediabox = page.mediabox
pw, ph = float(mediabox.width), float(mediabox.height)
page_area = max(pw * ph, 1.0)
placements = image_placements(page)
if not placements:
# No usable content stream: fall back to the single largest image
# with a conservative centred box rather than dropping everything.
png = embedded_page_image_png(page, warnings=warnings)
if not png or len(png) > max_bytes:
return out
out.append(
Block(
type=BlockType.figure,
text="[Image]",
bbox=BBox(x=72, y=ph * 0.3, w=min(pw - 144, 400), h=min(ph * 0.4, 300)),
image_png=png,
reading_order=reading_order,
)
)
return out
# Decide from geometry *before* decoding. Decoding a full-page raster
# only to discard it costs seconds per page on an image-heavy document,
# and the placement already says it will be discarded.
keep: list[ImagePlacement] = []
skipped_large = 0
for placement in placements:
if placement.w < MIN_FIGURE_POINTS or placement.h < MIN_FIGURE_POINTS:
# Rules, bullets and spacer pixels are not figures.
continue
if placement.area / page_area >= max_page_coverage:
skipped_large += 1
continue
keep.append(placement)
if keep:
wanted = {p.name for p in keep}
images = decode_page_images(page, warnings=warnings, only=wanted)
# Top-to-bottom, then left-to-right: the order a reader meets them.
keep.sort(key=lambda p: (-(p.y + p.h), p.x))
for offset, placement in enumerate(keep):
png = images.get(placement.name)
if not png:
continue
if len(png) > max_bytes:
skipped_large += 1
continue
out.append(
Block(
type=BlockType.figure,
text="[Image]",
bbox=BBox(x=placement.x, y=placement.y, w=placement.w, h=placement.h),
image_png=png,
reading_order=reading_order + offset,
)
)
if skipped_large and warnings is not None:
warnings.append(
f"Skipped {skipped_large} large/full-page embedded image(s); prefer OCR text."
)
except Exception as exc:
if warnings is not None:
warnings.append(f"figure block extract failed: {exc}")
return out
return out
def figure_blocks_from_engine_images(
img_meta: list[dict], png_blobs: list[bytes], *, reading_order: int = 0
) -> list:
from app.services.convert.idm.model import BBox, Block, BlockType
out: list = []
for i, png in enumerate(png_blobs):
meta = img_meta[i] if i < len(img_meta) else {}
w = float(meta.get("w") or meta.get("width") or 300)
h = float(meta.get("h") or meta.get("height") or 200)
x = float(meta.get("x", 72))
y = float(meta.get("y", 200))
out.append(
Block(
type=BlockType.figure,
text=f"[Image {i + 1}]",
bbox=BBox(x=x, y=y, w=w, h=h),
image_png=png,
reading_order=reading_order + i,
)
)
return out
gateway/app/services/convert/layout/logo_crop.py
+415
View File
@@ -0,0 +1,415 @@
"""Masthead logo segmentation for pages that embed only full-page rasters.
Some scanned documents carry no separate image XObject at all: every page is
one full-page bitmap, so the figure extractor correctly declines to embed any
of it (a page-sized raster per page is not a figure, it is the page). The
emblem at the top of page one is then lost entirely, even though it is the one
piece of artwork a reader expects to survive.
This module recovers it by segmenting the raster instead of the PDF: ink in the
masthead band that OCR did *not* claim as text is, by elimination, artwork.
Deliberately dependency-light — Pillow and numpy only, both already required by
the OCR path. No OpenCV, no paid SDK.
"""
from __future__ import annotations
import io
from dataclasses import dataclass
# At most this many emblems per masthead. A letterhead carries an emblem and
# perhaps a wordmark or seal; more clusters than this means the band is text
# the OCR mask failed to claim, not artwork.
MAX_LOGOS = 3
# A second mark must be at least this share of the largest one's area to be a
# mark rather than a speck the text mask missed.
MIN_RELATIVE_MARK_AREA = 0.08
# Masthead band as a fraction of page height, measured from the top.
BAND_RATIO = 0.22
# A pixel this much darker than the page's paper white counts as ink.
INK_DELTA = 42
# Side strips of the masthead, as a fraction of page width. Letterhead marks
# live here; a full-width OCR envelope for the banner must not decide them.
SIDE_STRIP_RATIO = 0.36
# Fainter ink still counts as a mark in a side strip — gold/grey emblems sit
# closer to paper than black body text.
SIDE_INK_DELTA = 18
# Text boxes are grown by this fraction of their size before masking, so
# antialiased edges of glyphs do not survive as stray ink.
TEXT_PAD_RATIO = 0.14
# A logo must occupy at least this fraction of the band's width and height.
MIN_SIDE_RATIO = 0.025
# ...and no more than this, or it is the whole banner rather than an emblem.
MAX_WIDTH_RATIO = 0.55
MAX_HEIGHT_RATIO = 0.95
# A leftover banner stroke is a wide, short bar. Emblems are compact.
MAX_MARK_ASPECT = 2.0
# Below this ink density the crop is speckle, not artwork.
MIN_INK_DENSITY = 0.04
MIN_FAINT_INK_DENSITY = 0.022
# Columns with less than this share of the band's peak ink are gaps.
COLUMN_GAP_RATIO = 0.06
# Output is scaled down to keep the DOCX small; a masthead needs no more.
MAX_OUTPUT_SIDE = 420
@dataclass
class LogoCrop:
"""One emblem cropped from a masthead, with where it sat on the raster."""
png: bytes
# Pixel box on the page raster: (x, y, w, h), top-left origin.
bbox_px: tuple[int, int, int, int]
raster_size: tuple[int, int]
def bbox_points(self, page_width: float, page_height: float) -> tuple[float, float, float, float]:
"""The same box in PDF points on a page of the given size.
Without this the recovered artwork reached the writer with no geometry
at all, and every figure with no geometry was emitted at a hard-coded
5.5 inches — which is how a two-centimetre emblem became a full-width
banner across the top of the Word document.
"""
rw, rh = self.raster_size
if rw <= 0 or rh <= 0:
return 0.0, 0.0, 0.0, 0.0
sx = float(page_width) / float(rw)
sy = float(page_height) / float(rh)
x, y, w, h = self.bbox_px
# Raster y grows downwards, PDF y grows upwards.
return (x * sx, page_height - (y + h) * sy, w * sx, h * sy)
def extract_masthead_logos(
page_png: bytes,
text_boxes: list[tuple[float, float, float, float]] | None = None,
*,
band_ratio: float = BAND_RATIO,
max_logos: int = MAX_LOGOS,
) -> list[LogoCrop]:
"""Crop every emblem from the top band of a page raster, left to right.
``text_boxes`` are OCR boxes in the same pixel space as ``page_png``, as
``(x, y, w, h)``. Anything they cover is text and is masked out; the compact
ink regions left in the band are artwork.
Letterheads very often carry two marks — an emblem on one side and a
wordmark or seal on the other. Returning only the largest lost one of them
and, worse, stretched the survivor across the page. Every qualifying cluster
is returned, in reading order.
An empty list is the common case and must stay cheap and silent.
"""
try:
import numpy as np
from PIL import Image
except Exception:
return []
try:
with Image.open(io.BytesIO(page_png)) as im:
page = im.convert("L")
width, height = page.size
band_h = max(1, int(height * band_ratio))
band = np.asarray(page.crop((0, 0, width, band_h)), dtype="int16")
if band.size == 0:
return []
# Paper white is the band's brightest common value, not 255: a scan is
# grey, and thresholding against pure white marks the whole page as ink.
paper = int(np.percentile(band, 90))
ink = band < (paper - INK_DELTA)
ink_raw = ink.copy()
faint = band < (paper - SIDE_INK_DELTA)
_mask_text(ink, text_boxes, band_h, page_width=width)
clusters = _ink_regions(ink, np) if ink.any() else []
out = _crops_from_boxes(clusters, page_png, width, height, band_h, ink, max_logos)
# Prefer one mark in each side strip from ink that the banner envelope
# has not been allowed to erase. A letterhead is those two corners.
out = _prefer_side_marks(
out,
_side_strip_crops(
page_png, ink_raw, faint, width, height, band_h, np, text_boxes
),
width,
max_logos,
)
return out
except Exception:
return []
def extract_masthead_logo(
page_png: bytes,
text_boxes: list[tuple[float, float, float, float]] | None = None,
*,
band_ratio: float = BAND_RATIO,
) -> bytes | None:
"""The single most prominent emblem, as PNG bytes. Prefer the plural form."""
crops = extract_masthead_logos(page_png, text_boxes, band_ratio=band_ratio, max_logos=MAX_LOGOS)
if not crops:
return None
return max(crops, key=lambda c: c.bbox_px[2] * c.bbox_px[3]).png
def _qualifies_as_mark(w: int, h: int, width: int, band_h: int) -> bool:
if w < width * MIN_SIDE_RATIO or h < band_h * MIN_SIDE_RATIO:
return False
if w > width * MAX_WIDTH_RATIO or h > band_h * MAX_HEIGHT_RATIO:
return False
if h > 0 and (w / h) > MAX_MARK_ASPECT and h < band_h * 0.40:
return False
return True
def _crop_area(crop: LogoCrop) -> int:
return crop.bbox_px[2] * crop.bbox_px[3]
def _crops_from_boxes(boxes, page_png, width, height, band_h, ink, max_logos) -> list[LogoCrop]:
out: list[LogoCrop] = []
for x0, y0, x1, y1 in boxes:
w, h = x1 - x0, y1 - y0
if not _qualifies_as_mark(w, h, width, band_h):
continue
if ink[y0:y1, x0:x1].mean() < MIN_INK_DENSITY:
continue
png = _crop_png(page_png, (x0, y0, x1, y1))
if not png:
continue
out.append(
LogoCrop(png=png, bbox_px=(x0, y0, w, h), raster_size=(width, height))
)
if len(out) >= max_logos:
break
return out
def _best_in_strip(
ink, x0: int, x1: int, page_png, width, height, band_h, np, *, min_density: float = MIN_INK_DENSITY
) -> LogoCrop | None:
if x1 <= x0 or ink.size == 0:
return None
strip = ink[:, x0:x1]
if not strip.any():
return None
boxes = _ink_regions(strip, np)
if not boxes:
return None
bx0, by0, bx1, by1 = max(boxes, key=lambda b: (b[2] - b[0]) * (b[3] - b[1]))
abs0, abs1 = bx0 + x0, bx1 + x0
w, h = abs1 - abs0, by1 - by0
if not _qualifies_as_mark(w, h, width, band_h):
return None
if ink[by0:by1, abs0:abs1].mean() < min_density:
return None
png = _crop_png(page_png, (abs0, by0, abs1, by1))
if not png:
return None
return LogoCrop(png=png, bbox_px=(abs0, by0, w, h), raster_size=(width, height))
def _mask_strip_text(ink, text_boxes, x0: int, x1: int, band_h: int) -> None:
"""Mask OCR boxes inside a side strip, with no centre carve-out.
Page-wide banner envelopes are skipped: those are the case the side pass
exists to recover. A compact box in the strip is real text and must go.
"""
if not text_boxes:
return
rows, cols = ink.shape
wide = cols * 0.55
outer = cols * 0.16
for bx, by, bw, bh in text_boxes:
if bw <= 0 or bh <= 0 or bw >= wide:
continue
if bh < band_h * 0.28 and bw > cols * 0.35:
continue
cx = bx + bw / 2
if cx < outer or cx > cols - outer:
continue
sx0 = max(x0, 0, int(bx))
sx1 = min(x1, cols, int(bx + bw))
y0 = max(0, int(by))
y1 = min(rows, int(by + bh), band_h)
if sx1 > sx0 and y1 > y0:
ink[y0:y1, sx0:sx1] = False
def _side_strip_crops(
page_png, ink_raw, faint, width, height, band_h, np, text_boxes
) -> list[LogoCrop]:
left_end = max(1, int(width * SIDE_STRIP_RATIO))
right_start = min(width - 1, int(width * (1.0 - SIDE_STRIP_RATIO)))
found: list[LogoCrop] = []
for x0, x1 in ((0, left_end), (right_start, width)):
masked = ink_raw.copy()
_mask_strip_text(masked, text_boxes, x0, x1, band_h)
crop = _best_in_strip(masked, x0, x1, page_png, width, height, band_h, np)
if crop is None:
# Do not mask OCR boxes on the faint pass: letterhead artwork is
# often boxed as text and the second mark disappears.
crop = _best_in_strip(
faint,
x0,
x1,
page_png,
width,
height,
band_h,
np,
min_density=MIN_FAINT_INK_DENSITY,
)
if crop is not None:
found.append(crop)
return found
def _centres_overlap(a: LogoCrop, b: LogoCrop) -> bool:
ax, ay, aw, ah = a.bbox_px
bx, by, bw, bh = b.bbox_px
acx, acy = ax + aw / 2, ay + ah / 2
return bx <= acx <= bx + bw and by <= acy <= by + bh
def _prefer_side_marks(
primary: list[LogoCrop],
sides: list[LogoCrop],
width: int,
max_logos: int,
) -> list[LogoCrop]:
"""Keep at most one mark per third of the masthead, sides first.
A leftover banner fragment in the centre is often larger than the real
emblems. Ranking by area then dropped the letterhead. The corners are the
letterhead; the middle is only kept when it is comparable to them.
"""
combined: list[LogoCrop] = []
for crop in list(sides) + list(primary):
if any(_centres_overlap(crop, existing) or _centres_overlap(existing, crop) for existing in combined):
continue
combined.append(crop)
def _bucket(crop: LogoCrop) -> str:
cx = crop.bbox_px[0] + crop.bbox_px[2] / 2
if cx < width * 0.40:
return "left"
if cx > width * 0.60:
return "right"
return "mid"
buckets: dict[str, list[LogoCrop]] = {"left": [], "mid": [], "right": []}
for crop in combined:
buckets[_bucket(crop)].append(crop)
kept: list[LogoCrop] = []
for key in ("left", "right"):
if buckets[key]:
kept.append(max(buckets[key], key=_crop_area))
if buckets["mid"] and len(kept) < max_logos:
mid = max(buckets["mid"], key=_crop_area)
floor = max((_crop_area(c) for c in kept), default=1) * MIN_RELATIVE_MARK_AREA
if _crop_area(mid) >= floor:
kept.append(mid)
kept.sort(key=lambda c: c.bbox_px[0])
return kept[:max_logos]
def _mask_text(ink, text_boxes, band_h: int, page_width: int | None = None) -> None:
"""Zero every pixel a text box covers, padded for antialiasing."""
if not text_boxes:
return
rows, cols = ink.shape
page_w = float(page_width or cols)
wide = page_w * 0.55
# Letterhead marks sit in the outer sixths. OCR often boxes them as if they
# were glyphs; masking those boxes is how one emblem survived and the other
# vanished. Never erase that margin.
margin = int(page_w * 0.16)
side_keep = int(page_w * 0.20)
for bx, by, bw, bh in text_boxes:
if bw <= 0 or bh <= 0:
continue
if bw >= page_w * 0.35 and bh < band_h * 0.28:
continue
pad_x = bw * TEXT_PAD_RATIO
pad_y = bh * TEXT_PAD_RATIO
x0 = max(0, int(bx - pad_x))
y0 = max(0, int(by - pad_y))
x1 = min(cols, int(bx + bw + pad_x))
y1 = min(rows, int(by + bh + pad_y))
if bw >= wide:
x0 = max(x0, side_keep)
x1 = min(x1, cols - side_keep)
x0 = max(x0, margin)
x1 = min(x1, cols - margin)
if y0 >= band_h:
continue
if x1 > x0 and y1 > y0:
ink[y0:y1, x0:x1] = False
def _ink_regions(ink, np) -> list[tuple[int, int, int, int]]:
"""Bounding boxes of the contiguous column runs of ink, left to right.
Column projection rather than connected components: an emblem is one
horizontal cluster separated from the rest of the masthead by whitespace,
and projection needs no scipy and no arbitrary structuring element.
"""
col_ink = ink.sum(axis=0)
peak = col_ink.max()
if peak <= 0:
return []
gap = max(1.0, peak * COLUMN_GAP_RATIO)
spans: list[tuple[int, int]] = []
start = None
for x, value in enumerate(col_ink):
if value >= gap:
if start is None:
start = x
elif start is not None:
spans.append((start, x))
start = None
if start is not None:
spans.append((start, len(col_ink)))
boxes: list[tuple[int, int, int, int]] = []
for x0, x1 in spans:
rows = np.where(ink[:, x0:x1].any(axis=1))[0]
if rows.size == 0:
continue
boxes.append((x0, int(rows[0]), x1, int(rows[-1]) + 1))
return boxes
def _largest_ink_region(ink, np):
"""Bounding box of the densest contiguous column run. Kept for callers."""
boxes = _ink_regions(ink, np)
if not boxes:
return None
return max(boxes, key=lambda b: (b[2] - b[0]) * (b[3] - b[1]))
def _crop_png(page_png: bytes, box: tuple[int, int, int, int]) -> bytes | None:
from PIL import Image
x0, y0, x1, y1 = box
with Image.open(io.BytesIO(page_png)) as im:
crop = im.convert("RGB").crop((x0, y0, x1, y1))
if crop.width < 2 or crop.height < 2:
return None
longest = max(crop.size)
if longest > MAX_OUTPUT_SIDE:
scale = MAX_OUTPUT_SIDE / longest
crop = crop.resize(
(max(1, int(crop.width * scale)), max(1, int(crop.height * scale))),
Image.LANCZOS,
)
buf = io.BytesIO()
crop.save(buf, format="PNG", optimize=True)
return buf.getvalue()
gateway/app/services/convert/layout/ml_merge.py
+396
View File
@@ -0,0 +1,396 @@
"""Merge ML layout regions with heuristic glyph lines into IDM blocks."""
from __future__ import annotations
from app.services.convert.idm.model import BBox, Block, BlockType, PageKind
from app.services.convert.layout.blocks import (
estimate_body_size,
lines_to_block,
)
from app.services.convert.layout.glyphs import Line
from app.services.convert.layout.ml_regions import Region
from app.services.convert.layout.paragraphs import group_lines as group_paragraph_lines
from app.services.convert.layout.paragraphs import lexical_hyphen_pairs
from app.services.convert.layout.paragraphs import measure as measure_page
from app.services.convert.layout.reading_order import order_lines
from app.services.convert.layout.tables import coalesce_lines_by_y, extract_tables
from app.services.convert.options import get_options
_LABEL_TO_BLOCK = {
"title": BlockType.heading,
"text": BlockType.paragraph,
"list": BlockType.list_item,
"table": BlockType.table,
"figure": BlockType.figure,
"header": BlockType.header,
"footer": BlockType.footer,
}
def _line_center(line: Line) -> tuple[float, float]:
return ((line.x0 + line.x1) / 2.0, line.y)
def _contains(bbox: BBox, x: float, y: float, pad: float = 2.0) -> bool:
return (
bbox.x - pad <= x <= bbox.x + bbox.w + pad
and bbox.y - pad <= y <= bbox.y + bbox.h + pad
)
def _overlap_line(bbox: BBox, line: Line, *, y_pad: float = 0.0) -> bool:
cx, cy = _line_center(line)
if _contains(bbox, cx, cy, pad=max(2.0, y_pad)):
return True
# horizontal overlap + y near region vertical span
lx0, lx1 = line.x0, line.x1
y0, y1 = bbox.y - y_pad, bbox.y + bbox.h + y_pad
return not (lx1 < bbox.x or lx0 > bbox.x + bbox.w) and y0 <= cy <= y1
def lines_in_region(lines: list[Line], region: Region) -> list[Line]:
return [ln for ln in lines if _overlap_line(region.bbox_pdf, ln, y_pad=8.0)]
def _paragraph_blocks(
lines: list[Line],
start_order: int,
body: float,
page_width: float,
*,
force_type: BlockType | None = None,
) -> list[Block]:
"""Group ordered lines into paragraph blocks.
The ML path previously emitted one block per visual line, so a document
routed through layout detection came out with every line as its own
``<w:p>`` even though the deterministic path reconstructed paragraphs.
Both paths now share :mod:`layout.paragraphs`.
``force_type`` applies a region's label to the resulting blocks. Headings
and list items are never merged across lines: a heading region holds one
heading, and each list item is its own block.
"""
if not lines:
return []
unmergeable = force_type in (
BlockType.heading,
BlockType.list_item,
BlockType.header,
BlockType.footer,
)
metrics = measure_page(lines, body_size=body, page_width=page_width)
pairs = lexical_hyphen_pairs("\n".join(ln.text for ln in lines))
runs = [[ln] for ln in lines] if unmergeable else group_paragraph_lines(lines, metrics)
out: list[Block] = []
order = start_order
for run in runs:
block = lines_to_block(run, order, body, lexical_pairs=pairs)
if force_type is BlockType.heading:
block.type = BlockType.heading
block.level = block.level or 1
elif force_type is BlockType.paragraph:
# Adobe-style: a text region is body, not a size-based H1.
if block.type == BlockType.heading:
block.type = BlockType.paragraph
block.level = 0
elif force_type is not None:
block.type = force_type
out.append(block)
order += 1
return out
def _structure_rulings(
reg: Region,
*,
page_png: bytes | None,
page_width: float,
page_height: float,
page_index: int,
vertical_rulings: list[float] | None,
warnings: list[str],
) -> list[float] | None:
"""Column separators for one ML table region, or None to keep the heuristics.
Every failure path here returns None and leaves ``vertical_rulings``
untouched: the optional structure model may sharpen a grid, never break one.
"""
from app.services.convert.layout import ml_table_structure as mts
if not mts.table_structure_enabled() or not page_png or not page_height:
return None
try:
structure, err = mts.structure_for_region(
page_png, reg.bbox_pdf, page_width, page_height
)
except Exception as exc: # pragma: no cover - defensive; hook must never raise
warnings.append(f"Page {page_index + 1}: table_structure_fallback=heuristic ({exc})")
return None
if structure is None:
if err:
warnings.append(f"Page {page_index + 1}: table_structure_fallback=heuristic ({err})")
return None
merged = mts.merge_rulings(vertical_rulings, structure.column_x)
warnings.append(
f"Page {page_index + 1}: table_structure=onnx cols={structure.columns} "
f"score={structure.score:.2f}."
)
return merged
def merge_regions_with_lines(
lines: list[Line],
regions: list[Region],
*,
page_width: float,
page_index: int,
kind: PageKind,
warnings: list[str],
vertical_rulings: list[float] | None = None,
page_rects: list | None = None,
path_ops: list[dict] | None = None,
page_png: bytes | None = None,
page_height: float | None = None,
) -> list[Block]:
"""
Build blocks using ML region order + heuristic tables.
Order of operations (best practice fusion):
1. Coalesce gutter-split column fragments into logical rows.
2. Extract grids from explicit ML *table* regions (relaxed / ml_hinted).
3. Run page-level heuristic extract_tables on leftovers (catches pipe
tables that ML mislabeled as per-row ``text`` regions).
4. Type remaining lines via non-table ML regions (heading/list/header…).
5. Leftover lines → paragraphs.
"""
if not regions:
return []
working = coalesce_lines_by_y(lines) if lines else []
if not any(ln.text.strip() for ln in working):
# A full-page bitmap: the detector found rectangles, but there is no
# text layer for them to describe. Emitting region blocks here — empty
# tables in particular — puts structure on the page that nothing has
# read yet, and the OCR rebuild that follows has the actual detection
# boxes and assembles far better grids from them. The regions are not
# wasted: the OCR pass still uses the table rectangles as crop hints.
warnings.append(
f"Page {page_index + 1}: layout_ml regions on a raster page; "
"deferring structure to OCR."
)
return []
assigned: set[int] = set()
blocks: list[Block] = []
order = 0
body = estimate_body_size(working) if working else 12.0
detect_tables = get_options().detect_tables
table_regs = sorted(
[r for r in regions if r.label == "table"] if detect_tables else [],
key=lambda r: r.reading_index,
)
other_regs = sorted(
[r for r in regions if r.label != "table"],
key=lambda r: r.reading_index,
)
# --- Pass A: explicit ML table regions (relaxed grid builder) ---
for reg in table_regs:
idxs = [
i
for i, ln in enumerate(working)
if i not in assigned and _overlap_line(reg.bbox_pdf, ln, y_pad=10.0)
]
# Expand vertically: include nearby unassigned lines inside region Y span
y0, y1 = reg.bbox_pdf.y - 6.0, reg.bbox_pdf.y + reg.bbox_pdf.h + 6.0
for i, ln in enumerate(working):
if i in assigned or i in idxs:
continue
if y0 <= ln.y <= y1 and not (ln.x1 < reg.bbox_pdf.x or ln.x0 > reg.bbox_pdf.x + reg.bbox_pdf.w):
idxs.append(i)
idxs = sorted(set(idxs))
reg_lines = [working[i] for i in idxs]
for i in idxs:
assigned.add(i)
if not reg_lines:
# An empty table is not a table. It reached the document as a
# bordered box around nothing, and it displaced the grid that OCR
# would have built from the same rectangle.
warnings.append(
f"Page {page_index + 1}: ML table region with no text lines; skipped."
)
continue
# Optional structure model: column separators for THIS region only.
# None (disabled, no weights, bad output) leaves the heuristics alone.
region_rulings = _structure_rulings(
reg,
page_png=page_png,
page_width=page_width,
page_height=float(page_height or 0.0),
page_index=page_index,
vertical_rulings=vertical_rulings,
warnings=warnings,
)
table_blocks, remaining, conf = extract_tables(
order_lines(reg_lines, page_width),
start_order=order,
vertical_rulings=region_rulings if region_rulings is not None else vertical_rulings,
page_rects=page_rects,
path_ops=path_ops,
ml_hinted=True,
)
# Require a real multi-row grid; weak 1-row hits release lines for Pass B.
strong = bool(
table_blocks
and table_blocks[0].cells
and len(table_blocks[0].cells) >= 2
and max(len(r) for r in table_blocks[0].cells) >= 2
)
if strong:
for tb in table_blocks:
tb.reading_order = order
tb.bbox = reg.bbox_pdf
if tb.table_confidence < 0.5:
tb.table_confidence = max(tb.table_confidence, float(reg.score or 0.5))
blocks.append(tb)
order += 1
para_blocks = _paragraph_blocks(remaining, order, body, page_width)
blocks.extend(para_blocks)
order += len(para_blocks)
warnings.append(
f"Page {page_index + 1}: ML table->grid rows={len(table_blocks[0].cells)} "
f"cols={len(table_blocks[0].cells[0])} conf={conf:.2f}."
)
else:
# Release region lines so page-level heuristics can still build a grid
for i in idxs:
assigned.discard(i)
warnings.append(
f"Page {page_index + 1}: ML table region grid_failed conf={conf:.2f}; "
"deferring to heuristic."
)
# --- Pass B: page-level heuristic tables on unassigned lines ---
# Fixes ML that labels each table row as a separate "text" region.
leftover = [working[i] for i in range(len(working)) if i not in assigned]
if leftover and detect_tables:
table_blocks, remaining, conf = extract_tables(
order_lines(leftover, page_width),
start_order=order,
vertical_rulings=vertical_rulings,
page_rects=page_rects,
path_ops=path_ops,
ml_hinted=False,
)
if table_blocks and conf >= 0.60 and table_blocks[0].cells:
remaining_ids = {id(ln) for ln in remaining}
leftover_ids = {id(ln) for ln in leftover}
for i, ln in enumerate(working):
if id(ln) in leftover_ids and id(ln) not in remaining_ids:
assigned.add(i)
for tb in table_blocks:
tb.reading_order = order
blocks.append(tb)
order += 1
warnings.append(
f"Page {page_index + 1}: heuristic table under ML regions "
f"rows={len(table_blocks[0].cells)} conf={conf:.2f}."
)
# --- Pass C: non-table ML regions for typing ---
for reg in other_regs:
idxs = [
i
for i, ln in enumerate(working)
if i not in assigned and _overlap_line(reg.bbox_pdf, ln, y_pad=4.0)
]
reg_lines = [working[i] for i in idxs]
for i in idxs:
assigned.add(i)
btype = _LABEL_TO_BLOCK.get(reg.label, BlockType.paragraph)
if not reg_lines:
if btype in (BlockType.figure, BlockType.header, BlockType.footer):
blocks.append(
Block(type=btype, text="", bbox=reg.bbox_pdf, reading_order=order)
)
order += 1
continue
region_blocks = _paragraph_blocks(
order_lines(reg_lines, page_width),
order,
body,
page_width,
force_type=btype,
)
blocks.extend(region_blocks)
order += len(region_blocks)
# --- Pass D: true leftovers ---
leftover = [working[i] for i in range(len(working)) if i not in assigned]
if leftover:
ordered = order_lines(leftover, page_width)
table_blocks, remaining, conf = ([], ordered, 0.0)
if detect_tables:
table_blocks, remaining, conf = extract_tables(
ordered,
start_order=order,
vertical_rulings=vertical_rulings,
page_rects=page_rects,
path_ops=path_ops,
)
if table_blocks and conf >= 0.65 and table_blocks[0].cells:
for tb in table_blocks:
tb.reading_order = order
blocks.append(tb)
order += 1
para_blocks = _paragraph_blocks(remaining, order, body, page_width)
blocks.extend(para_blocks)
order += len(para_blocks)
else:
para_blocks = _paragraph_blocks(ordered, order, body, page_width)
blocks.extend(para_blocks)
order += len(para_blocks)
if kind == PageKind.blank and not blocks:
warnings.append(f"Page {page_index + 1}: ML regions produced no blocks.")
# Order blocks by page reading order (XY-cut of text lines) and geometry,
# rather than the order of detector passes (which previously put tables before titles).
ordered_page_lines = order_lines(lines, page_width, page_height=float(page_height or 792.0))
def _block_sort_key(b: Block) -> tuple:
if b.type == BlockType.header:
return (0, 0, 0.0, 0.0)
if b.type == BlockType.footer:
return (2, 0, 0.0, 0.0)
box = b.bbox
if box is not None:
inside = [
i
for i, ln in enumerate(ordered_page_lines)
if box.x - 3.0 <= (ln.x0 + ln.x1) / 2.0 <= box.x + box.w + 3.0
and box.y - 3.0 <= ln.y <= box.y + box.h + 3.0
]
if inside:
return (1, 0, min(inside), 0.0)
top_y = box.y + box.h
return (1, 1, -top_y, box.x)
return (1, 2, b.reading_order, 0.0)
blocks.sort(key=_block_sort_key)
for i, b in enumerate(blocks):
b.reading_order = i
return blocks
def ml_header_footer_hints(regions: list[Region]) -> tuple[list[BBox], list[BBox]]:
headers = [r.bbox_pdf for r in regions if r.label == "header"]
footers = [r.bbox_pdf for r in regions if r.label == "footer"]
return headers, footers
gateway/app/services/convert/layout/ml_regions.py
+431
View File
@@ -0,0 +1,431 @@
"""Optional Apache ONNX layout region detector (fail-open)."""
from __future__ import annotations
import io
import os
import threading
from dataclasses import dataclass
from pathlib import Path
from app.services import model_paths
from app.services.convert.idm.model import BBox
from app.services.convert.layout.coords import PageGeom, pixel_bbox_to_pdf
from app.services.convert.validation import run_with_timeout
_lock = threading.Lock()
_session = None
_session_path: str | None = None
_load_error: str | None = None
# PP-DocLayoutV3 (Paddle) class list — see models/layout inference.yml / HF card.
_PP_DOCLAYOUT_V3_LABELS = (
"abstract",
"algorithm",
"aside_text",
"chart",
"content",
"display_formula",
"doc_title",
"figure_title",
"footer",
"footer_image",
"footnote",
"formula_number",
"header",
"header_image",
"image",
"inline_formula",
"number",
"paragraph_title",
"reference",
"reference_content",
"seal",
"table",
"text",
"vertical_text",
"vision_footnote",
)
# Map raw detector labels → DocQube Region.label
_LABEL_MAP = {
0: "text",
1: "title",
2: "list",
3: "table",
4: "figure",
5: "header",
6: "footer",
"text": "text",
"title": "title",
"list": "list",
"table": "table",
"figure": "figure",
"header": "header",
"footer": "footer",
"Text": "text",
"Title": "title",
"List": "list",
"Table": "table",
"Figure": "figure",
"Header": "header",
"Footer": "footer",
# PP-DocLayoutV3 names
"abstract": "text",
"algorithm": "text",
"aside_text": "text",
"chart": "figure",
"content": "text",
"display_formula": "text",
"doc_title": "title",
"figure_title": "title",
"footer_image": "figure",
"footnote": "footer",
"formula_number": "text",
"header_image": "figure",
"image": "figure",
"inline_formula": "text",
"number": "text",
"paragraph_title": "title",
"reference": "text",
"reference_content": "text",
"seal": "figure",
"vertical_text": "text",
"vision_footnote": "footer",
}
# Default input size for PP-DocLayoutV3 ONNX (inference.yml: Resize 800x800, keep_ratio false)
_DEFAULT_TARGET = 800
@dataclass
class Region:
label: str
bbox_pdf: BBox
score: float
reading_index: int = 0
def layout_ml_enabled() -> bool:
return os.environ.get("CONVERT_LAYOUT_ML", "0").strip().lower() in ("1", "true", "yes", "on")
def _weights_path() -> Path:
explicit = (os.environ.get("CONVERT_LAYOUT_ML_WEIGHTS") or "").strip()
if explicit:
return model_paths.resolve(explicit)
int8_cand = model_paths.resolve("models/layout/v1/layout_int8.onnx")
if int8_cand.is_file():
return int8_cand
return model_paths.resolve("models/layout/v1/layout.onnx")
def _min_score() -> float:
try:
return float(os.environ.get("CONVERT_LAYOUT_ML_MIN_SCORE", "0.5"))
except ValueError:
return 0.5
def _device() -> str:
return os.environ.get("CONVERT_LAYOUT_ML_DEVICE", "cpu").strip().lower()
def _page_timeout() -> float:
try:
return float(os.environ.get("CONVERT_LAYOUT_ML_PAGE_TIMEOUT", "15"))
except ValueError:
return 15.0
def _target_size() -> int:
try:
return max(64, int(os.environ.get("CONVERT_LAYOUT_ML_INPUT_SIZE", str(_DEFAULT_TARGET))))
except ValueError:
return _DEFAULT_TARGET
def _get_session():
global _session, _session_path, _load_error
path = str(_weights_path())
with _lock:
if _session is not None and _session_path == path:
return _session
if _load_error and _session_path == path:
return None
try:
import onnxruntime as ort
if not Path(path).is_file():
_load_error = f"weights missing: {path}"
_session = None
_session_path = path
return None
sess_options = ort.SessionOptions()
sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
# Limit ONNX intra-op threads to prevent CPU starvation for concurrent requests
max_threads = max(1, min(4, os.cpu_count() or 2))
sess_options.intra_op_num_threads = max_threads
providers = ["CPUExecutionProvider"]
if _device() == "cuda":
providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
_session = ort.InferenceSession(path, sess_options=sess_options, providers=providers)
_session_path = path
_load_error = None
return _session
except Exception as exc:
_load_error = str(exc)
_session = None
_session_path = path
return None
def reset_session_for_tests() -> None:
global _session, _session_path, _load_error
with _lock:
_session = None
_session_path = None
_load_error = None
def last_load_error() -> str | None:
return _load_error
def _map_label(cls: int | str) -> str:
if isinstance(cls, (int, float)):
idx = int(cls)
if 0 <= idx < len(_PP_DOCLAYOUT_V3_LABELS):
name = _PP_DOCLAYOUT_V3_LABELS[idx]
return _LABEL_MAP.get(name, "text")
return _LABEL_MAP.get(idx, "text")
return _LABEL_MAP.get(str(cls), _LABEL_MAP.get(cls, "text"))
def preprocess_pp_doclayout(
rgb_hwc: np.ndarray,
*,
target_size: int = _DEFAULT_TARGET,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
PP-DocLayoutV3 preprocess (matches official inference.yml):
Resize 800x800 keep_ratio=false, NormalizeImage mean=0 std=1 (÷255), Permute NCHW.
Returns (image NCHW float32, im_shape [1,2], scale_factor [1,2] as [scale_h, scale_w]).
"""
import numpy as np
from PIL import Image
if rgb_hwc.ndim != 3 or rgb_hwc.shape[2] != 3:
raise ValueError(f"expected HWC RGB, got shape {getattr(rgb_hwc, 'shape', None)}")
orig_h, orig_w = int(rgb_hwc.shape[0]), int(rgb_hwc.shape[1])
target = int(target_size)
if rgb_hwc.dtype != np.uint8:
# Accept float 0-1 or 0-255
scaled = rgb_hwc * 255.0 if float(np.nanmax(rgb_hwc)) <= 1.5 else rgb_hwc
img = Image.fromarray(np.clip(scaled, 0, 255).astype(np.uint8))
else:
img = Image.fromarray(rgb_hwc)
resized = img.resize((target, target), Image.BILINEAR)
blob = np.asarray(resized).astype("float32") / 255.0
image = np.transpose(blob, (2, 0, 1))[None, ...].astype("float32")
scale_h = float(target) / float(orig_h)
scale_w = float(target) / float(orig_w)
im_shape = np.array([[float(target), float(target)]], dtype=np.float32)
scale_factor = np.array([[scale_h, scale_w]], dtype=np.float32)
return image, im_shape, scale_factor
def build_onnx_feed(sess, rgb_hwc: np.ndarray, *, target_size: int | None = None) -> dict:
"""
Build ORT feed dict from session input metadata.
PP-DocLayoutV3 expects: im_shape, image, scale_factor.
Simpler single-input models get NCHW (or NHWC if shape[-1]==3) image only.
"""
import numpy as np
t = _target_size() if target_size is None else int(target_size)
image, im_shape, scale_factor = preprocess_pp_doclayout(rgb_hwc, target_size=t)
inputs = list(sess.get_inputs())
names = {i.name for i in inputs}
feed: dict = {}
# Named PP-DocLayout / Paddle Detection I/O
if "image" in names or "scale_factor" in names or "im_shape" in names:
if "image" in names:
feed["image"] = image
if "im_shape" in names:
feed["im_shape"] = im_shape
if "scale_factor" in names:
feed["scale_factor"] = scale_factor
# Fill any remaining unknown inputs best-effort
for inp in inputs:
if inp.name in feed:
continue
shape = inp.shape
if len(shape) == 4:
if shape[-1] == 3:
feed[inp.name] = np.transpose(image, (0, 2, 3, 1))
else:
feed[inp.name] = image
elif len(shape) == 2:
feed[inp.name] = im_shape
return feed
# Single (or primary) image tensor models
if not inputs:
return feed
name = inputs[0].name
shape = inputs[0].shape
if len(shape) == 4 and shape[-1] == 3:
feed[name] = np.transpose(image, (0, 2, 3, 1))
else:
feed[name] = image
return feed
def _parse_pp_doclayout_boxes(boxes: np.ndarray, geom: PageGeom, min_score: float) -> list[Region]:
"""Parse PP-DocLayoutV3 rows: [cls, score, x1, y1, x2, y2, read_order] (pixel space)."""
import numpy as np
regions: list[Region] = []
a = np.asarray(boxes)
if a.ndim != 2 or a.shape[-1] < 6:
return regions
for row in a:
row = [float(x) for x in row]
n = len(row)
# 7-col: cls, score, xyxy, order | 8-col: img_idx, cls, score, xyxy, order
if n >= 8 and row[2] <= 1.0 and row[1] < 50:
cls, sc, x1, y1, x2, y2 = int(row[1]), row[2], row[3], row[4], row[5], row[6]
order = int(row[7]) if n > 7 else len(regions)
elif n >= 7 and row[1] <= 1.5:
cls, sc, x1, y1, x2, y2 = int(row[0]), row[1], row[2], row[3], row[4], row[5]
order = int(row[6]) if n > 6 else len(regions)
elif n >= 6 and row[4] <= 1.0 and row[5] < 50:
x1, y1, x2, y2, sc, cls = row[0], row[1], row[2], row[3], row[4], int(row[5])
order = len(regions)
else:
continue
if sc < min_score:
continue
if x2 < x1 or y2 < y1:
x2, y2 = x1 + max(0.0, x2), y1 + max(0.0, y2)
if max(x1, y1, x2, y2) <= 1.5:
x1 *= geom.pixel_w
x2 *= geom.pixel_w
y1 *= geom.pixel_h
y2 *= geom.pixel_h
label = _map_label(cls)
bbox = pixel_bbox_to_pdf(float(x1), float(y1), float(x2), float(y2), geom)
regions.append(Region(label=label, bbox_pdf=bbox, score=float(sc), reading_index=order))
regions.sort(key=lambda r: (r.reading_index, -(r.bbox_pdf.y + r.bbox_pdf.h), r.bbox_pdf.x))
for i, r in enumerate(regions):
r.reading_index = i
return regions
def _parse_outputs(outs: list, geom: PageGeom, min_score: float) -> list[Region]:
"""Best-effort parse of common detection outputs (boxes, scores, labels)."""
import numpy as np
# Prefer PP-DocLayout-style (N, 6|7|8) float matrix
for arr in outs:
a = np.asarray(arr)
if a.ndim == 2 and a.shape[-1] in (6, 7, 8) and a.dtype.kind == "f":
parsed = _parse_pp_doclayout_boxes(a, geom, min_score)
if parsed:
return parsed
boxes = scores = labels = None
for arr in outs:
a = np.asarray(arr)
if a.ndim >= 2 and a.shape[-1] >= 4 and boxes is None and a.dtype.kind == "f":
boxes = a.reshape(-1, a.shape[-1])
elif a.ndim == 1 and a.dtype != object and scores is None and a.size > 0:
if a.dtype.kind == "f":
scores = a.reshape(-1)
else:
labels = a.reshape(-1)
elif a.ndim == 2 and a.shape[0] == 1 and a.shape[1] > 4 and a.dtype.kind == "f":
boxes = a.reshape(-1, a.shape[-1])
if boxes is None:
return []
# Reuse PP parser when shape matches; else legacy path
if boxes.shape[-1] in (6, 7, 8):
return _parse_pp_doclayout_boxes(boxes, geom, min_score)
regions: list[Region] = []
for i, row in enumerate(boxes):
row = list(row)
if len(row) < 4:
continue
if len(row) >= 6:
if row[4] <= 1.0 and row[5] < 20:
x1, y1, x2, y2, sc, cls = row[0], row[1], row[2], row[3], row[4], int(row[5])
else:
sc, cls, x1, y1, x2, y2 = row[0], int(row[1]), row[2], row[3], row[4], row[5]
else:
x1, y1, x2, y2 = row[0], row[1], row[2], row[3]
sc = float(scores[i]) if scores is not None and i < len(scores) else 1.0
cls = int(labels[i]) if labels is not None and i < len(labels) else 0
if sc < min_score:
continue
if x2 < x1 or y2 < y1:
x2, y2 = x1 + max(0.0, x2), y1 + max(0.0, y2)
if max(x1, y1, x2, y2) <= 1.5:
x1 *= geom.pixel_w
x2 *= geom.pixel_w
y1 *= geom.pixel_h
y2 *= geom.pixel_h
label = _map_label(cls)
bbox = pixel_bbox_to_pdf(float(x1), float(y1), float(x2), float(y2), geom)
regions.append(Region(label=label, bbox_pdf=bbox, score=float(sc), reading_index=len(regions)))
regions.sort(key=lambda r: (-(r.bbox_pdf.y + r.bbox_pdf.h), r.bbox_pdf.x))
for i, r in enumerate(regions):
r.reading_index = i
return regions
def detect_regions(
png_bytes: bytes,
page_width: float,
page_height: float,
*,
dpi: int | None = None,
) -> tuple[list[Region], str | None]:
"""
Returns (regions, error_or_none).
Empty regions + error ⇒ caller must fail-open to heuristics.
"""
from app.services.convert.layout.page_raster import layout_ml_dpi
d = dpi if dpi is not None else layout_ml_dpi()
geom = PageGeom(page_width=page_width, page_height=page_height, dpi=float(d))
sess = _get_session()
if sess is None:
return [], _load_error or "layout ML session unavailable"
try:
import numpy as np
from PIL import Image
img = Image.open(io.BytesIO(png_bytes)).convert("RGB")
arr = np.asarray(img)
feed = build_onnx_feed(sess, arr)
def _run():
with _lock:
return sess.run(None, feed)
outs = run_with_timeout(_run, _page_timeout(), label="layout ML infer")
regions = _parse_outputs(list(outs), geom, _min_score())
return regions, None
except Exception as exc:
return [], str(exc)
gateway/app/services/convert/layout/ml_table_structure.py
+401
View File
@@ -0,0 +1,401 @@
"""Optional table-structure model (fail-open) — column/row separators only.
The heuristics in :mod:`layout.tables` recover a grid from token x-clustering
and from ruled rectangles. They are good at ruled tables and at tables whose
columns are separated by real gutters. They are weak exactly where a human
also has to squint: a borderless table whose columns are close together, or one
where a long cell in row 3 closes the gutter that rows 1, 2 and 4 make obvious.
A structure model sees the whole table at once and answers the one question the
heuristics get wrong: *where are the column boundaries*. That answer feeds the
existing grid builder as ``vertical_rulings`` — the same input a ruled table
already supplies — so the model never builds cells itself and can never invent a
table where the heuristics found none. It narrows an existing decision; it does
not make a new one.
Licensing (the reason this is a hook rather than a bundled model): the intended
weights are Microsoft's Table Transformer structure model (MIT) or PaddleOCR's
SLANet (Apache-2.0). Nothing is downloaded and nothing is vendored — the
operator points ``CONVERT_LAYOUT_ML_TABLE_STRUCTURE_WEIGHTS`` at a file they
chose.
**Fail-open is the contract.** Disabled, weights missing, onnxruntime missing,
inference error, timeout, or output that fails the plausibility check — every
one of those returns "no rulings" and the heuristic path runs exactly as it does
today. This module can make table detection sharper; it can never be the reason
a conversion is worse or fails.
"""
from __future__ import annotations
import io
import os
import threading
from dataclasses import dataclass, field
from pathlib import Path
from app.services import model_paths
from app.services.convert.idm.model import BBox
from app.services.convert.layout.coords import PageGeom, pdf_bbox_to_pixel
from app.services.convert.validation import run_with_timeout
_lock = threading.Lock()
_session = None
_session_path: str | None = None
_load_error: str | None = None
# Table Transformer (TATR) structure classes, in label order.
_TATR_LABELS = (
"table",
"table column",
"table row",
"table column header",
"table projected row header",
"table spanning cell",
)
# A table with more columns than this is a parse error, not a table.
_MAX_COLUMNS = 40
_MAX_ROWS = 200
# Two separators closer together than this are the same separator seen twice.
_MIN_SEPARATOR_GAP_POINTS = 4.0
# A column strip thinner than this is noise from a low-confidence box.
_MIN_COLUMN_WIDTH_POINTS = 6.0
@dataclass
class TableStructure:
"""Column and row separator positions, in PDF points on the page."""
column_x: list[float] = field(default_factory=list)
row_y: list[float] = field(default_factory=list)
score: float = 0.0
@property
def columns(self) -> int:
# n separators between the outer edges ⇒ n-1 columns.
return max(0, len(self.column_x) - 1)
@property
def rows(self) -> int:
return max(0, len(self.row_y) - 1)
def table_structure_enabled() -> bool:
return os.environ.get("CONVERT_LAYOUT_ML_TABLE_STRUCTURE", "0").strip().lower() in (
"1",
"true",
"yes",
"on",
)
def _weights_path() -> Path:
return model_paths.from_env(
"CONVERT_LAYOUT_ML_TABLE_STRUCTURE_WEIGHTS", "models/layout/table/structure.onnx"
)
def _min_score() -> float:
try:
return float(os.environ.get("CONVERT_LAYOUT_ML_TABLE_STRUCTURE_MIN_SCORE", "0.5"))
except ValueError:
return 0.5
def _timeout() -> float:
try:
return float(os.environ.get("CONVERT_LAYOUT_ML_TABLE_STRUCTURE_TIMEOUT", "10"))
except ValueError:
return 10.0
def _input_size() -> int:
try:
return max(64, int(os.environ.get("CONVERT_LAYOUT_ML_TABLE_STRUCTURE_INPUT_SIZE", "800")))
except ValueError:
return 800
def _device() -> str:
return os.environ.get("CONVERT_LAYOUT_ML_DEVICE", "cpu").strip().lower()
def _get_session():
"""ONNX session, or None with :func:`last_load_error` explaining why."""
global _session, _session_path, _load_error
path = str(_weights_path())
with _lock:
if _session is not None and _session_path == path:
return _session
if _load_error and _session_path == path:
return None
try:
if not Path(path).is_file():
_load_error = f"table structure weights missing: {path}"
_session = None
_session_path = path
return None
import onnxruntime as ort
providers = ["CPUExecutionProvider"]
if _device() == "cuda":
providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
_session = ort.InferenceSession(path, providers=providers)
_session_path = path
_load_error = None
return _session
except Exception as exc:
_load_error = str(exc)
_session = None
_session_path = path
return None
def reset_session_for_tests() -> None:
global _session, _session_path, _load_error
with _lock:
_session = None
_session_path = None
_load_error = None
def last_load_error() -> str | None:
return _load_error
def _label_name(cls: int | str) -> str:
if isinstance(cls, (int, float)):
idx = int(cls)
if 0 <= idx < len(_TATR_LABELS):
return _TATR_LABELS[idx]
return "unknown"
return str(cls).strip().lower().replace("_", " ")
def _crop_png(png_bytes: bytes, bbox: BBox, geom: PageGeom) -> tuple[bytes, float, float]:
"""Crop the table's rectangle out of the page raster.
Returns the crop plus its top-left pixel offset on the page, so boxes the
model reports inside the crop can be put back on the page.
"""
from PIL import Image
img = Image.open(io.BytesIO(png_bytes)).convert("RGB")
x0, y0, x1, y1 = pdf_bbox_to_pixel(bbox, geom)
x0 = max(0.0, min(float(img.width), x0))
x1 = max(0.0, min(float(img.width), x1))
y0 = max(0.0, min(float(img.height), y0))
y1 = max(0.0, min(float(img.height), y1))
if x1 - x0 < 8 or y1 - y0 < 8:
raise ValueError("table region too small to crop")
left, top = int(x0), int(y0)
crop = img.crop((left, top, int(x1), int(y1)))
buf = io.BytesIO()
crop.save(buf, format="PNG")
# The offset returned is the one the crop actually used, not the unrounded
# float: a half-pixel disagreement here shifts every separator on the page.
return buf.getvalue(), float(left), float(top)
def _build_feed(sess, png_bytes: bytes) -> dict:
"""Feed dict from session input metadata — DETR-style and plain-NCHW both."""
import numpy as np
from PIL import Image
target = _input_size()
img = Image.open(io.BytesIO(png_bytes)).convert("RGB").resize((target, target), Image.BILINEAR)
blob = (np.asarray(img).astype("float32") / 255.0).transpose(2, 0, 1)[None, ...]
feed: dict = {}
for inp in sess.get_inputs():
shape = list(inp.shape)
if len(shape) == 4:
feed[inp.name] = (
np.transpose(blob, (0, 2, 3, 1)) if shape[-1] == 3 else blob
).astype("float32")
elif len(shape) == 2:
# DETR exports take the padding mask or the original size here.
feed[inp.name] = np.array([[float(target), float(target)]], dtype="float32")
return feed
def _boxes_from_outputs(outs: list, min_score: float) -> list[tuple[str, float, float, float, float, float]]:
"""(label, score, x0, y0, x1, y1) in crop-normalised or crop-pixel units."""
import numpy as np
rows: list[tuple[str, float, float, float, float, float]] = []
for arr in outs:
a = np.asarray(arr)
if a.dtype.kind != "f" or a.ndim < 2:
continue
m = a.reshape(-1, a.shape[-1])
if m.shape[-1] < 6:
continue
for row in m:
row = [float(v) for v in row]
# cls, score, xyxy | xyxy, score, cls
if row[1] <= 1.0 and row[0] < len(_TATR_LABELS) + 1:
cls, sc, x0, y0, x1, y1 = row[0], row[1], row[2], row[3], row[4], row[5]
elif row[4] <= 1.0 and row[5] < len(_TATR_LABELS) + 1:
x0, y0, x1, y1, sc, cls = row[0], row[1], row[2], row[3], row[4], row[5]
else:
continue
if sc < min_score:
continue
if x1 < x0 or y1 < y0:
continue
rows.append((_label_name(cls), float(sc), x0, y0, x1, y1))
if rows:
break
return rows
def _separators_from_strips(
strips: list[tuple[float, float]], *, min_gap: float, min_size: float
) -> list[float]:
"""Column/row *strips* → the separator positions between them.
The model reports each column as a rectangle. What the grid builder wants is
the boundaries, so adjacent strips are bridged at the middle of the space
between them and the outer edges are kept.
"""
usable = [(a, b) for a, b in sorted(strips) if b - a >= min_size]
if len(usable) < 2:
return []
seps = [usable[0][0]]
for (_, prev_end), (next_start, _) in zip(usable, usable[1:], strict=False):
seps.append((prev_end + next_start) / 2.0 if next_start >= prev_end else next_start)
seps.append(usable[-1][1])
out: list[float] = []
for s in seps:
if not out or s - out[-1] >= min_gap:
out.append(s)
else:
out[-1] = (out[-1] + s) / 2.0
return out
def structure_for_region(
png_bytes: bytes,
bbox: BBox,
page_width: float,
page_height: float,
*,
dpi: int | None = None,
) -> tuple[TableStructure | None, str | None]:
"""Column/row separators for one table region, in PDF points.
Returns ``(None, reason)`` whenever the caller must fall back to the
heuristics — which is every failure mode, by design.
"""
if not table_structure_enabled():
return None, None
if not png_bytes:
return None, "no page raster"
from app.services.convert.layout.page_raster import layout_ml_dpi
d = float(dpi if dpi is not None else layout_ml_dpi())
geom = PageGeom(page_width=page_width, page_height=page_height, dpi=d)
sess = _get_session()
if sess is None:
return None, _load_error or "table structure session unavailable"
try:
crop, off_x, off_y = _crop_png(png_bytes, bbox, geom)
feed = _build_feed(sess, crop)
if not feed:
return None, "table structure model has no image input"
def _run():
with _lock:
return sess.run(None, feed)
outs = run_with_timeout(_run, _timeout(), label="table structure infer")
boxes = _boxes_from_outputs(list(outs), _min_score())
if not boxes:
return None, "table structure produced no boxes"
except Exception as exc:
return None, str(exc)
# The model works in the crop's own frame; put it back on the page.
from PIL import Image
crop_img = Image.open(io.BytesIO(crop))
crop_w, crop_h = float(crop_img.width), float(crop_img.height)
scale = geom.scale
col_strips: list[tuple[float, float]] = []
row_strips: list[tuple[float, float]] = []
best = 0.0
for label, score, x0, y0, x1, y1 in boxes:
if max(x0, y0, x1, y1) <= 1.5: # normalised
x0, x1 = x0 * crop_w, x1 * crop_w
y0, y1 = y0 * crop_h, y1 * crop_h
if label in ("table column", "table column header"):
if label == "table column":
col_strips.append((x0, x1))
best = max(best, score)
elif label == "table row":
row_strips.append((y0, y1))
best = max(best, score)
col_px = _separators_from_strips(
col_strips,
min_gap=_MIN_SEPARATOR_GAP_POINTS * scale,
min_size=_MIN_COLUMN_WIDTH_POINTS * scale,
)
row_px = _separators_from_strips(
row_strips,
min_gap=_MIN_SEPARATOR_GAP_POINTS * scale,
min_size=_MIN_SEPARATOR_GAP_POINTS * scale,
)
if len(col_px) < 3:
# Fewer than three separators means fewer than two columns: nothing the
# heuristics could not already see.
return None, "table structure found no usable columns"
if len(col_px) - 1 > _MAX_COLUMNS or len(row_px) - 1 > _MAX_ROWS:
return None, "table structure output implausible"
# Crop pixels → page pixels → PDF points. x needs no flip; y is handled by
# the callers that need rows, and is reported top-down from the page top.
column_x = [(off_x + px) / scale for px in col_px]
row_y = [page_height - ((off_y + py) / scale) for py in row_px]
left, right = bbox.x - 4.0, bbox.x + bbox.w + 4.0
column_x = [x for x in column_x if left <= x <= right]
if len(column_x) < 3:
return None, "table structure columns fell outside the region"
return TableStructure(column_x=column_x, row_y=row_y, score=best), None
def merge_rulings(
heuristic: list[float] | None, structure: list[float] | None, *, tolerance: float = 3.0
) -> list[float]:
"""Union of ruled-line x positions and model separators, near-duplicates fused.
A real ruling is measured from the page and is always right; a model
separator is a guess about a gutter. Where they agree the ruling wins the
position, so enabling the model can never move a line that was actually
drawn on the page.
"""
out = sorted(float(x) for x in (heuristic or []))
for x in sorted(float(x) for x in (structure or [])):
if not any(abs(x - kept) <= tolerance for kept in out):
out.append(x)
out.sort()
return out
__all__ = [
"TableStructure",
"last_load_error",
"merge_rulings",
"reset_session_for_tests",
"structure_for_region",
"table_structure_enabled",
]
gateway/app/services/convert/layout/page_classify.py
+129
View File
@@ -0,0 +1,129 @@
"""Page classification: digital | scan | hybrid | blank (hardened)."""
from __future__ import annotations
from itertools import pairwise
from app.services.convert.idm.model import PageKind
def classify_page(
*,
glyph_count: int,
text_chars: int,
width: float,
height: float,
image_coverage: float = 0.0,
glyph_size_variance: float = 0.0,
) -> PageKind:
"""
Classify a page.
image_coverage: fraction of page area covered by full-page-ish images (0..1).
glyph_size_variance: variance of glyph font sizes (high + sparse text → hybrid/scan).
"""
area = max(width * height, 1.0)
density = text_chars / area
if text_chars < 8 and glyph_count < 8:
if image_coverage >= 0.35:
return PageKind.scan
return PageKind.blank if text_chars == 0 else PageKind.scan
# Large image backdrop with some digital text → hybrid (OCR empty regions only)
if image_coverage >= 0.45 and text_chars >= 20:
return PageKind.hybrid
if image_coverage >= 0.55 and text_chars < 40:
return PageKind.scan
if glyph_count >= 40 or text_chars >= 80 or density > 0.00015:
# High size variance with modest glyphs can still be hybrid
if glyph_size_variance > 40.0 and image_coverage >= 0.25:
return PageKind.hybrid
return PageKind.digital
# Short digital pages (title-only / few lines) with real glyphs and no backdrop
if glyph_count >= 8 and image_coverage < 0.25:
return PageKind.digital
if text_chars < 40:
return PageKind.scan
return PageKind.hybrid
def estimate_image_coverage(image_blocks: list[dict] | None, width: float, height: float) -> float:
"""Fraction of the page covered by images, in [0, 1].
When the blocks carry real positions the *union* area is measured, not the
sum: a scan is often drawn as several overlapping strips, and summing them
exceeds the page area and pins coverage at 1.0. Since coverage decides
whether a page is classified as a scan — and therefore whether it takes the
expensive OCR path — over-counting sends text pages to OCR needlessly.
Blocks without positions fall back to the summed estimate.
"""
if not image_blocks or width <= 0 or height <= 0:
return 0.0
page_area = width * height
rects: list[tuple[float, float, float, float]] = []
summed = 0.0
positioned = True
for img in image_blocks:
w = float(img.get("w") or img.get("width") or 0)
h = float(img.get("h") or img.get("height") or 0)
w, h = max(0.0, w), max(0.0, h)
summed += w * h
if "x" not in img or "y" not in img:
positioned = False
continue
x, y = float(img["x"]), float(img["y"])
if w > 0 and h > 0:
rects.append((x, y, x + w, y + h))
if not positioned or not rects:
return min(1.0, summed / page_area)
return min(1.0, _union_area(rects) / page_area)
def _union_area(rects: list[tuple[float, float, float, float]]) -> float:
"""Area of the union of axis-aligned rectangles, by coordinate sweep.
Exact rather than approximate, and cheap at the handful of images a PDF
page carries.
"""
xs = sorted({v for r in rects for v in (r[0], r[2])})
if len(xs) < 2:
return 0.0
total = 0.0
for left, right in pairwise(xs):
strip_w = right - left
if strip_w <= 0:
continue
spans = sorted(
(r[1], r[3]) for r in rects if r[0] <= left and r[2] >= right and r[3] > r[1]
)
covered = 0.0
cur_lo = cur_hi = None
for lo, hi in spans:
if cur_hi is None or lo > cur_hi:
if cur_hi is not None:
covered += cur_hi - cur_lo
cur_lo, cur_hi = lo, hi
elif hi > cur_hi:
cur_hi = hi
if cur_hi is not None:
covered += cur_hi - cur_lo
total += strip_w * covered
return total
def glyph_size_variance(glyphs: list[dict] | None) -> float:
if not glyphs:
return 0.0
sizes = [float(g.get("fontSize") or g.get("h") or 0) for g in glyphs if g]
sizes = [s for s in sizes if s > 0]
if len(sizes) < 2:
return 0.0
mean = sum(sizes) / len(sizes)
return sum((s - mean) ** 2 for s in sizes) / len(sizes)
gateway/app/services/convert/layout/page_raster.py
+31
View File
@@ -0,0 +1,31 @@
"""Rasterize a PDF page to PNG for layout ML / OCR crops."""
from __future__ import annotations
import os
from app.services.convert.pdf_bridge import render_page_png
from app.services.convert.validation import run_with_timeout
def layout_ml_dpi(default: float = 150.0) -> int:
try:
return max(72, int(float(os.environ.get("CONVERT_LAYOUT_ML_DPI", str(default)))))
except ValueError:
return int(default)
def raster_page(
pdf_bytes: bytes,
page_index: int,
*,
dpi: int | None = None,
timeout: float = 15.0,
) -> bytes:
"""Return PNG bytes. Raises on hard failure (caller should fail-open)."""
d = dpi if dpi is not None else layout_ml_dpi()
return run_with_timeout(
lambda: render_page_png(pdf_bytes, page_index, dpi=d, allow_blank=True),
timeout,
label=f"layout raster page {page_index + 1}",
)
gateway/app/services/convert/layout/paragraphs.py
+485
View File
@@ -0,0 +1,485 @@
"""Line → paragraph reconstruction for the deterministic (non-ML) layout path.
A PDF has no paragraph concept: it has positioned glyphs that ``glyphs.py``
clusters into visual lines. Emitting one block per visual line produces a
DOCX where every line is its own ``<w:p>`` — text is present but the document
does not reflow, so it cannot be edited. This module groups consecutive lines
back into paragraphs using the geometric signals a typesetter used to create
them.
A line ends its paragraph when any of these holds:
* the vertical gap to the next line is materially larger than the body leading
(paragraph spacing);
* the line is *short* — its right edge stops well before the column's right
margin, which is how the last line of a paragraph looks;
* the next line is indented relative to the current one (first-line indent
marks a new paragraph);
* the next line starts a list item, or either line is a heading;
* the style changes materially (font family, or size beyond a tolerance);
* the flow leaves the column (the next line sits higher on the page, or its
left edge jumps by more than a column's width).
All thresholds are relative to measured page statistics — body font size,
modal leading, column right margin — so the module does not assume a page
size, a font, or a language.
"""
from __future__ import annotations
import re
import statistics
from dataclasses import dataclass
from itertools import pairwise
from app.services.convert.layout.glyphs import Line
from app.services.convert.layout.styles import TERMINAL_PUNCT_RE
# A line whose right edge falls short of the column margin by more than
# SHORT_LINE_RATIO of the column width is treated as a paragraph's last line.
SHORT_LINE_RATIO = 0.12
# Next line indented by more than this fraction of column width starts a paragraph.
INDENT_RATIO = 0.035
# Vertical gap beyond leading * PARA_GAP_RATIO separates paragraphs.
PARA_GAP_RATIO = 1.42
# Font size change beyond this fraction ends the paragraph. A line's size is
# taken from its first span (see ``glyphs``), so it is a stable number and the
# tolerance only needs to absorb text-matrix scaling drift, which is well under
# 1%. It used to be 0.18, which permitted a two-point change on 11pt body text --
# the exact distance from Word's default body to its default Heading 3 -- so a
# heading following a paragraph was absorbed into it. Type size changing between
# two lines means their roles differ; that is what this test is for. At 0.06 a
# one-point change on 11pt or 12pt text (8-9%) separates, with ample headroom
# left for scaling drift. ``idm_optimize`` imports this so the two passes that
# decide paragraph boundaries cannot disagree about what "the same size" means.
SIZE_TOLERANCE = 0.06
# Left edges within this fraction of column width count as aligned.
ALIGN_RATIO = 0.02
# ``prev`` inset from the column's left margin by more than this is not
# left-aligned: it is centred, right-aligned or a block inset. Set above any
# plausible first-line indent (a one-inch indent on a 6.5in column is 15%) so
# ordinary indented paragraphs are untouched.
INSET_LINE_RATIO = 0.20
# ruff: noqa: RUF001 — the non-ASCII characters in these patterns are the
# subject matter: CJK and Arabic sentence punctuation, and the hyphen
# variants (soft, non-breaking, unicode) that typesetters use to wrap lines.
_BULLET_RE = re.compile(r"^\s*(•|[-*–—]|○|●|◦|▪)\s+")
_NUMBER_RE = re.compile(r"^\s*(\(?\d{1,3}[.)]|\(\d{1,3}\))\s+")
_LETTER_RE = re.compile(r"^\s*(\(?[A-Za-z][.)]|\([A-Za-z]\))\s+")
_ROMAN_RE = re.compile(r"^\s*\(?(?=[ivxlcdm]{1,7}[.)])[ivxlcdm]+[.)]\s+", re.I)
# Sentence-final punctuation lives in ``styles`` -- see TERMINAL_PUNCT_RE there
# for why. ``styles`` used it without defining it, which raised NameError and
# failed the whole conversion on any document containing a bold, larger-than-body,
# short line that does not end a sentence (the IRS 1040 and Mozilla PDF-spec
# corpus samples both do).
_HYPHEN_END_RE = re.compile(r"(\w)[\-­‐‑]\s*$")
def is_list_start(text: str) -> bool:
"""True when the line opens with a bullet, number, letter or roman marker."""
t = text or ""
return bool(
_BULLET_RE.match(t) or _NUMBER_RE.match(t) or _LETTER_RE.match(t) or _ROMAN_RE.match(t)
)
@dataclass
class PageMetrics:
"""Measured page statistics that thresholds are expressed relative to."""
body_size: float
leading: float
col_left: float
col_right: float
@property
def col_width(self) -> float:
return max(self.col_right - self.col_left, 1.0)
def measure(lines: list[Line], *, body_size: float, page_width: float) -> PageMetrics:
"""Derive body leading and column margins from the lines themselves."""
if not lines:
return PageMetrics(body_size or 12.0, (body_size or 12.0) * 1.2, 0.0, page_width or 612.0)
gaps: list[float] = []
for prev, cur in pairwise(lines):
dy = prev.y - cur.y
# Ignore column jumps and pathological spacing when measuring leading.
if 0.5 < dy < (body_size or 12.0) * 3.5:
gaps.append(dy)
leading = statistics.median(gaps) if gaps else max((body_size or 12.0) * 1.2, 1.0)
lefts = sorted(ln.x0 for ln in lines)
rights = sorted(ln.x1 for ln in lines)
# Robust margins: ignore outliers from stray marks and over-wide rules.
col_left = lefts[max(0, int(len(lefts) * 0.10))]
col_right = rights[min(len(rights) - 1, int(len(rights) * 0.90))]
if col_right <= col_left:
col_left, col_right = 0.0, page_width or 612.0
return PageMetrics(
body_size=body_size or 12.0, leading=leading, col_left=col_left, col_right=col_right
)
def _style_changed(a: Line, b: Line, metrics: PageMetrics) -> bool:
fa, fb = a.font_size or metrics.body_size, b.font_size or metrics.body_size
if fa > 0 and abs(fa - fb) / fa > SIZE_TOLERANCE:
return True
na = (a.font_name or "").split("+")[-1].split("-")[0].lower()
nb = (b.font_name or "").split("+")[-1].split("-")[0].lower()
return bool(na and nb and na != nb)
def _column_lefts(lines: list[Line], metrics: PageMetrics) -> list[float]:
"""Modal left edges of two (or three) text columns, or empty if one column.
First-line indents must not create a fake column: they sit a few points
inward of a real column's left edge and are absorbed by the cluster snap.
"""
if len(lines) < 4:
return []
xs = sorted(ln.x0 for ln in lines)
snap = max(18.0, metrics.col_width * 0.10)
clusters: list[list[float]] = [[xs[0]]]
for x in xs[1:]:
if x - clusters[-1][-1] <= snap:
clusters[-1].append(x)
else:
clusters.append([x])
min_n = max(2, int(len(lines) * 0.12))
lefts = sorted(sum(c) / len(c) for c in clusters if len(c) >= min_n)
if len(lefts) < 2:
return []
sep = metrics.col_width * 0.18
kept = [lefts[0]]
for x in lefts[1:]:
if x - kept[-1] >= sep:
kept.append(x)
return kept if len(kept) >= 2 else []
def _line_column(ln: Line, lefts: list[float]) -> int:
return min(range(len(lefts)), key=lambda i: abs(ln.x0 - lefts[i]))
def _peer_column_edges(
prev: Line,
peers: list[Line] | None,
metrics: PageMetrics,
*,
column_lefts: list[float] | None = None,
) -> tuple[float, float]:
"""Left/right margins of the column ``prev`` actually sits in.
Measuring one pair of margins across a two-column page makes every line in
the left column look short of the right column's edge, so each typeset
line became its own paragraph. Peers in the same column vote their own
right edge; first-line indents still belong to that column.
That vote is only taken when columns were actually detected. The old code
fell back to "peers sharing ``prev``'s left edge" whenever they were not,
which is unsound on a single-column page: selecting peers by left edge
selects for headings, list labels and table cells, so the 90th percentile of
their right edges sits far inside the real margin. On the fidelity fixture it
put the margin at 359.9 when the text column runs to 521.2, and a caption
ending at 334.2 -- a clear 41% short of the true margin -- was read as a full
line and welded onto the paragraph above it. The bias runs one way: it makes
short lines look full, and a short line is the main evidence a paragraph
ended. When no columns are detected the page-level margins are the honest
answer, because then the page *is* the column.
"""
if not peers:
return metrics.col_left, metrics.col_right
lefts = column_lefts if column_lefts is not None else _column_lefts(peers, metrics)
if not lefts:
return metrics.col_left, metrics.col_right
cid = _line_column(prev, lefts)
col = [ln for ln in peers if _line_column(ln, lefts) == cid]
if len(col) < 2:
return metrics.col_left, metrics.col_right
lefts_px = sorted(ln.x0 for ln in col)
rights = sorted(ln.x1 for ln in col)
col_left = lefts_px[max(0, int(len(lefts_px) * 0.10))]
col_right = rights[min(len(rights) - 1, int(len(rights) * 0.90))]
if col_right <= col_left:
return metrics.col_left, metrics.col_right
return col_left, col_right
def breaks_paragraph(
prev: Line,
cur: Line,
metrics: PageMetrics,
*,
peers: list[Line] | None = None,
column_lefts: list[float] | None = None,
) -> bool:
"""True when ``cur`` starts a new paragraph rather than continuing ``prev``."""
dy = prev.y - cur.y
col_left, col_right = _peer_column_edges(
prev, peers, metrics, column_lefts=column_lefts
)
col_width = max(col_right - col_left, 1.0)
# Flow left the column: reading order moved up the page or across a gutter.
if dy <= -metrics.leading * 0.5:
return True
if abs(cur.x0 - prev.x0) > col_width * 0.55:
return True
# Explicit list markers always start their own block.
if is_list_start(cur.text) or is_list_start(prev.text):
return True
# A trailing hyphen is the typesetter's own wrap marker — always continue,
# provided the flow has not left the column (checked above).
if _HYPHEN_END_RE.search(prev.text or ""):
return False
if _style_changed(prev, cur, metrics):
return True
# Paragraph spacing.
if dy > metrics.leading * PARA_GAP_RATIO:
return True
# Short previous line = last line of a paragraph. A line that ends with
# sentence-final punctuation and stops short is a strong signal; one that
# merely stops short is weaker, so require a wider shortfall.
shortfall = col_right - prev.x1
terminal = bool(TERMINAL_PUNCT_RE.search(prev.text or ""))
if terminal and shortfall > col_width * SHORT_LINE_RATIO:
return True
if not terminal and shortfall > col_width * (SHORT_LINE_RATIO * 2.2):
return True
# First-line indent on the incoming line.
if cur.x0 - prev.x0 > col_width * INDENT_RATIO:
return True
# Alignment returned to the margin: ``prev`` starts well inside the column
# while ``cur`` starts at its left edge. Lines of one left-aligned paragraph
# share a left edge, and a centred or right-aligned run keeps every line
# inset, so a line dropping back to the margin under an inset line belongs to
# a new paragraph. Without this a right-aligned line swallowed the paragraph
# after it: right alignment puts prev.x1 *at* the margin, which defeats the
# short-line test above -- the usual evidence that a paragraph ended.
if (
prev.x0 - col_left > col_width * INSET_LINE_RATIO
and abs(cur.x0 - col_left) <= col_width * ALIGN_RATIO
):
return True
# Centred standalone line (both edges inset) reads as a caption or title.
left_inset = prev.x0 - col_left
right_inset = col_right - prev.x1
return (
left_inset > col_width * 0.15
and right_inset > col_width * 0.15
and abs(left_inset - right_inset) < col_width * 0.08
)
# --- text-only path (fabricated geometry) ----------------------------------
# Minimum characters before a line is considered wide enough to have wrapped.
MIN_WRAP_CHARS = 24
# A line must reach this fraction of the measured column width to count as full.
FULL_LINE_RATIO = 0.85
# Percentile of line lengths taken as the column width. The maximum is a poor
# estimator — a single long table row or URL skews it by 2x on real documents.
FULL_LINE_PERCENTILE = 0.90
# A line this many times longer than the median is a spanning title or banner,
# not a wrapped body line, and stands as its own block.
SPANNING_LINE_RATIO = 1.8
def measure_full_line_chars(lines: list[Line]) -> int:
"""Estimate the column width in characters from the line-length distribution."""
lengths = sorted(len((ln.text or "").strip()) for ln in lines if (ln.text or "").strip())
if not lengths:
return 0
idx = min(len(lengths) - 1, int(len(lengths) * FULL_LINE_PERCENTILE))
return lengths[idx]
def measure_median_line_chars(lines: list[Line]) -> int:
lengths = sorted(len((ln.text or "").strip()) for ln in lines if (ln.text or "").strip())
if not lengths:
return 0
return lengths[len(lengths) // 2]
def _breaks_paragraph_text_only(
prev: Line, cur: Line, full_width_chars: int, median_chars: int = 0
) -> bool:
"""Paragraph break decision when line coordinates are fabricated.
Without real geometry the only trustworthy signals are the text itself and
the blank lines the extractor preserved. The rule is deliberately
conservative: two lines join only when the first is a *full* line with no
sentence-final punctuation, which is what a wrapped line looks like.
Anything else stays separate, because a wrong merge destroys structure
while a missed merge merely leaves it as found.
"""
prev_text = (prev.text or "").strip()
cur_text = (cur.text or "").strip()
if not prev_text or not cur_text:
return True
# Blank line in the source is encoded as a wider vertical step.
if (prev.y - cur.y) > 20.0:
return True
if is_list_start(cur_text) or is_list_start(prev_text):
return True
# A trailing hyphen is the typesetter's own wrap marker — always continue.
if _HYPHEN_END_RE.search(prev_text):
return False
if TERMINAL_PUNCT_RE.search(prev_text):
return True
# A line far longer than the page median is a spanning title or banner —
# the percentile column-width estimate is skewed by exactly such lines, so
# guard against absorbing the body text that follows one.
if median_chars and len(prev_text) > median_chars * SPANNING_LINE_RATIO:
return True
if len(prev_text) < MIN_WRAP_CHARS:
return True
return bool(full_width_chars and len(prev_text) < full_width_chars * FULL_LINE_RATIO)
def group_lines(lines: list[Line], metrics: PageMetrics) -> list[list[Line]]:
"""Group ordered lines into paragraph runs.
Lines whose geometry was fabricated (``synthetic_geometry``) are grouped by
the conservative text-only rule; measured lines use the full geometric one.
"""
if not lines:
return []
synthetic = sum(1 for ln in lines if getattr(ln, "synthetic_geometry", False))
text_only = synthetic > len(lines) / 2
full_width_chars = measure_full_line_chars(lines)
median_chars = measure_median_line_chars(lines)
column_lefts = [] if text_only else _column_lefts(lines, metrics)
if not text_only and len(column_lefts) >= 2:
return _group_lines_by_column(lines, metrics, column_lefts)
groups: list[list[Line]] = [[lines[0]]]
for prev, cur in pairwise(lines):
if text_only:
brk = _breaks_paragraph_text_only(prev, cur, full_width_chars, median_chars)
else:
brk = breaks_paragraph(
prev, cur, metrics, peers=lines, column_lefts=column_lefts
)
if brk:
groups.append([cur])
else:
groups[-1].append(cur)
return groups
def _group_consecutive(lines: list[Line], metrics: PageMetrics, column_lefts: list[float]) -> list[list[Line]]:
if not lines:
return []
groups: list[list[Line]] = [[lines[0]]]
for prev, cur in pairwise(lines):
if breaks_paragraph(prev, cur, metrics, peers=lines, column_lefts=column_lefts):
groups.append([cur])
else:
groups[-1].append(cur)
return groups
def _group_lines_by_column(
lines: list[Line], metrics: PageMetrics, column_lefts: list[float]
) -> list[list[Line]]:
"""Group wrap inside each column, then concatenate (titles that span first)."""
spanning: list[Line] = []
buckets: list[list[Line]] = [[] for _ in column_lefts]
for ln in lines:
if (ln.x1 - ln.x0) >= metrics.col_width * 0.72:
spanning.append(ln)
else:
buckets[_line_column(ln, column_lefts)].append(ln)
groups: list[list[Line]] = []
for bucket in [spanning, *buckets]:
bucket = sorted(bucket, key=lambda ln: -ln.y)
groups.extend(_group_consecutive(bucket, metrics, column_lefts))
return groups
_MIDLINE_HYPHEN_RE = re.compile(r"(?<![\w-])([A-Za-z]{2,})-([A-Za-z]{2,})(?![\w-])")
def lexical_hyphen_pairs(text: str) -> set[tuple[str, str]]:
"""Hyphenated word pairs the document itself uses *mid-line*.
A compound the author writes as ``well-known`` in running text is lexical,
not a typesetter's line break. Collecting those pairs from the document
gives a reliable, language-independent keep-list at no cost and without a
dictionary dependency.
"""
pairs: set[tuple[str, str]] = set()
for raw_line in (text or "").splitlines():
# Only mid-line hyphens count; a hyphen at end of line is the ambiguous case.
line = raw_line.rstrip()
if line.endswith("-"):
line = line[:-1]
for m in _MIDLINE_HYPHEN_RE.finditer(line):
pairs.add((m.group(1).lower(), m.group(2).lower()))
return pairs
def join_line_texts(
texts: list[str], *, lexical_pairs: set[tuple[str, str]] | None = None
) -> str:
"""Join wrapped line texts, resolving end-of-line hyphenation.
End-of-line hyphens are ambiguous: they may be a typesetter's break
(``inter-`` / ``national`` → ``international``) or part of a real compound
(``well-`` / ``known`` → ``well-known``). The hyphen is kept when:
* the pair appears hyphenated mid-line elsewhere in the document, or
* the following fragment starts with a capital or a digit, or
* either fragment is too short to be a plausible split.
Otherwise the hyphen is removed, which is the common case in wrapped prose.
"""
if not texts:
return ""
pairs = lexical_pairs or set()
out = texts[0].strip()
for raw in texts[1:]:
nxt = raw.strip()
if not nxt:
continue
if not out:
out = nxt
continue
# Soft hyphen: always a break marker, never part of the word.
if out.endswith("­"):
out = out[:-1] + nxt
continue
m = _HYPHEN_END_RE.search(out)
if not m:
out = f"{out} {nxt}"
continue
head = out[: m.end(1)] # everything through the letter, hyphen excluded
left_frag = re.search(r"([A-Za-z]+)[\-‐‑]\s*$", out)
right_frag = re.match(r"([A-Za-z0-9]+)", nxt)
keep_hyphen = True
if left_frag and right_frag:
lf, rf = left_frag.group(1), right_frag.group(1)
if (lf.lower(), rf.lower()) in pairs or rf[:1].isupper() or rf[:1].isdigit() or len(lf) < 2 or len(rf) < 2:
keep_hyphen = True
else:
keep_hyphen = False
out = f"{head}-{nxt}" if keep_hyphen else f"{head}{nxt}"
return out
gateway/app/services/convert/layout/pdf_router.py
+224
View File
@@ -0,0 +1,224 @@
"""Document-level PDF routing (pdf-inspector-inspired, in-house).
Classifies TextBased / Scanned / ImageBased / Mixed and lists pages needing OCR.
Does not call Firecrawl or pdf-inspector — pure heuristics over pypdf + optional engine.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from enum import Enum
from pypdf import PdfReader
from app.services.convert import doc_cache
class PdfDocType(str, Enum):
text_based = "text_based"
scanned = "scanned"
image_based = "image_based"
mixed = "mixed"
class ScanStrategy(str, Enum):
early_exit = "early_exit"
full = "full"
sample = "sample"
@dataclass
class PdfRouteResult:
doc_type: PdfDocType
confidence: float
pages_needing_ocr: list[int] = field(default_factory=list)
pages_sampled: list[int] = field(default_factory=list)
strategy: str = "full"
def as_dict(self) -> dict:
return {
"doc_type": self.doc_type.value,
"confidence": round(self.confidence, 4),
"pages_needing_ocr": list(self.pages_needing_ocr),
"pages_sampled": list(self.pages_sampled),
"strategy": self.strategy,
}
def _strategy_from_env() -> tuple[ScanStrategy, int]:
raw = (os.environ.get("CONVERT_PDF_SCAN_STRATEGY") or "full").strip().lower()
try:
n = int(os.environ.get("CONVERT_PDF_SCAN_SAMPLE", "5"))
except ValueError:
n = 5
n = max(2, min(n, 50))
if raw in ("early_exit", "early-exit", "early"):
return ScanStrategy.early_exit, n
if raw in ("sample", "sampled"):
return ScanStrategy.sample, n
return ScanStrategy.full, n
def _page_needs_ocr(reader: PdfReader, index: int, data: bytes | None = None) -> tuple[bool, str]:
"""Heuristic: sparse / garbled extractable text → OCR. Returns (needs_ocr, reason)."""
from app.services.convert.layout.text_quality import is_garbled_text, page_text_needs_ocr
try:
page = reader.pages[index]
text = (
doc_cache.page_text(data, index) if data is not None else (page.extract_text() or "")
).strip()
except Exception:
return True, "extract_failed"
# Image XObjects hint
has_image = False
try:
resources = page.get("/Resources") or {}
if hasattr(resources, "get_object"):
resources = resources.get_object()
xobj = resources.get("/XObject") if resources else None
if xobj is not None:
if hasattr(xobj, "get_object"):
xobj = xobj.get_object()
for _name, ref in (xobj.items() if hasattr(xobj, "items") else []):
try:
obj = ref.get_object() if hasattr(ref, "get_object") else ref
if obj.get("/Subtype") == "/Image":
has_image = True
break
except Exception:
continue
except Exception:
pass
chars = len(text)
# Image-only / nearly empty → OCR. Decorative images beside real text → keep digital.
if chars < 12:
return True, "image_only" if has_image else "sparse_text"
# Broken ToUnicode / CID garbage (Huwiyati-class) must not count as "text_ok"
if is_garbled_text(text) or page_text_needs_ocr(text, image_coverage=0.2 if has_image else 0.0):
return True, "garbled_text"
return False, "text_ok"
def _sample_indices(n_pages: int, strategy: ScanStrategy, sample_n: int) -> list[int]:
if n_pages <= 0:
return []
if strategy == ScanStrategy.full or strategy == ScanStrategy.early_exit:
return list(range(n_pages))
# sample: first, last, evenly spaced middle
if n_pages <= sample_n:
return list(range(n_pages))
idxs = {0, n_pages - 1}
mid_slots = sample_n - 2
if mid_slots > 0:
step = (n_pages - 1) / (mid_slots + 1)
for i in range(1, mid_slots + 1):
idxs.add(min(n_pages - 1, max(0, int(round(i * step)))))
return sorted(idxs)
def route_pdf(data: bytes, *, strategy: ScanStrategy | None = None) -> PdfRouteResult:
"""Classify PDF and list 0-based page indices that need OCR."""
strat, sample_n = _strategy_from_env()
if strategy is not None:
strat = strategy
try:
reader = doc_cache.get_reader(data)
except Exception:
return PdfRouteResult(
doc_type=PdfDocType.scanned,
confidence=0.3,
pages_needing_ocr=[],
pages_sampled=[],
strategy=strat.value,
)
n_pages = len(reader.pages)
indices = _sample_indices(n_pages, strat, sample_n)
needing: list[int] = []
sampled: list[int] = []
text_ok = 0
image_heavy = 0
garbled_heavy = 0
for i in indices:
sampled.append(i)
needs, reason = _page_needs_ocr(reader, i, data)
if needs:
needing.append(i)
if reason in ("image_only", "hybrid_sparse"):
image_heavy += 1
if reason == "garbled_text":
garbled_heavy += 1
else:
text_ok += 1
if strat == ScanStrategy.early_exit and needing and not text_ok:
# First page already needs OCR and no prior text → treat as scanned early
break
# Expand early_exit / sample results to full page lists when mixed
if strat == ScanStrategy.early_exit and needing and text_ok == 0 and len(sampled) < n_pages:
# Likely fully scanned — mark all pages
needing = list(range(n_pages))
sampled = list(range(n_pages))
elif strat == ScanStrategy.sample and needing:
# Conservative: any sampled OCR page → include that page; do not invent others
pass
total = max(len(sampled), 1)
ocr_frac = len(needing) / total
text_frac = text_ok / total
garbled_frac = garbled_heavy / total
# Majority garbled / OCR-needed → promote whole document (hybrid RFP class)
if ocr_frac >= 0.5 and len(sampled) < n_pages:
needing = list(range(n_pages))
sampled = list(range(n_pages))
ocr_frac = 1.0
text_frac = 0.0
elif ocr_frac >= 0.5 and n_pages > 0 and len(needing) < n_pages:
needing = list(range(n_pages))
ocr_frac = 1.0
text_frac = 0.0
if ocr_frac >= 0.95:
if garbled_frac >= 0.4:
doc_type = PdfDocType.mixed if text_ok > 0 else PdfDocType.scanned
else:
doc_type = PdfDocType.image_based if image_heavy >= text_ok else PdfDocType.scanned
conf = 0.55 + 0.4 * ocr_frac
elif ocr_frac <= 0.05:
doc_type = PdfDocType.text_based
conf = 0.55 + 0.4 * text_frac
else:
doc_type = PdfDocType.mixed
conf = 0.5 + 0.3 * (1.0 - abs(0.5 - ocr_frac))
# Unique sorted
needing = sorted(set(needing))
return PdfRouteResult(
doc_type=doc_type,
confidence=min(1.0, conf),
pages_needing_ocr=needing,
pages_sampled=sampled,
strategy=strat.value,
)
def page_kind_hint(doc_type: PdfDocType, page_index: int, pages_needing_ocr: list[int]):
"""Map route into PageKind-compatible hint for a page."""
from app.services.convert.idm.model import PageKind
if page_index in pages_needing_ocr:
if doc_type == PdfDocType.mixed:
return PageKind.hybrid
return PageKind.scan
if doc_type == PdfDocType.text_based:
return PageKind.digital
if doc_type == PdfDocType.mixed:
return PageKind.digital
return PageKind.scan
gateway/app/services/convert/layout/pipeline.py
+1278
View File
File diff suppressed because it is too large Load Diff
gateway/app/services/convert/layout/pypdf_geometry.py
+1291
View File
File diff suppressed because it is too large Load Diff
gateway/app/services/convert/layout/reading_order.py
+591
View File
@@ -0,0 +1,591 @@
"""Column detection and reading order via recursive XY-Cut (Ha et al. 1995)."""
from __future__ import annotations
import bisect
from app.services.convert.layout.glyphs import Line
MIN_COL_LINES = 2
MIN_GUTTER_RATIO = 0.05
MIN_BLOCK_LINES = 2
# A band is read as a column only when it is not simply the left-hand cells of
# rows that continue to its right. Above this share of shared baselines the
# two sides are rows of one structure — a table, a definition list, a form —
# and cutting between them interleaves the document.
MAX_SHARED_BASELINES = 0.6
# ...and only when the candidate side does not fill its own width. Real prose
# columns run to their margin; table cells and row labels do not.
COLUMN_FILL_RATIO = 0.55
# Share of body lines that must sit clear of a candidate margin band for it to be
# read as a sidebar. Not all of them: a footnote or an indented quote may reach
# into the margin without making the band beside it a sidebar.
SIDEBAR_CLEAR_RATIO = 0.9
def _mid_x(ln: Line) -> float:
return (ln.x0 + ln.x1) / 2.0
def _shared_baseline_ratio(a: list[Line], b: list[Line]) -> float:
"""Share of the smaller group whose lines sit on a baseline the other uses.
Two prose columns have independent line rhythms. The label column of a
table sits on exactly the same baselines as the values beside it, which is
what tells the two apart without having to detect the table first.
"""
if not a or not b:
return 0.0
small, large = (a, b) if len(a) <= len(b) else (b, a)
others = sorted(ln.y for ln in large)
if not others:
return 0.0
hits = 0
for ln in small:
tol = max(3.0, float(ln.font_size or 12.0) * 0.35)
i = bisect.bisect_left(others, ln.y - tol)
if i < len(others) and others[i] <= ln.y + tol:
hits += 1
return hits / len(small)
def _fills_its_width(lines: list[Line]) -> bool:
"""Whether a group's lines are all about as wide as its widest line.
Set text is justified or near-justified: every line but the last runs to
the same right margin, so mean width sits close to maximum width. Table
cells do not — a column holding "5" beside one holding "1199 sec, n=1"
has a mean far below its maximum. Measuring against the widest *line*
rather than the group's total extent keeps the test meaningful when the
group is itself several columns, which is exactly the case a three-column
page presents on the first cut.
"""
if len(lines) < 2:
return True
widths = [max(0.0, ln.x1 - ln.x0) for ln in lines]
widest = max(widths)
if widest <= 0:
return True
return (sum(widths) / len(widths)) / widest >= COLUMN_FILL_RATIO
def _is_row_structure(left: list[Line], right: list[Line]) -> bool:
"""True when a candidate vertical cut runs *through* rows, not between columns."""
if len(left) < 2 or len(right) < 2:
return False
if _shared_baseline_ratio(left, right) < MAX_SHARED_BASELINES:
return False
return not (_fills_its_width(left) and _fills_its_width(right))
def _mid_y(ln: Line) -> float:
return ln.y
def _outside_text_column(
cand: list[Line], rest: list[Line], *, on_left: bool, gutter: float
) -> bool:
"""Whether ``cand`` sits outside the body column rather than inside it.
This is the property that actually makes a margin note a margin note, and
without it the sidebar test is a trap. The test used to ask only whether a
line's midpoint fell in the outer fifth of the page and whether the line was
narrow. On US Letter with a one-inch margin the body itself starts at x0=72,
so every line shorter than about 125pt -- roughly twenty characters -- passes
both. Three such lines anywhere on a page were enough to relocate them all to
the end of it, and short lines at the left margin are not unusual: they are
headings, the first column of a table, one-word labels, dates, signature
blocks. On the fidelity fixture this moved a Heading 3, a Heading 2 and two
table cells out of the middle of the page and appended them after the page
number, from x0=67.2 -- the body's own margin.
A real sidebar is *separated*: the body column begins to its right (or ends
to its left) with clear space between. Requiring that separation is what
distinguishes the two, and it cannot be inferred from width alone. The share
is not 100% because an indented quotation or a footnote may reach into the
margin without turning the band beside it into a sidebar.
"""
if not cand or not rest:
return False
if on_left:
edge = max(ln.x1 for ln in cand) + gutter
clear = sum(1 for ln in rest if ln.x0 >= edge)
else:
edge = min(ln.x0 for ln in cand) - gutter
clear = sum(1 for ln in rest if ln.x1 <= edge)
return (clear / len(rest)) >= SIDEBAR_CLEAR_RATIO
def _arabic_ratio(lines: list[Line]) -> float:
chars = 0
ar = 0
for ln in lines:
for c in ln.text or "":
chars += 1
if "\u0600" <= c <= "\u06FF":
ar += 1
return (ar / chars) if chars else 0.0
def _min_gutter_pt(page_width: float, page_height: float) -> float:
"""Narrowest horizontal gap that may be read as a column gutter.
Scaled from the page *width*, not its longest side. Using the longest side
tied the required gap to the page height on portrait pages -- 792 * 0.03 =
23.76pt on US Letter -- which is wider than the gutter most two-column
documents actually use. Measured on page 13 of the TraceMonkey paper, whose
columns are cleanly separated by 21.66pt (left text ends at x=295.4, right
begins at x=317.0): the cut was rejected, the page fell through to a plain
top-to-bottom sort, and the two columns were interleaved line by line. That
spliced the right column's "...sponsor Sun Microsystems under Project No.
07-127." into the middle of a left-column sentence, corrupting both.
Nothing detects this downstream, because every word is still present -- only
the order is wrong, and recall cannot see order.
3% of the width is 18.36pt on Letter and 17.5pt on A4, which admits a normal
two-column gutter while staying far above an inter-word space (about 3pt at
10pt type). The floor keeps small or unusual page boxes sane.
"""
# The gutter is a horizontal distance, so scale it from the page width.
# Using the longer page side made portrait pages demand a 23.8pt gap on a
# 612pt sheet and rejected legitimate 21-24pt academic-paper columns.
page = max(page_width, 1.0)
return max(8.0, page * 0.03)
def _best_x_gutter(
lines: list[Line],
x0: float,
x1: float,
*,
min_gutter: float,
) -> float | None:
"""Return split X at largest empty vertical gutter, or None."""
width = x1 - x0
if width <= 0 or len(lines) < MIN_COL_LINES * 2:
return None
# Resolution has to be finer than the gutter being looked for, and 48 bins
# across a 612pt page is 12.75pt per bin -- coarser than the 24pt gutter of a
# standard two-column paper. Because a bin counts as occupied when text
# covers any part of it, that gutter could never register as more than one
# empty bin, and one bin measures 12.75pt, which is below every sane
# ``min_gutter``. So ``_best_x_gutter`` returned None for pages with a wide,
# perfectly clean column gap: measured on page 13 of the TraceMonkey paper,
# where every line respects a 23.9pt gap (left column ends at x=293.1, right
# begins at x=317.0) and the detector still saw no columns.
#
# The page then fell through to a plain top-to-bottom sort, which interleaves
# the columns line by line. That is what spliced the right column's
# "...sponsor Sun Microsystems under Project No. 07-127." through the middle
# of the left column's sentence and destroyed both. Recall barely moves --
# every word is still on the page -- which is why this survived so long.
#
# 4pt bins resolve any gutter worth finding. The scan below is linear in the
# bin count, so this is not a meaningful cost.
bins = 48
hist = [0] * bins
# Mark every bin a line *covers*, not the single bin its midpoint lands in.
# A gutter is a vertical strip with no text in it, so the emptiness test has
# to look at extents: scoring midpoints made a line invisible everywhere but
# its centre, and a strip crossed by running text was still read as empty.
# On the fidelity fixture that put the page's only column cut at x=337.9 with
# four full lines passing through it, which sent a right-aligned paragraph
# and the last column of a table to the end of the page. Column detection is
# the first decision made about a page and everything downstream inherits it.
# Membership still goes by midpoint -- that is what decides which side of a
# real gutter a line belongs to, and it is a separate question from whether
# the gutter is there at all.
for ln in lines:
m = _mid_x(ln)
if m < x0 or m >= x1:
continue
lo = min(bins - 1, max(0, int((max(ln.x0, x0) - x0) / width * bins)))
hi = min(bins - 1, max(0, int((min(ln.x1, x1) - x0) / width * bins)))
for idx in range(lo, hi + 1):
hist[idx] += 1
# One crossing is a rule, a stray figure label or a wide caption inside an
# otherwise two-column region; a handful is running text.
empty_max = max(1, int(len(lines) * 0.05))
mid_lo, mid_hi = int(bins * 0.08), int(bins * 0.92)
empty_runs: list[tuple[int, int]] = []
i = mid_lo
while i < mid_hi:
if hist[i] <= empty_max:
j = i
while j < mid_hi and hist[j] <= empty_max:
j += 1
empty_runs.append((i, j))
i = j
else:
i += 1
candidates: list[tuple[float, float, float, float]] = []
for start, end in empty_runs:
estimated_gap = (end - start) / bins * width
if estimated_gap < min_gutter * 0.55:
continue
split = x0 + ((start + end) / 2.0) / bins * width
left = [ln for ln in lines if _mid_x(ln) < split]
right = [ln for ln in lines if _mid_x(ln) >= split]
if len(left) < MIN_COL_LINES or len(right) < MIN_COL_LINES:
continue
left_edge = max((ln.x1 for ln in left), default=0.0)
right_edge = min((ln.x0 for ln in right), default=0.0)
gutter_pt = right_edge - left_edge
if gutter_pt < min_gutter:
continue
left_span = max((ln.x1 for ln in left), default=0.0) - min(
(ln.x0 for ln in left), default=0.0
)
right_span = max((ln.x1 for ln in right), default=0.0) - min(
(ln.x0 for ln in right), default=0.0
)
# Reject if either side still spans most of the page (false gutter)
if min(left_span, right_span) < width * 0.12:
continue
if left_span > width * 0.75 or right_span > width * 0.75:
continue
# The gap between a table's first column and its values is a wide, empty,
# full-height band — indistinguishable from a column gutter by geometry
# alone. Reading it as a gutter emits every row label first and then every
# value, which is how "Blind 5 1 4" became "Blind / Low Vision / ... / 5".
if _is_row_structure(left, right):
continue
candidates.append(
(
min(left_span, right_span),
gutter_pt,
-abs(split - (x0 + x1) / 2.0),
split,
)
)
if not candidates:
return None
return max(candidates)[-1]
def _best_y_gutter(
lines: list[Line],
y0: float,
y1: float,
*,
min_gutter: float,
) -> float | None:
"""Return horizontal split Y (PDF y-up: higher = top) at largest empty band."""
height = y1 - y0
if height <= 0 or len(lines) < MIN_BLOCK_LINES * 2:
return None
bins = 48
hist = [0] * bins
for ln in lines:
m = _mid_y(ln)
if m < y0 or m > y1:
continue
idx = min(bins - 1, max(0, int((m - y0) / height * bins)))
hist[idx] += 1
mid_lo, mid_hi = int(bins * 0.08), int(bins * 0.92)
best_gap: tuple[int, int] | None = None
best_width = 0
i = mid_lo
while i < mid_hi:
if hist[i] <= 1:
j = i
while j < mid_hi and hist[j] <= 1:
j += 1
w = j - i
if w > best_width:
best_width = w
best_gap = (i, j)
i = j
else:
i += 1
if not best_gap or best_width < max(1, int(bins * MIN_GUTTER_RATIO)):
return None
split = y0 + ((best_gap[0] + best_gap[1]) / 2.0) / bins * height
below = [ln for ln in lines if _mid_y(ln) < split]
above = [ln for ln in lines if _mid_y(ln) >= split]
if len(below) < MIN_BLOCK_LINES or len(above) < MIN_BLOCK_LINES:
return None
gutter_pt = best_width / bins * height
if gutter_pt < min_gutter:
return None
return split
def _xy_cut(
lines: list[Line],
*,
x0: float,
x1: float,
y0: float,
y1: float,
page_width: float,
page_height: float,
rtl: bool,
depth: int = 0,
) -> list[Line]:
if not lines or depth > 12:
return sorted(lines, key=lambda ln: (-ln.y, ln.x0 if not rtl else -ln.x0))
min_gutter = _min_gutter_pt(page_width, page_height)
# Prefer largest empty gutter: try X then Y (or vice-versa by gap size)
x_split = _best_x_gutter(lines, x0, x1, min_gutter=min_gutter)
y_split = _best_y_gutter(lines, y0, y1, min_gutter=min_gutter)
# Score gutters by relative width if both exist
use_x = False
use_y = False
if x_split is not None and y_split is not None:
# Prefer vertical (column) cuts when both are viable — reading-order focus
use_x = True
elif x_split is not None:
use_x = True
elif y_split is not None:
use_y = True
else:
return sorted(lines, key=lambda ln: (-ln.y, ln.x0 if not rtl else -ln.x0))
ordered: list[Line] = []
if use_x and x_split is not None:
left = [ln for ln in lines if _mid_x(ln) < x_split]
right = [ln for ln in lines if _mid_x(ln) >= x_split]
# Vertical cuts: LTR normally; RTL page → right column first
first, second = (right, left) if rtl else (left, right)
fx0, fx1 = (x_split, x1) if rtl else (x0, x_split)
sx0, sx1 = (x0, x_split) if rtl else (x_split, x1)
if rtl:
# first is right band
fx0, fx1 = x_split, x1
sx0, sx1 = x0, x_split
ordered.extend(
_xy_cut(
first,
x0=fx0,
x1=fx1,
y0=y0,
y1=y1,
page_width=page_width,
page_height=page_height,
rtl=rtl,
depth=depth + 1,
)
)
ordered.extend(
_xy_cut(
second,
x0=sx0,
x1=sx1,
y0=y0,
y1=y1,
page_width=page_width,
page_height=page_height,
rtl=rtl,
depth=depth + 1,
)
)
return ordered
if use_y and y_split is not None:
# PDF y-up: process top (higher y) first
top = [ln for ln in lines if _mid_y(ln) >= y_split]
bottom = [ln for ln in lines if _mid_y(ln) < y_split]
ordered.extend(
_xy_cut(
top,
x0=x0,
x1=x1,
y0=y_split,
y1=y1,
page_width=page_width,
page_height=page_height,
rtl=rtl,
depth=depth + 1,
)
)
ordered.extend(
_xy_cut(
bottom,
x0=x0,
x1=x1,
y0=y0,
y1=y_split,
page_width=page_width,
page_height=page_height,
rtl=rtl,
depth=depth + 1,
)
)
return ordered
return sorted(lines, key=lambda ln: (-ln.y, ln.x0 if not rtl else -ln.x0))
def _column_bands(lines: list[Line], page_width: float) -> list[tuple[float, float]]:
"""Legacy helper kept for callers/tests that inspect bands; single best X cut."""
if not lines or page_width <= 0:
return [(0.0, page_width or 612.0)]
ys = [ln.y for ln in lines]
y0, y1 = min(ys) - 1, max(ys) + 1
split = _best_x_gutter(lines, 0.0, page_width, min_gutter=_min_gutter_pt(page_width, 792.0))
if split is None:
return [(0.0, page_width)]
left = [ln for ln in lines if _mid_x(ln) < split]
right = [ln for ln in lines if _mid_x(ln) >= split]
if len(left) < MIN_COL_LINES or len(right) < MIN_COL_LINES:
return [(0.0, page_width)]
return [(0.0, split), (split, page_width)]
def _cut_band(
band: list[Line],
*,
page_width: float,
page_height: float,
rtl: bool,
) -> list[Line]:
"""Order one horizontal band, resolving any columns inside it."""
if len(band) <= 1:
return list(band)
ys = [ln.y for ln in band]
out = _xy_cut(
list(band),
x0=0.0,
x1=page_width,
y0=min(ys) - 1.0,
y1=max(ys) + 1.0,
page_width=page_width,
page_height=page_height,
rtl=rtl,
)
return out or sorted(band, key=lambda ln: -ln.y)
def _order_with_spanning(
spanning: list[Line],
main: list[Line],
*,
page_width: float,
page_height: float,
rtl: bool,
) -> list[Line]:
"""Interleave full-width lines with the columnar bands between them.
A wide line is only a *banner* relative to what sits below it. Emitting every
wide line before all narrower lines -- which is what concatenating the two
groups does -- reorders any single-column page whose line lengths vary, and
line lengths always vary: headings, list items, captions and the last line of
each paragraph are all short, while body prose runs to the margin. The result
is that prose floats to the top of the page and everything else sinks,
measured at 4,497 line inversions across this corpus and concentrated in the
structured real-world documents (IRS 1040, govinfo CFR, USGS, Mozilla spec).
Banding on the wide lines keeps the original intent -- a true banner above a
multi-column region still precedes those columns -- while a wide line in the
middle of running text keeps its place. A page that is entirely one column
degenerates to a plain top-to-bottom sort, which is correct for it.
"""
ordered: list[Line] = []
body_desc = sorted(main, key=lambda ln: -ln.y)
i = 0
for sp in sorted(spanning, key=lambda ln: -ln.y):
band: list[Line] = []
while i < len(body_desc) and body_desc[i].y > sp.y:
band.append(body_desc[i])
i += 1
if band:
ordered.extend(
_cut_band(band, page_width=page_width, page_height=page_height, rtl=rtl)
)
ordered.append(sp)
if i < len(body_desc):
ordered.extend(
_cut_band(
body_desc[i:], page_width=page_width, page_height=page_height, rtl=rtl
)
)
return ordered
def order_lines(
lines: list[Line],
page_width: float,
*,
page_height: float = 792.0,
) -> list[Line]:
"""Return lines in reading order via recursive XY-Cut.
Spanning titles (wide lines) are emitted first in Y order, then body columns
LTR (RTL when majority Arabic). Optional narrow sidebars are read after body (LTR).
"""
if not lines:
return []
pw = page_width or 612.0
ph = page_height or 792.0
rtl = _arabic_ratio(lines) >= 0.55
# 1) Spanning lines (titles / banners) — process before column cuts
spanning: list[Line] = []
body: list[Line] = []
for ln in lines:
span = max(0.0, ln.x1 - ln.x0)
if span >= pw * 0.55 and len((ln.text or "").strip()) >= 3:
spanning.append(ln)
else:
body.append(ln)
spanning.sort(key=lambda ln: -ln.y)
if not body:
return spanning
# 2) Sidebar: narrow tall band at left or right edge
sidebar: list[Line] = []
main = list(body)
mids = [_mid_x(ln) for ln in body]
if len(body) >= 4 and mids:
left_edge = [ln for ln in body if _mid_x(ln) < pw * 0.22 and (ln.x1 - ln.x0) < pw * 0.28]
right_edge = [ln for ln in body if _mid_x(ln) > pw * 0.78 and (ln.x1 - ln.x0) < pw * 0.28]
# Prefer the denser sidebar candidate if it has ≥3 lines and body remains
for cand, on_left in ((left_edge, True), (right_edge, False)):
if len(cand) >= 3 and len(body) - len(cand) >= 2:
ys = [ln.y for ln in cand]
if max(ys) - min(ys) <= ph * 0.15:
continue
rest = [ln for ln in body if id(ln) not in {id(c) for c in cand}]
# A margin note occupies vertical space of its own. A table's
# label column shares every baseline with the values beside
# it, and moving it to the end of the page destroys the table.
if _shared_baseline_ratio(cand, rest) >= MAX_SHARED_BASELINES:
continue
# ...and it sits outside the text column, not inside it. Narrow
# is not the same as separate.
if not _outside_text_column(
cand, rest, on_left=on_left, gutter=_min_gutter_pt(pw, ph)
):
continue
sidebar = cand
main = rest
break
ordered_main = _order_with_spanning(
spanning,
main,
page_width=pw,
page_height=ph,
rtl=rtl,
)
if not ordered_main:
ordered_main = sorted(list(main) + list(spanning), key=lambda ln: -ln.y)
# Sidebars after body for LTR; before body for RTL (margin notes often outside)
sidebar_ordered = sorted(sidebar, key=lambda ln: -ln.y)
if rtl:
out = sidebar_ordered + ordered_main
else:
out = ordered_main + sidebar_ordered
return out if out else sorted(lines, key=lambda ln: -ln.y)
gateway/app/services/convert/layout/styles.py
+196
View File
@@ -0,0 +1,196 @@
"""Heading detection and font-flag inference.
Heading detection drives the document outline, the navigation pane and any
downstream table of contents, so a false positive is expensive: a page banner
promoted to Heading 1 on every page produces an outline that is unusable.
The ordering of signals below is deliberate. An explicit section number is
strong and language-independent evidence and is trusted on its own. Everything
else — capitalisation, font size — is weak on its own and is required to
corroborate.
"""
from __future__ import annotations
import re
from collections import Counter
_BOLD_RE = re.compile(r"(bold|black|heavy|extrabold|semibold|demi|bd\b|boldmt)", re.I)
_ITALIC_RE = re.compile(r"(italic|oblique|it\b|ital)", re.I)
_CAPTION_RE = re.compile(r"^(figure|table|source|fig\.|tab\.)\b", re.I)
# Sentence-final punctuation, including CJK and Arabic full stops. Defined here
# rather than in ``paragraphs`` because both modules need it and the dependency
# only runs one way: ``glyphs`` imports this module, and ``paragraphs`` imports
# ``glyphs``, so importing ``paragraphs`` from here would close a cycle. One
# definition matters -- the bold-heading rule below and the paragraph-break rule
# in ``paragraphs`` must agree on what ends a sentence, or a line can be both a
# heading and the continuation of the prose above it.
TERMINAL_PUNCT_RE = re.compile(r"[.!?;:。！？۔؟]['\"”’)\]]*\s*$") # noqa: RUF001
# "1 Introduction", "2.1 Invitation", "3.4.2 Scope" — depth sets the level.
_SECTION_NUM_RE = re.compile(r"^(\d{1,2}(?:\.\d{1,2}){0,3})[.)]?\s+([A-Za-z\u0600-\u06ff].*)$")
# The same heading with the space lost: "4.4.1.3United Arab Emirates". Bilingual
# PDFs produce these constantly — the number and the title are separate text
# runs, and a run boundary is not a space. At least one dot is required, so a
# year or a quantity ("2024Report", "5Items") is left alone: those are far more
# often a genuine token than a section number.
_GLUED_SECTION_RE = re.compile(r"^(\d{1,2}(?:\.\d{1,2}){1,3})[.)]?(?=[A-Za-z\u0600-\u06ff])")
# Dates open with digits and contain letters, so they look like numbered
# sections. "22 -JUN -2026" is not a heading.
_DATE_LIKE_RE = re.compile(
r"^\s*\d{1,4}\s*[-/ ]\s*(?:\d{1,2}|jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)"
r"[a-z]*\s*[-/ ]\s*\d{2,4}\s*$",
re.I,
)
# "Chapter 4", "Appendix B", "Part II", "Annex 3"
_NAMED_SECTION_RE = re.compile(
r"^(chapter|section|appendix|annex|part|schedule)\s+([0-9]+|[ivxlcdm]+|[a-z])\b",
re.I,
)
# A line that is mostly punctuation or digits is not a heading.
_MOSTLY_WORDS_RE = re.compile(r"[A-Za-z؀-ۿ一-鿿]")
MAX_HEADING_CHARS = 120
MAX_HEADING_WORDS = 14
# An all-caps line must be at least this much larger than body text to count as
# a heading on capitalisation alone. Page banners are set at or below body size.
CAPS_SIZE_RATIO = 1.05
# A wholly bold line must be at least this much larger than body text to be read
# as a heading on weight. Bold set *at* body size is emphasis, not structure.
BOLD_HEADING_RATIO = 1.04
def _tier_edges(body_size: float) -> list[float]:
"""Font-size tier boundaries relative to body text."""
b = max(body_size, 8.0)
return [b * 1.15, b * 1.30, b * 1.55, b * 1.90]
def split_glued_section_number(text: str) -> str:
"""Restore the space in ``4.4.1.3United Arab Emirates``.
The section number is never dropped — it is how the reader and the
document's own cross-references identify the clause. Only the missing
separator is added, and only when what follows the number is real words.
"""
stripped = (text or "").strip()
if not stripped or _DATE_LIKE_RE.match(stripped):
return text
m = _GLUED_SECTION_RE.match(stripped)
if not m:
return text
number = m.group(0)
rest = stripped[len(number):]
if not _MOSTLY_WORDS_RE.search(rest):
return text
return f"{number} {rest}"
def section_number_level(text: str) -> int:
"""Heading level implied by an explicit section number, or 0."""
stripped = (text or "").strip()
if _DATE_LIKE_RE.match(stripped):
return 0
m = _SECTION_NUM_RE.match(split_glued_section_number(stripped).strip())
if m and _MOSTLY_WORDS_RE.search(m.group(2)):
depth = m.group(1).count(".") + 1
return min(depth, 3)
if _NAMED_SECTION_RE.match((text or "").strip()):
return 1
return 0
def heading_level(
text: str, font_size: float, body_size: float, *, bold: bool = False
) -> int:
"""Heading level 1-3, or 0 for body text."""
t = (text or "").strip()
if not t or len(t) > MAX_HEADING_CHARS:
return 0
from app.services.convert.layout.text_quality import is_ocr_noise_line
if is_ocr_noise_line(t):
return 0
if _CAPTION_RE.match(t):
return 0
# Quoted / OCR-bullet prose is not a heading (Huwiyati-class false H1s).
if t[:1] in "'\"`" or t.startswith(("'", '"', "`")):
return 0
if not _MOSTLY_WORDS_RE.search(t):
# Pure numbers, dates and rule characters are not headings.
return 0
words = t.split()
if len(words) > MAX_HEADING_WORDS:
return 0
# Strongest signal: an explicit section number. Language-independent and
# carries its own depth, so it is trusted without a size check.
numbered = section_number_level(t)
if numbered:
return numbered
ratio = (font_size / body_size) if body_size > 0 else 0.0
# Capitalisation alone is weak. A running page banner is often set in caps
# at body size or smaller — requiring a size increase keeps it out of the
# outline while still catching genuine caps headings.
if t.isupper() and len(words) <= 12 and ratio >= CAPS_SIZE_RATIO:
return 1 if ratio >= 1.30 else 2
if body_size > 0:
edges = _tier_edges(body_size)
fs = round(font_size * 2) / 2.0 # snap to half points
if fs >= edges[3]:
return 1
if fs >= edges[2]:
return 1
if fs >= edges[1]:
return 2
if fs >= edges[0]:
return 3
# Weight is the other half of the convention, and size alone cannot see it.
# Word's default Heading 3 is 12pt bold on an 11pt body -- 1.09x, below the
# smallest size tier -- so every H3 in a default-styled document arrived as
# ordinary body text. Bold *and* larger than body *and* short *and* not
# ending a sentence is a heading; requiring all four keeps bold emphasis
# inside prose out, since an emphasised phrase is part of a longer line that
# is not wholly bold and usually ends in punctuation. Level 3 is provisional
# -- ``doc_wide._rank_heading_levels`` sets the real depth once every heading
# size in the document is known.
if (
bold
and body_size > 0
and (font_size / body_size) >= BOLD_HEADING_RATIO
and len(words) <= 12
and not TERMINAL_PUNCT_RE.search(t)
):
return 3
return 0
def is_caption(text: str) -> bool:
return bool(text and _CAPTION_RE.match(text.strip()))
def infer_font_flags(font_name: str | None) -> tuple[bool, bool]:
"""Infer (bold, italic) from a PDF font name when the engine omits flags."""
name = (font_name or "").strip()
if not name:
return False, False
return bool(_BOLD_RE.search(name)), bool(_ITALIC_RE.search(name))
def style_key(font_name: str, font_size: float, bold: bool, italic: bool) -> tuple:
return (font_name.lower().strip(), round(font_size, 1), bold, italic)
def dominant_body_size(sizes: list[float]) -> float:
"""Most common half-point size — the body size of a page."""
if not sizes:
return 12.0
snapped = [round(s * 2) / 2.0 for s in sizes if s > 0]
if not snapped:
return 12.0
return Counter(snapped).most_common(1)[0][0]
gateway/app/services/convert/layout/table_plausibility.py
+429
View File
@@ -0,0 +1,429 @@
"""Is this grid a table, or a figure that happened to have lines in it?
Both detectors used the same stand-in for confidence: ``0.55 + 0.05 * filled``,
capped at 0.95. Any grid with eight non-empty cells scored 0.95, so a
conference paper's figure — boxes and arrows, drawn with the same stroke
operators a ruled table uses — came back as a high-confidence table with the
caption shredded across its cells, and nothing downstream could tell the
difference.
This module asks the questions that actually separate the two. Every one is a
property of the *content* laid into the grid, not of how the boundaries were
found, so the rectangle detector, the ruling detector and any future one all
answer to the same standard:
* **fill** — a table's cells hold values; a figure's grid is mostly air.
* **column occupancy** — a table's columns are populated down their length. A
figure drops a label here and there.
* **brevity** — a table cell holds a value; a caption is not a value.
* **row regularity** — a table's rows have the same shape as each other.
* **typing** — a column of numbers, or of short labels, is strong evidence.
Nothing in a figure is typed.
Nothing here can create a table: it only scores one that was already found,
and the caller compares the score against its own floor.
"""
from __future__ import annotations
import re
from dataclasses import dataclass, replace
# A cell longer than this is prose that fell inside a box. Table cells are
# values and short labels; the longest legitimate cell seen in the real corpus
# (a multi-line "98.3% n=2 (97.7%, n=3)" result) is well under 60 characters,
# and a wrapped address or a footnote reaches ~200.
MAX_PLAUSIBLE_CELL = 320
# Beyond this share of cells being long, the grid is holding paragraphs.
MAX_LONG_CELL_SHARE = 0.25
LONG_CELL = 120
# And beyond this share of the grid's whole text living in those long cells,
# the grid is one paragraph with labels around it however few cells are long.
MAX_LONG_TEXT_SHARE = 0.60
# What makes a column a description column is not that most of its cells are
# long — real descriptions run from a phrase to a paragraph, and on the RFP
# corpus only 40% of them cleared any absolute bar. It is that the column is
# *much* longer than the labels beside it. These two are the shape of "labels
# opposite sentences": a typical cell of at least this many characters, and at
# least this many times the typical cell of the shortest column.
DESCRIPTION_COLUMN_MIN_MEDIAN = 60
DESCRIPTION_COLUMN_RATIO = 2.2
# And a column is not a column on the strength of one data row.
DESCRIPTION_COLUMN_MIN_VALUES = 3
# Even a description column has a limit. Past this a "cell" is a section of
# the document that a grid closed around.
MAX_DESCRIPTION_CELL = 700
# A grid emptier than this is a bounding box, not a table.
MIN_FILL = 0.30
# The single strongest discriminator, and the one the other measures cannot
# stand in for: a table's columns *mean* something, so they are populated down
# their length. On the real corpus every true table scores 0.82 or better
# here — the W3C ruled grid 0.82, the IRS tax tables 0.88 to 1.00 — while
# every figure read as a grid scores 0.43 or worse. The floor sits between,
# with room on both sides, because both errors are expensive: a lost table is
# unrecoverable for the reader, and an invented one shreds a caption.
MIN_COLUMN_CONSISTENCY = 0.55
_NUMBER = re.compile("^[\\s(]*[-+$\u20ac\u00a3\u00a5]?\\s*\\d[\\d,.\u00a0\u202f ]*\\s*%?\\s*\\)?$")
_SHORT_LABEL = 32
_LINE_NUMBER = re.compile(r"^\s*\d{1,4}[.)]?\s*$")
_CODE_MARKERS = re.compile(
r"(?:[{};=<>\[\]]|\+\+|--|"
r"\b(?:for|while|if)\s*\(|"
r"\b(?:return|continue|break)\s*[;({]|"
r"\b(?:var|let|const|function)\s+[A-Za-z_$]|"
r"\b(?:true|false|null|new)\b)"
)
@dataclass(frozen=True)
class Plausibility:
"""A score in [0, 1] and the reason it is not higher."""
score: float
fill: float
column_consistency: float
brevity: float
row_regularity: float
typed_columns: float
reason: str = ""
def __bool__(self) -> bool:
return self.score > 0.0
def _is_number(text: str) -> bool:
return bool(_NUMBER.match(text)) and any(ch.isdigit() for ch in text)
def _column_consistency(grid: list[list[str]], rows: int, cols: int) -> float:
"""How evenly the columns are populated down the grid.
A table fills most of most columns. A figure's grid has one or two busy
cells and a lot of nothing, which reads here as a low minimum occupancy.
"""
if rows < 2 or cols < 2:
return 0.0
occupancy = []
for ci in range(cols):
filled = sum(1 for ri in range(rows) if ci < len(grid[ri]) and grid[ri][ci].strip())
occupancy.append(filled / rows)
populated = [o for o in occupancy if o > 0]
if len(populated) < 2:
return 0.0
mean = sum(populated) / len(populated)
# Reward columns that are used consistently, and penalise a grid whose
# columns are mostly empty even when a couple are full.
spread = sum(abs(o - mean) for o in populated) / len(populated)
coverage = len(populated) / cols
return max(0.0, min(1.0, (mean - spread) * coverage))
def _row_regularity(grid: list[list[str]], rows: int) -> float:
"""How alike the rows are in how many cells they fill."""
if rows < 2:
return 0.0
counts = [sum(1 for c in row if c.strip()) for row in grid]
used = [c for c in counts if c]
if len(used) < 2:
return 0.0
mean = sum(used) / len(used)
if mean <= 0:
return 0.0
spread = sum(abs(c - mean) for c in used) / len(used)
return max(0.0, min(1.0, 1.0 - spread / mean))
def _populated_columns(grid: list[list[str]], rows: int, cols: int) -> int:
return sum(
1
for ci in range(cols)
if any(ci < len(grid[ri]) and grid[ri][ci].strip() for ri in range(rows))
)
def _median_length(values: list[str]) -> float:
if not values:
return 0.0
lengths = sorted(len(v) for v in values)
return float(lengths[len(lengths) // 2])
def _description_columns(grid: list[list[str]], rows: int, cols: int) -> set[int]:
"""Columns that consistently hold long text, and legitimately so.
"Type | Description", "Section | Section Description", "Requirement |
Acceptance criteria" — a column of sentences opposite a column of labels
is one of the most common real tables there is. Judging its cells by the
same length rule as a price column throws it away, which is exactly what
happened to two real tables in a bilingual RFP.
The test is relative because descriptions are not uniformly long. On that
RFP the description cells ran 11, 99, 227, 100 and 201 characters against
labels of 4, 6, 10, 12 and 19: only two cells cleared any absolute bar,
but the column is ten times its neighbour and obviously what it is.
"""
medians: dict[int, float] = {}
for ci in range(cols):
values = [
grid[ri][ci].strip()
for ri in range(rows)
if ci < len(grid[ri]) and grid[ri][ci].strip()
]
# Three values — a header and two rows — before a column can be
# called anything. With two, "this column is long" and "this cell
# happens to be long" are the same observation.
if len(values) >= DESCRIPTION_COLUMN_MIN_VALUES:
medians[ci] = _median_length(values)
if len(medians) < 2:
return set()
shortest = min(medians.values())
out: set[int] = set()
for ci, median in medians.items():
if median < DESCRIPTION_COLUMN_MIN_MEDIAN:
continue
if shortest <= 0 or median >= shortest * DESCRIPTION_COLUMN_RATIO:
out.add(ci)
return out
def _typed_columns(grid: list[list[str]], rows: int, cols: int) -> float:
"""Share of populated columns whose values are all one kind of thing."""
if rows < 2 or cols < 1:
return 0.0
typed = 0
populated = 0
for ci in range(cols):
values = [
grid[ri][ci].strip() for ri in range(rows) if ci < len(grid[ri]) and grid[ri][ci].strip()
]
if len(values) < 2:
continue
populated += 1
# The header is allowed to break the type: skip the first value when
# the rest agree, which is what a real table looks like.
body = values[1:] if len(values) > 2 else values
if all(_is_number(v) for v in body) or all(len(v) <= _SHORT_LABEL for v in body):
typed += 1
return typed / populated if populated else 0.0
def _looks_like_annotated_code_listing(grid: list[list[str]], rows: int, cols: int) -> bool:
"""Recognise prose + line numbers + source code mistaken for a table.
Borderless alignment is deliberately conservative about geometry, but a
code sample printed beside explanatory prose has exactly the same regular
columns as a data table. It is a common shape in papers and manuals: one
column is a run of consecutive line numbers, one is punctuation-heavy
source, and the remaining column is sentence fragments. Treating that
arrangement as a data grid shreds the prose and is worse than leaving all
three columns as flowing text.
The test is intentionally conjunctive. A numeric table has no code
markers, a normal description table has no consecutive line-number
column, and a source listing without annotations has no prose column.
"""
if rows < 4 or cols < 3:
return False
number_column = False
for ci in range(cols):
values = [
grid[ri][ci].strip()
for ri in range(rows)
if ci < len(grid[ri]) and grid[ri][ci].strip()
]
numbered = [int(v.rstrip(".)")) for v in values if _LINE_NUMBER.match(v)]
if len(numbered) < max(3, (rows + 1) // 2):
continue
if len(numbered) == 1:
continue
consecutive = sum(1 for a, b in zip(numbered, numbered[1:]) if 0 < b - a <= 2)
if consecutive / max(len(numbered) - 1, 1) >= 0.70:
number_column = True
break
if not number_column:
return False
code_column = False
for ci in range(cols):
values = [
grid[ri][ci].strip()
for ri in range(rows)
if ci < len(grid[ri]) and grid[ri][ci].strip()
]
if len(values) < max(3, (rows + 1) // 2):
continue
marked = [v for v in values if _CODE_MARKERS.search(v)]
if len(marked) / len(values) >= 0.60 and sum(len(_CODE_MARKERS.findall(v)) for v in marked) / len(marked) >= 1.5:
code_column = True
break
if not code_column:
return False
prose_column = False
for ci in range(cols):
values = [
grid[ri][ci].strip()
for ri in range(rows)
if ci < len(grid[ri]) and grid[ri][ci].strip()
]
if len(values) < 3:
continue
prose = [
v
for v in values
if not _LINE_NUMBER.match(v)
and (len(v) >= 28 or len(v.split()) >= 5)
and len(_CODE_MARKERS.findall(v)) <= 1
]
if len(prose) / len(values) >= 0.60:
prose_column = True
break
return prose_column
def assess(grid: list[list[str]] | None) -> Plausibility:
"""Score *grid* as a table. Zero means "this is not one"."""
empty = Plausibility(0.0, 0.0, 0.0, 0.0, 0.0, 0.0)
if not grid:
return replace(empty, reason="no grid")
rows = len(grid)
cols = max((len(r) for r in grid), default=0)
if rows < 2 or cols < 2:
return replace(empty, reason=f"{rows}x{cols} is not a grid")
cells = [(grid[ri][ci] if ci < len(grid[ri]) else "") for ri in range(rows) for ci in range(cols)]
filled = [c.strip() for c in cells if c.strip()]
if len(filled) < 4:
return replace(empty, reason=f"only {len(filled)} cells carry text")
if _looks_like_annotated_code_listing(grid, rows, cols):
return replace(
empty,
reason=(
"consecutive line numbers, source-code punctuation, and prose "
"indicate an annotated code listing rather than a data table"
),
)
fill = len(filled) / len(cells)
if fill < MIN_FILL:
return replace(empty, reason=f"only {fill:.0%} of cells carry text")
# A description column is a real and common thing: "Public | Information
# intended for public release…", "Section 1 | A concise statement of…".
# Long text there is the table working as designed. Long text *scattered*
# is a paragraph that fell into a grid. So the prose tests below look
# everywhere except the columns that are consistently long, and a grid
# with no short column left is prose however it is arranged.
description = _description_columns(grid, rows, cols)
if description and len(description) >= _populated_columns(grid, rows, cols):
return replace(
empty,
fill=fill,
reason="every column holds long text — this is prose in a grid, not a table",
)
elsewhere = [
grid[ri][ci].strip()
for ri in range(rows)
for ci in range(cols)
if ci not in description and ci < len(grid[ri]) and grid[ri][ci].strip()
]
longest_free = max((len(c) for c in elsewhere), default=0)
longest_described = max(
(
len(grid[ri][ci].strip())
for ri in range(rows)
for ci in description
if ci < len(grid[ri]) and grid[ri][ci].strip()
),
default=0,
)
if longest_free > MAX_PLAUSIBLE_CELL:
return replace(
empty, reason=f"a cell holds {longest_free} characters — that is prose, not a value"
)
if longest_described > MAX_DESCRIPTION_CELL:
return replace(
empty,
reason=(
f"a description cell holds {longest_described} characters — "
"longer than any column of descriptions"
),
)
if elsewhere:
long_share = sum(1 for c in elsewhere if len(c) > LONG_CELL) / len(elsewhere)
if long_share > MAX_LONG_CELL_SHARE:
return replace(
empty, reason=f"{long_share:.0%} of cells hold more than {LONG_CELL} characters"
)
# Counting long cells is not enough on a small grid: a caption of 291
# characters beside five two-character labels is one long cell out of
# six, under any per-cell share, and is still a paragraph with
# decoration around it. What settles it is where the text lives.
free_chars = sum(len(c) for c in elsewhere)
long_chars = sum(len(c) for c in elsewhere if len(c) > LONG_CELL)
if free_chars and long_chars / free_chars > MAX_LONG_TEXT_SHARE:
return replace(
empty,
reason=(
f"{long_chars / free_chars:.0%} of the grid's text sits in cells over "
f"{LONG_CELL} characters — that is prose, not a table"
),
)
column_consistency = _column_consistency(grid, rows, cols)
if column_consistency < MIN_COLUMN_CONSISTENCY:
return replace(
empty,
column_consistency=column_consistency,
fill=fill,
reason=(
f"columns are populated too unevenly ({column_consistency:.2f}) — "
"the boxes are arrangement, not a grid"
),
)
# Brevity as a smooth signal once the disqualifying cases are out.
mean_len = sum(len(c) for c in filled) / len(filled)
brevity = max(0.0, min(1.0, 1.0 - (mean_len - 8.0) / 90.0))
row_regularity = _row_regularity(grid, rows)
typed = _typed_columns(grid, rows, cols)
score = (
0.30 * fill
+ 0.28 * column_consistency
+ 0.16 * brevity
+ 0.14 * row_regularity
+ 0.12 * typed
)
# A grid this well-formed is still only ever "probably"; leaving headroom
# above the score keeps the number honest against a hand-checked table.
score = max(0.0, min(0.97, score / 0.92))
reason = ""
if score < 0.60:
weakest = min(
(
(fill, "cells are mostly empty"),
(column_consistency, "columns are not populated consistently"),
(brevity, "cells are long for a table"),
(row_regularity, "rows do not have the same shape"),
),
key=lambda pair: pair[0],
)
reason = weakest[1]
return Plausibility(
score=score,
fill=fill,
column_consistency=column_consistency,
brevity=brevity,
row_regularity=row_regularity,
typed_columns=typed,
reason=reason,
)
gateway/app/services/convert/layout/tables.py
+1110
View File
File diff suppressed because it is too large Load Diff
gateway/app/services/convert/layout/tables_lattice.py
+268
View File
@@ -0,0 +1,268 @@
"""Lattice table detection — grids recovered from the page's own ruling lines.
The rectangle detector next door works when a producer draws each cell as a
filled rectangle. Most business and government PDFs do not: they draw the
*rules*, as hairline strokes or as rectangles a fraction of a point thick, and
leave the cells empty. On the W3C table fixture the rectangles cover only the
shaded header — six of the nine rows have nothing but rules — and on the IRS
forms there are no cell rectangles at all.
This module takes the horizontal and vertical segments the geometry extractor
publishes, groups them into connected lattices, and hands the resulting edge
sets to :func:`tables_rects.fill_grid`. It is the classic lattice method, and
it is what makes ruled tables in real documents work at all.
Multiple tables on one page are found separately: horizontal rules are grouped
into vertically contiguous bands, so a page of three stacked tables yields
three grids rather than one grid spanning the gaps between them.
"""
from __future__ import annotations
from app.services.convert.idm.model import BBox, Block, BlockType
from app.services.convert.layout import table_plausibility, tables_stream
from app.services.convert.layout.glyphs import Line
from app.services.convert.layout.tables_rects import _merge_close, fill_grid
# Minimum length for a segment to count as a rule. Shorter strokes are
# underlines, bullet glyphs, or the tick marks on a chart axis.
MIN_RULE_LENGTH = 12.0
# How far a segment may deviate from axis-aligned and still count.
AXIS_TOLERANCE = 1.5
# Coordinates within this distance are the same rule drawn twice (a border
# shared by two cells is emitted once per cell by many producers).
EDGE_MERGE = 3.0
# A vertical rule must reach across this share of the candidate table's height
# before it is treated as a column boundary rather than a stray mark.
MIN_SPAN_RATIO = 0.35
# Rows further apart than this multiple of the median row height belong to
# different tables.
ROW_GROUP_GAP = 3.0
MAX_CELLS = 2000
def _segments(ops: list[dict] | None) -> tuple[list[tuple], list[tuple]]:
"""``(vertical, horizontal)`` rules as ``(pos, lo, hi)`` triples."""
if not ops:
return [], []
vertical: list[tuple[float, float, float]] = []
horizontal: list[tuple[float, float, float]] = []
for op in ops:
typ = str(op.get("type") or op.get("op") or "").lower()
if typ in ("re", "rect"):
try:
if "args" in op and isinstance(op["args"], (list, tuple)) and len(op["args"]) >= 4:
x, y, w, h = float(op["args"][0]), float(op["args"][1]), float(op["args"][2]), float(op["args"][3])
else:
x = float(op.get("x", op.get("x0", 0.0)))
y = float(op.get("y", op.get("y0", 0.0)))
w = float(op.get("w", op.get("width", 0.0)))
h = float(op.get("h", op.get("height", 0.0)))
except (TypeError, ValueError):
continue
if w <= AXIS_TOLERANCE and h >= MIN_RULE_LENGTH:
vertical.append((x + w / 2.0, min(y, y + h), max(y, y + h)))
elif h <= AXIS_TOLERANCE and w >= MIN_RULE_LENGTH:
horizontal.append((y + h / 2.0, min(x, x + w), max(x, x + w)))
continue
if typ not in ("line", "vline", "hline", "stroke", "path"):
continue
try:
x0 = float(op.get("x0", 0.0))
y0 = float(op.get("y0", 0.0))
x1 = float(op.get("x1", x0))
y1 = float(op.get("y1", y0))
except (TypeError, ValueError):
continue
dx, dy = abs(x1 - x0), abs(y1 - y0)
if dx <= AXIS_TOLERANCE and dy >= MIN_RULE_LENGTH:
vertical.append(((x0 + x1) / 2.0, min(y0, y1), max(y0, y1)))
elif dy <= AXIS_TOLERANCE and dx >= MIN_RULE_LENGTH:
horizontal.append(((y0 + y1) / 2.0, min(x0, x1), max(x0, x1)))
return vertical, horizontal
def _cluster(segments: list[tuple], gap: float) -> list[tuple[float, float, float]]:
"""Merge collinear segments into ``(position, lo, hi)`` spans."""
if not segments:
return []
out: list[list[float]] = []
for pos, lo, hi in sorted(segments):
if out and pos - out[-1][0] <= gap:
cur = out[-1]
cur[0] = (cur[0] + pos) / 2.0
cur[1] = min(cur[1], lo)
cur[2] = max(cur[2], hi)
else:
out.append([pos, lo, hi])
return [(p, lo, hi) for p, lo, hi in out]
def _row_bands(rows: list[tuple[float, float, float]]) -> list[list[tuple]]:
"""Split horizontal rules into vertically contiguous groups."""
if len(rows) < 2:
return [rows] if rows else []
rows = sorted(rows)
gaps = [b[0] - a[0] for a, b in zip(rows, rows[1:], strict=False)]
median = sorted(gaps)[len(gaps) // 2] if gaps else 0.0
limit = median * ROW_GROUP_GAP if median > 0 else float("inf")
bands: list[list[tuple]] = [[rows[0]]]
for prev, cur in zip(rows, rows[1:], strict=False):
if cur[0] - prev[0] > limit:
bands.append([cur])
else:
bands[-1].append(cur)
return [b for b in bands if len(b) >= 2]
def _best_reading(
candidates: list[Line],
col_edges: list[float],
row_edges: list[float],
) -> tuple[list[list[str]], float, set[int], list[float], list[float]] | None:
"""Read the band four ways and keep the most table-like result.
A ruled band is not always a row and a ruled column is not always a
column. Tax tables rule one band per *group* of four rows and one column
per panel, leaving the rows and the ten numeric columns to alignment;
taking the rules literally puts forty values in a single cell. But the
opposite is just as common: a header cell wraps onto three lines, and
splitting the band at every baseline shreds "Disability Category" into
two rows.
Neither case can be told apart from the rules alone — only from what the
resulting grid looks like. So both refinements are attempted and scored,
and the reading that looks most like a table wins. Refinement can only
ever be chosen by beating the plain reading on the same measure.
"""
plain = fill_grid(candidates, col_edges, row_edges)
if not plain:
return None
best_grid, _conf, best_consumed = plain
best_cols, best_rows = col_edges, row_edges
best_score = table_plausibility.assess(best_grid).score
split_rows = tables_stream.row_edges_from_baselines(candidates, row_edges)
variants: list[tuple[list[float], list[float]]] = []
if len(split_rows) != len(row_edges):
variants.append((col_edges, split_rows))
for cols, rows in [(col_edges, row_edges), *variants]:
current = fill_grid(candidates, cols, rows)
if not current:
continue
refined = tables_stream.refine_grid(current[0], candidates, cols, rows)
if refined:
variants.append((refined[1], rows))
for cols, rows in variants:
attempt = fill_grid(candidates, cols, rows)
if not attempt:
continue
grid, _c, consumed = attempt
score = table_plausibility.assess(grid).score
if score > best_score:
best_grid, best_consumed, best_score = grid, consumed, score
best_cols, best_rows = cols, rows
if best_score <= 0.0:
return None
return best_grid, best_score, best_consumed, best_cols, best_rows
def detect_tables_from_rulings(
lines: list[Line],
path_ops: list[dict] | None,
*,
start_order: int = 0,
min_conf: float = 0.60,
) -> tuple[list[Block], float, set[int]]:
"""Table blocks built from ruling lines, plus the line indices consumed."""
if not lines or not path_ops:
return [], 0.0, set()
raw_v, raw_h = _segments(path_ops)
if len(raw_v) < 2 or len(raw_h) < 2:
return [], 0.0, set()
verticals = _cluster(raw_v, EDGE_MERGE)
horizontals = _cluster(raw_h, EDGE_MERGE)
if len(verticals) < 2 or len(horizontals) < 2:
return [], 0.0, set()
blocks: list[Block] = []
used: set[int] = set()
best_conf = 0.0
for band in _row_bands(horizontals):
top, bottom = band[-1][0], band[0][0]
height = top - bottom
if height <= 0:
continue
left = min(seg[1] for seg in band)
right = max(seg[2] for seg in band)
width = right - left
# Column boundaries are the verticals that actually run through this
# band. A rule bounding the table above or below it is not one.
cols = [
seg[0]
for seg in verticals
if seg[0] >= left - EDGE_MERGE
and seg[0] <= right + EDGE_MERGE
and min(seg[2], top) - max(seg[1], bottom) >= height * MIN_SPAN_RATIO
]
col_edges = _merge_close(sorted(cols), EDGE_MERGE)
row_segs = [
seg[0]
for seg in band
if width <= 0 or (seg[2] - seg[1]) >= width * 0.45
]
row_edges = _merge_close(sorted(row_segs), EDGE_MERGE)
if len(col_edges) < 3 or len(row_edges) < 3:
# Fewer than two cells across or down is a rule under a heading,
# not a table. Under-detection is the cheaper error here.
continue
if (len(col_edges) - 1) * (len(row_edges) - 1) > MAX_CELLS:
continue
available = [(i, ln) for i, ln in enumerate(lines) if i not in used]
candidates = [ln for _i, ln in available]
got = _best_reading(candidates, col_edges, row_edges)
if not got:
continue
grid, conf, consumed, col_edges, row_edges = got
if conf < min_conf:
continue
taken = {available[i][0] for i in consumed}
used |= taken
best_conf = max(best_conf, conf)
blocks.append(
Block(
type=BlockType.table,
cells=grid,
table_confidence=conf,
bbox=BBox(
x=col_edges[0],
y=row_edges[0],
w=col_edges[-1] - col_edges[0],
h=row_edges[-1] - row_edges[0],
),
reading_order=start_order + len(blocks),
text="",
)
)
if not blocks:
return [], best_conf, set()
# Read top table first.
blocks.sort(key=lambda b: -(b.bbox.y + b.bbox.h if b.bbox else 0.0))
for order, block in enumerate(blocks):
block.reading_order = start_order + order
return blocks, best_conf, used
gateway/app/services/convert/layout/tables_rects.py
+397
View File
@@ -0,0 +1,397 @@
"""Rectangle-based table detection (pdf-inspector-inspired union-find).
MIT-licensed algorithm family (Firecrawl pdf-inspector detect_rects): cluster
cell-sized rectangles by spatial overlap, then build a grid when regular.
"""
from __future__ import annotations
from dataclasses import dataclass
from app.services.convert.idm.model import BBox, Block, BlockType, TextSpan
from app.services.convert.layout import table_plausibility
from app.services.convert.layout.glyphs import Line
@dataclass
class PageRect:
x: float
y: float
w: float
h: float
@property
def x1(self) -> float:
return self.x + self.w
@property
def y1(self) -> float:
return self.y + self.h
# Share of the lines inside a rectangle cluster that must land in a cell
# before the grid is believed. Below this the rectangles describe something
# other than a table — a form's boxes, a chart's plot area — and the
# ruling/gap heuristics downstream do better.
MIN_PLACEMENT = 0.75
class _UnionFind:
def __init__(self, n: int) -> None:
self.p = list(range(n))
self.sz = [1] * n
def find(self, i: int) -> int:
while self.p[i] != i:
self.p[i] = self.p[self.p[i]]
i = self.p[i]
return i
def union(self, a: int, b: int) -> None:
ra, rb = self.find(a), self.find(b)
if ra == rb:
return
if self.sz[ra] < self.sz[rb]:
ra, rb = rb, ra
self.p[rb] = ra
self.sz[ra] += self.sz[rb]
def _overlap(a: PageRect, b: PageRect, tol: float) -> bool:
return not (
a.x1 + tol < b.x
or b.x1 + tol < a.x
or a.y1 + tol < b.y
or b.y1 + tol < a.y
)
def _cluster_rects(rects: list[PageRect], *, tol: float = 2.0) -> list[list[int]]:
n = len(rects)
if n == 0:
return []
uf = _UnionFind(n)
# Bucket by coarse grid to avoid O(n^2) on huge pages
for i in range(n):
for j in range(i + 1, n):
if _overlap(rects[i], rects[j], tol):
uf.union(i, j)
if uf.sz[uf.find(i)] > 800:
break
groups: dict[int, list[int]] = {}
for i in range(n):
r = uf.find(i)
groups.setdefault(r, []).append(i)
return [g for g in groups.values() if len(g) >= 4]
def _merge_close(values: list[float], gap: float) -> list[float]:
"""Collapse coordinates within *gap* of each other into one boundary."""
if not values:
return []
out = [values[0]]
for v in values[1:]:
if abs(v - out[-1]) <= gap:
out[-1] = (out[-1] + v) / 2.0
else:
out.append(v)
return out
def _bucket(edges: list[float], value: float) -> int | None:
"""Index of the band ``[edges[i], edges[i+1])`` holding *value*."""
if len(edges) < 2 or value < edges[0] or value > edges[-1]:
return None
lo, hi = 0, len(edges) - 2
while lo <= hi:
mid = (lo + hi) // 2
if value < edges[mid]:
hi = mid - 1
elif value >= edges[mid + 1]:
lo = mid + 1
else:
return mid
return max(0, min(len(edges) - 2, lo))
def _grid_from_cluster(
rects: list[PageRect],
idxs: list[int],
lines: list[Line],
) -> tuple[list[list[str]], float, set[int]] | None:
"""Build a cell grid from a rectangle cluster and fill it from *lines*.
The grid is defined by the rectangles' **edges**, not their centres. Edges
partition the table area into bands, so every point inside falls in exactly
one cell and no tolerance has to be guessed; matching on nearest centre
with a fixed ±14pt/±40pt window silently discarded any line that sat
between two centres, which is how whole data rows went missing.
Text is placed span by span rather than line by line. A row of a ruled
table often reaches the layout as one line — "Low Vision 5 2 3" — because
the gaps between its cells are narrower than the column-gutter threshold;
assigning that whole line to the column under its midpoint puts five
values in one cell and leaves four empty.
Returns the grid, a confidence, and the indices of the lines it consumed,
so the caller can return everything else to the page instead of dropping
it inside the table's bounding box.
"""
cluster = [rects[i] for i in idxs]
# Filter page-background giants
areas = [max(r.w * r.h, 1.0) for r in cluster]
med = sorted(areas)[len(areas) // 2]
cells = [r for r in cluster if r.w * r.h <= med * 8 and r.w > 4 and r.h > 4]
if len(cells) < 4:
return None
col_edges = _merge_close(sorted({round(v, 1) for c in cells for v in (c.x, c.x1)}), 6.0)
row_edges = _merge_close(sorted({round(v, 1) for c in cells for v in (c.y, c.y1)}), 4.0)
return fill_grid(lines, col_edges, row_edges)
def _merge_spans_text(spans: list[TextSpan]) -> str:
"""Merge glyph/word spans into clean cell text, avoiding spurious spaces.
PDF extractors often split runs across kerning pairs, decimal points, and punctuation
(e.g. ['Catego', 'ry'], ['34', '.', '5%'], ['Co', 'm', 'pleted']). Naive joining
with spaces corrupts numbers and words.
"""
if not spans:
return ""
if len(spans) == 1:
return (spans[0].text or "").strip()
NO_LEADING_SPACE = {".", ",", ":", ";", "!", "?", "%", ")", "]", "}", "/", "-", "–", "—", "="}
NO_TRAILING_SPACE = {"(", "[", "{", "$", "£", "€", "/", "-", "–", "—", "="}
ordered = sorted(spans, key=lambda s: float(getattr(s, "x", 0.0) or 0.0))
result: list[str] = []
prev_span: TextSpan | None = None
for s in ordered:
raw = s.text or ""
txt = raw.strip()
if not txt:
if raw and result and not result[-1].endswith(" "):
result.append(" ")
continue
if not result:
result.append(txt)
prev_span = s
continue
cur_x = float(getattr(s, "x", 0.0) or 0.0)
prev_x = float(getattr(prev_span, "x", 0.0) or 0.0) if prev_span else 0.0
prev_w = float(getattr(prev_span, "w", 0.0) or 0.0) if prev_span else 0.0
prev_end = prev_x + prev_w
gap = cur_x - prev_end
prev_text = prev_span.text or "" if prev_span else ""
needs_space = False
if prev_text.endswith(" ") or raw.startswith(" "):
# Some walkers include the *next* text-matrix advance in a span,
# e.g. ``"= "``. If the next glyph starts before that span's
# reported right edge, the trailing blank is metadata rather than
# a visual separator (``n= 1`` must remain ``n=1``). A real word
# space starts at, or just after, the previous span's edge.
needs_space = cur_x >= prev_end - 0.5
elif txt in NO_LEADING_SPACE or (len(txt) > 0 and txt[0] in NO_LEADING_SPACE):
needs_space = False
elif prev_text.strip() and prev_text.strip()[-1] in NO_TRAILING_SPACE:
needs_space = False
elif prev_span and cur_x > 0 and prev_end > 0:
fsize = float(getattr(s, "font_size", 0.0) or getattr(prev_span, "font_size", 0.0) or 10.0)
space_thresh = max(1.8, fsize * 0.22)
if gap >= space_thresh:
needs_space = True
else:
needs_space = False
else:
needs_space = True
if needs_space and not result[-1].endswith(" "):
result.append(" ")
result.append(txt)
prev_span = s
return "".join(result).strip()
def fill_grid(
lines: list[Line],
col_edges: list[float],
row_edges: list[float],
) -> tuple[list[list[str]], float, set[int]] | None:
"""Place *lines* into the cells defined by ascending edge coordinates.
Shared by the rectangle and ruling detectors: once a grid's boundaries are
known, filling it is the same problem however the boundaries were found.
"""
n_cols, n_rows = len(col_edges) - 1, len(row_edges) - 1
if n_cols < 2 or n_rows < 2 or n_rows * n_cols > 2000:
return None
# row_edges ascend in PDF space (y up); rows read top-down.
grid: list[list[list[str]]] = [[[] for _ in range(n_cols)] for _ in range(n_rows)]
consumed: set[int] = set()
enclosed = 0
placed = 0
for idx, ln in enumerate(lines):
if not (ln.text or "").strip():
continue
mx = (ln.x0 + ln.x1) / 2.0
if not (col_edges[0] - 2 <= mx <= col_edges[-1] + 2):
continue
ri = _bucket(row_edges, ln.y)
if ri is None:
continue
enclosed += 1
row = n_rows - 1 - ri
# Distribute the line's spans across the columns they actually sit in.
parts: dict[int, list[TextSpan]] = {}
for span in ln.spans or []:
if not (span.text or "").strip():
continue
sx = float(span.x or ln.x0)
centre = sx + float(span.w or 0) / 2.0
ci = _bucket(col_edges, centre)
if ci is None:
ci = 0 if centre < col_edges[0] else n_cols - 1
parts.setdefault(ci, []).append(span)
if len(parts) <= 1:
ci = next(iter(parts.keys())) if parts else _bucket(col_edges, mx)
if ci is not None:
if parts and ci in parts:
t = _merge_spans_text(parts[ci])
else:
t = (ln.text or "").strip()
if t:
grid[row][ci].append(t)
else:
for ci, chunk in parts.items():
t = _merge_spans_text(chunk)
if t:
grid[row][ci].append(t)
consumed.add(idx)
placed += 1
if enclosed and placed / enclosed < MIN_PLACEMENT:
return None
flat = [[" ".join(cell).strip() for cell in row] for row in grid]
# Drop bands that carry nothing: edge sets include hairline separators,
# which would otherwise produce a blank row or column between every pair.
keep_rows = [i for i, row in enumerate(flat) if any(c for c in row)]
keep_cols = [j for j in range(n_cols) if any(flat[i][j] for i in range(n_rows))]
if len(keep_rows) < 2 or len(keep_cols) < 2:
return None
flat = [[flat[i][j] for j in keep_cols] for i in keep_rows]
filled = sum(1 for row in flat for c in row if c)
if filled < 4 or placed < 3:
return None
# Confidence is what the grid's own content says about it, not a count of
# non-empty cells: eight filled cells used to score 0.95 whether they held
# a price list or the labels of a figure.
verdict = table_plausibility.assess(flat)
if not verdict:
return None
return flat, verdict.score, consumed
def detect_tables_from_rects(
lines: list[Line],
page_rects: list[PageRect] | list[dict] | None,
*,
start_order: int = 0,
min_conf: float = 0.60,
) -> tuple[list[Block], float, set[int]]:
"""Table blocks from rectangle clusters, plus the line indices consumed.
The third value is what lets the caller keep every line the grid did not
take. Deciding "inside the table" from the block's bounding box instead
deletes a caption or a footnote that happens to sit between two rows.
"""
if not page_rects or not lines:
return [], 0.0, set()
rects: list[PageRect] = []
for r in page_rects:
if isinstance(r, PageRect):
rects.append(r)
else:
rects.append(
PageRect(
x=float(r.get("x", 0)),
y=float(r.get("y", 0)),
w=float(r.get("w", r.get("width", 0))),
h=float(r.get("h", r.get("height", 0))),
)
)
# Keep cell-ish sizes
rects = [r for r in rects if 8 < r.w < 400 and 8 < r.h < 200]
if len(rects) < 6:
return [], 0.0, set()
best_grid = None
best_conf = 0.0
best_used: set[int] = set()
for group in _cluster_rects(rects):
got = _grid_from_cluster(rects, group, lines)
if got and got[1] > best_conf:
best_grid, best_conf, best_used = got
if not best_grid or best_conf < min_conf:
return [], best_conf, set()
used = [lines[i] for i in sorted(best_used)] or lines
ys = [ln.y for ln in used]
xs0 = [ln.x0 for ln in used]
xs1 = [ln.x1 for ln in used]
block = Block(
type=BlockType.table,
cells=best_grid,
table_confidence=best_conf,
bbox=BBox(
x=min(xs0) if xs0 else 0,
y=min(ys) if ys else 0,
w=(max(xs1) - min(xs0)) if xs0 and xs1 else 0,
h=(max(ys) - min(ys) + 12) if ys else 0,
),
reading_order=start_order,
text="",
)
return [block], best_conf, best_used
def extract_rects_from_display_list(ops: list[dict] | None) -> list[PageRect]:
"""Pull rectangle operators from C++/engine display list ops."""
if not ops:
return []
out: list[PageRect] = []
for op in ops:
typ = str(op.get("type") or op.get("op") or "").lower()
if typ not in ("rect", "re", "rectangle", "fill_rect", "stroke_rect"):
continue
if "args" in op and isinstance(op["args"], (list, tuple)) and len(op["args"]) >= 4:
try:
x = float(op["args"][0])
y = float(op["args"][1])
w = float(op["args"][2])
h = float(op["args"][3])
except (ValueError, TypeError):
x = y = w = h = 0.0
else:
x = float(op.get("x", op.get("x0", 0)))
y = float(op.get("y", op.get("y0", 0)))
w = float(op.get("w", op.get("width", 0)))
h = float(op.get("h", op.get("height", 0)))
if w <= 0 and "x1" in op:
w = float(op["x1"]) - x
if h <= 0 and "y1" in op:
h = abs(float(op["y1"]) - y)
if w > 2 and abs(h) > 2:
out.append(PageRect(x=x, y=y, w=w, h=abs(h)))
return out
gateway/app/services/convert/layout/tables_stream.py
+383
View File
@@ -0,0 +1,383 @@
"""Borderless tables — columns recovered from alignment, not from rules.
The ruling and rectangle detectors need the producer to have drawn the grid.
Plenty of real tables have no grid to draw: a price list separated by tabs, a
financial statement with a rule only under the header, the IRS earned-income
tables whose rules bound a *block* of four rows and three panels and leave the
ten numeric columns to whitespace alone.
The signal a human uses on those is alignment: a column exists where several
consecutive lines all start (or all end, or all centre) at the same x, and no
line's text crosses it. This module measures exactly that, and refuses to
answer unless the evidence is strong — a wrong column boundary cuts a value in
half, which is worse than leaving the rows as text.
Two entry points:
* :func:`column_edges` — boundaries for a set of lines, or ``[]``.
* :func:`refine_grid` — subdivide the cells of a lattice grid whose content is
clearly several values wide. This is what makes a coarsely ruled table come
out right: the rules give the outer frame, the alignment gives the columns.
"""
from __future__ import annotations
from dataclasses import dataclass
from itertools import pairwise
from app.services.convert.layout.glyphs import Line
# A gap narrower than this is letter-spacing or a wide word space, not a
# column separator. Measured in points, and also required to clear the local
# font size so a 6pt footnote table is not split at every space.
MIN_GAP_POINTS = 4.0
MIN_GAP_EMS = 0.9
# A candidate boundary must be clear on at least this share of the rows. One
# long cell that closes a gutter is normal; half the rows crossing it means
# the boundary is imaginary.
MIN_CLEAR_SHARE = 0.80
# Fewer rows than this is not evidence of a column, it is a coincidence.
MIN_ROWS = 3
# More columns than this on one line is a rendering of spaced-out characters,
# not a table.
MAX_COLUMNS = 24
@dataclass(frozen=True)
class _Token:
text: str
x0: float
x1: float
def _tokens(line: Line) -> list[_Token]:
"""Spans of a line as positioned tokens, merged across intra-word gaps."""
out: list[_Token] = []
for span in line.spans or []:
text = (span.text or "").strip()
if not text:
continue
x0 = float(span.x or line.x0)
width = float(span.w or 0.0)
out.append(_Token(text=text, x0=x0, x1=x0 + width))
if not out:
text = (line.text or "").strip()
if text:
out.append(_Token(text=text, x0=float(line.x0), x1=float(line.x1)))
out.sort(key=lambda t: t.x0)
return out
def _line_gaps(line: Line, tokens: list[_Token], min_gap: float) -> list[tuple[float, float]]:
"""Whitespace runs wide enough to separate columns, as ``(lo, hi)``."""
gaps: list[tuple[float, float]] = []
for prev, cur in pairwise(tokens):
if cur.x0 - prev.x1 >= min_gap:
gaps.append((prev.x1, cur.x0))
return gaps
def _min_gap_for(lines: list[Line]) -> float:
sizes = [float(getattr(ln, "font_size", 0) or 0) for ln in lines]
sizes = [s for s in sizes if s > 0]
median = sorted(sizes)[len(sizes) // 2] if sizes else 10.0
return max(MIN_GAP_POINTS, median * MIN_GAP_EMS)
def column_edges_from_rows(
rows: list[list[_Token]], *, min_rows: int = MIN_ROWS, min_gap: float = MIN_GAP_POINTS
) -> list[float]:
"""Column boundaries a set of token rows agree on, or ``[]``.
A column boundary is a vertical strip of the page that *no* row's text
crosses. So the measurement is an occupancy profile: mark the interval
each token covers, and a strip clear on ``MIN_CLEAR_SHARE`` of rows and
wide enough to be a separator is a boundary.
Whitespace beyond a row's own last token counts as clear for that row.
Otherwise a short row — a subtotal line, a row whose last column is
blank — would veto every boundary to its right, which is precisely the
shape a real table has.
"""
populated = [r for r in rows if r]
if len(populated) < min_rows:
return []
left = min(t.x0 for r in populated for t in r)
right = max(t.x1 for r in populated for t in r)
span = int(right - left) + 1
if span <= 2 or right - left < min_gap * 2:
return []
clear = [0] * span
for row in populated:
occupied = bytearray(span)
for token in row:
a = max(0, int(token.x0 - left))
b = min(span - 1, int(token.x1 - left + 0.999))
for i in range(a, b + 1):
occupied[i] = 1
for i in range(span):
if not occupied[i]:
clear[i] += 1
need = len(populated) * MIN_CLEAR_SHARE
gutters: list[tuple[int, int]] = []
start: int | None = None
for i, count in enumerate(clear):
if count >= need:
if start is None:
start = i
elif start is not None:
gutters.append((start, i - 1))
start = None
if start is not None:
gutters.append((start, span - 1))
# A gutter must be wide enough to be a separator, and must have content on
# both sides — the page margins are clear on every row and are not columns.
edges: list[float] = []
for lo, hi in gutters:
if hi - lo + 1 < min_gap:
continue
centre = left + (lo + hi) / 2.0
has_left = any(t.x1 <= centre for r in populated for t in r)
has_right = any(t.x0 >= centre for r in populated for t in r)
if has_left and has_right:
edges.append(centre)
if not edges or len(edges) + 1 > MAX_COLUMNS:
return []
return [left - 1.0, *edges, right + 1.0]
def column_edges(lines: list[Line], *, min_rows: int = MIN_ROWS) -> list[float]:
"""Column boundaries shared by *lines*, treating each line as one row.
Callers that have already grouped lines onto shared baselines should use
:func:`column_edges_from_rows` instead: a table row frequently arrives as
several separate lines, and measuring each of them as its own row hides
every gap that falls *between* two of them.
"""
usable = [ln for ln in lines if (ln.text or "").strip()]
if len(usable) < min_rows:
return []
return column_edges_from_rows(
[_tokens(ln) for ln in usable], min_rows=min_rows, min_gap=_min_gap_for(usable)
)
def _baseline_rows(lines: list[Line], tolerance: float = 2.5) -> list[tuple[float, list[Line]]]:
"""Group lines onto shared baselines, top of the page first.
A table row often reaches the layout as several separate lines — column
detection upstream has already split "5,600 5,650 | 430 | 1,913" into
three — so the row has to be reassembled before its columns can be read.
"""
ordered = sorted(
(ln for ln in lines if (ln.text or "").strip()), key=lambda ln: -float(ln.y)
)
rows: list[tuple[float, list[Line]]] = []
for line in ordered:
if rows and abs(rows[-1][0] - float(line.y)) <= tolerance:
rows[-1][1].append(line)
else:
rows.append((float(line.y), [line]))
return rows
def _row_tokens(members: list[Line]) -> list[_Token]:
tokens: list[_Token] = []
for line in members:
tokens.extend(_tokens(line))
tokens.sort(key=lambda t: t.x0)
return tokens
def detect_aligned_table(
lines: list[Line], *, min_rows: int = 4, min_columns: int = 3, min_score: float = 0.72
) -> tuple[list[list[str]], float, set[int], list[float]] | None:
"""A table held together by alignment alone, or ``None``.
This is the case no ruling can reach: the IRS earned-income tables rule a
header box and a band every four rows, and leave ten numeric columns to
whitespace. The evidence required is deliberately steep — a long run of
rows, agreeing token counts, gutters clear on nearly every row, and a
plausibility score above the floor — because a wrong boundary here cuts a
value in half, and text left as paragraphs is still readable.
"""
from app.services.convert.layout import table_plausibility
from app.services.convert.layout.tables_rects import fill_grid
rows = _baseline_rows(lines)
if len(rows) < min_rows:
return None
# The run of consecutive baselines that all look like data rows. A table's
# rows agree about how many values they carry; a paragraph's lines do not.
counts = [len(_row_tokens(members)) for _y, members in rows]
best_run: tuple[int, int] | None = None
start = 0
while start < len(rows):
if counts[start] < min_columns:
start += 1
continue
end = start
while end + 1 < len(rows) and counts[end + 1] >= min_columns:
end += 1
if end - start + 1 >= min_rows and (
best_run is None or (end - start) > (best_run[1] - best_run[0])
):
best_run = (start, end)
start = end + 1
if best_run is None:
return None
lo, hi = best_run
members = [ln for _y, group in rows[lo : hi + 1] for ln in group]
edges = column_edges_from_rows(
[_row_tokens(group) for _y, group in rows[lo : hi + 1]],
min_rows=min_rows,
min_gap=_min_gap_for(members),
)
if len(edges) - 1 < min_columns:
return None
ys = [y for y, _g in rows[lo : hi + 1]]
heights = [abs(a - b) for a, b in pairwise(ys)]
pitch = sorted(heights)[len(heights) // 2] if heights else 12.0
row_edges = [ys[-1] - pitch * 0.6]
for a, b in pairwise(ys):
row_edges.append((a + b) / 2.0)
row_edges.append(ys[0] + pitch * 0.6)
row_edges = sorted(set(round(e, 2) for e in row_edges))
if len(row_edges) - 1 < min_rows:
return None
index_of = {id(ln): i for i, ln in enumerate(lines)}
built = fill_grid(members, edges, row_edges)
if not built:
return None
grid, _conf, consumed = built
verdict = table_plausibility.assess(grid)
if verdict.score < min_score:
return None
taken = {index_of[id(members[i])] for i in consumed if id(members[i]) in index_of}
return grid, verdict.score, taken, edges
def _cell_needs_splitting(text: str) -> bool:
"""Whether a cell's content looks like several values run together."""
parts = text.split()
return len(parts) >= 4
def row_edges_from_baselines(
lines: list[Line], row_edges: list[float], *, min_rows_per_band: int = 2
) -> list[float]:
"""Split ruled row bands that hold several text lines.
A ruled band is not always a row. Tax tables rule one band per *group* of
four rows and shade alternate groups; a financial statement rules only
above and below the totals. Where a band holds several baselines, each
baseline is a row, and the rules are boundaries between groups of them.
Bands that hold one line are left exactly as they are, so an ordinary
ruled table is unaffected.
"""
if len(row_edges) < 2:
return row_edges
baselines_by_band: dict[int, list[float]] = {}
for line in lines:
if not (line.text or "").strip():
continue
for band in range(len(row_edges) - 1):
if row_edges[band] <= line.y < row_edges[band + 1]:
baselines_by_band.setdefault(band, []).append(float(line.y))
break
out = list(row_edges)
for band, ys in baselines_by_band.items():
distinct = _cluster_values(sorted(ys), tolerance=2.0)
if len(distinct) < min_rows_per_band:
continue
lo, hi = row_edges[band], row_edges[band + 1]
# Cut halfway between consecutive baselines. Cutting *at* a baseline
# would put a row's own text on the boundary, where a half-point of
# rounding decides which cell it lands in.
for a, b in pairwise(distinct):
mid = (a + b) / 2.0
if lo + 1 < mid < hi - 1:
out.append(mid)
return sorted(set(round(e, 2) for e in out))
def _cluster_values(values: list[float], *, tolerance: float) -> list[float]:
"""Collapse values within *tolerance* into their means."""
if not values:
return []
groups: list[list[float]] = [[values[0]]]
for v in values[1:]:
if v - groups[-1][-1] <= tolerance:
groups[-1].append(v)
else:
groups.append([v])
return [sum(g) / len(g) for g in groups]
def refine_grid(
grid: list[list[str]],
lines: list[Line],
col_edges: list[float],
row_edges: list[float],
) -> tuple[list[list[str]], list[float]] | None:
"""Split coarse columns whose cells hold several aligned values.
A tax table's rules bound a band of four rows across three panels; inside
that frame ten numeric columns are held apart by whitespace alone, and the
lattice grid puts all ten in one cell. The rules are still right about
where the table is — only the column count is wrong — so the fix is to
subdivide, not to re-detect.
Returns ``(grid, col_edges)`` or ``None`` when no column survived the
evidence test, in which case the caller keeps the grid it has.
"""
if not grid or len(col_edges) < 2:
return None
wide = [
ci
for ci in range(len(col_edges) - 1)
if sum(1 for row in grid if ci < len(row) and _cell_needs_splitting(row[ci])) >= MIN_ROWS
]
if not wide:
return None
new_edges = list(col_edges)
added = False
for ci in wide:
lo, hi = col_edges[ci], col_edges[ci + 1]
inside = [
ln
for ln in lines
if (ln.text or "").strip() and lo - 1 <= (ln.x0 + ln.x1) / 2.0 <= hi + 1
]
inner = column_edges(inside)
# Only the interior boundaries matter; the outer two are this cell's
# own edges, which the lattice already knows.
interior = [e for e in inner[1:-1] if lo + 2 < e < hi - 2]
if interior:
new_edges.extend(interior)
added = True
if not added:
return None
new_edges = sorted(set(round(e, 2) for e in new_edges))
if len(new_edges) - 1 > MAX_COLUMNS:
return None
from app.services.convert.layout.tables_rects import fill_grid
rebuilt = fill_grid(lines, new_edges, row_edges)
if not rebuilt:
return None
return rebuilt[0], new_edges

Some files were not shown because too many files have changed in this diff Show More