11 Commits
24 changed files with 806 additions and 63 deletions
BIN
View File
Binary file not shown.
+3 -1
View File
@@ -73,7 +73,9 @@ find_package(freetype CONFIG REQUIRED)
find_package(harfbuzz CONFIG REQUIRED)
find_package(spdlog CONFIG REQUIRED)
find_package(nlohmann_json CONFIG REQUIRED)
find_package(qpdf CONFIG REQUIRED)
if(PDFENGINE_WITH_QPDF)
find_package(qpdf CONFIG REQUIRED)
endif()
if(WIN32 AND DEFINED VCPKG_TARGET_TRIPLET)
link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/lib")
link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/debug/lib")
+17 -1
View File
@@ -374,6 +374,11 @@ PYBIND11_MODULE(pdfengine, m) {
return py::make_tuple(img.width, img.height,
py::bytes(reinterpret_cast<const char*>(img.data.data()), img.data.size()));
}, py::arg("dpi"), py::arg("y_top_pt"), py::arg("height_pt") = 0.0)
.def("render_tile", [](const pdfengine::PdfPage& self, int dpi, double xPt, double yPt, double wPt, double hPt) {
auto img = get_or_throw(self.renderTile(dpi, xPt, yPt, wPt, hPt));
return py::make_tuple(img.width, img.height,
py::bytes(reinterpret_cast<const char*>(img.data.data()), img.data.size()));
}, py::arg("dpi"), py::arg("xPt"), py::arg("yPt"), py::arg("wPt"), py::arg("hPt"))
.def("extract_document_model", [](const pdfengine::PdfPage& self) {
return get_or_throw(self.extractDocumentModel());
})
@@ -498,7 +503,18 @@ PYBIND11_MODULE(pdfengine, m) {
return py::bytes(reinterpret_cast<const char*>(res->data()), res->size());
}, py::arg("internal_font_id"))
.def("apply_edits", [](pdfengine::PdfDocument& self, const std::string& editsJson) {
get_or_throw(self.applyEdits(editsJson));
auto regions = get_or_throw(self.applyEdits(editsJson));
py::list py_regions;
for (const auto& r : regions) {
py::dict d;
d["pageIndex"] = r.pageIndex;
d["x"] = r.x;
d["y"] = r.y;
d["width"] = r.width;
d["height"] = r.height;
py_regions.append(d);
}
return py_regions;
}, py::arg("edits_json"))
.def("save_incremental", [](const pdfengine::PdfDocument& self) {
std::vector<uint8_t> res = get_or_throw(self.saveIncremental());
+5 -4
View File
@@ -1,13 +1,14 @@
services:
gateway:
build:
context: ./gateway
dockerfile: Dockerfile
context: .
dockerfile: gateway/Dockerfile
network: host
image: pdf-engine-gateway:dev
container_name: pdf-engine-gateway
environment:
PDFENGINE_ENVIRONMENT: dev
PDFENGINE_ENGINE_AVAILABLE: "false"
PDFENGINE_ENGINE_AVAILABLE: "true"
PORT: 8000
ports:
- "8000:8000"
@@ -29,7 +30,7 @@ services:
image: pdf-engine-frontend:dev
container_name: pdf-engine-frontend
environment:
VITE_GATEWAY_URL: http://gateway:8000
VITE_GATEWAY_URL: http://localhost:8000
ports:
- "5173:5173"
volumes:
+21 -16
View File
@@ -5,32 +5,18 @@ configure_file(
find_package(PNG REQUIRED)
add_library(pdfengine STATIC
add_library(pdfengine OBJECT
src/core/engine_info.cpp
src/core/graphics_state.cpp
src/core/display_list.cpp
src/core/path_interpreter.cpp
src/core/skia_renderer.cpp
src/parser/pdfium_loader.cpp
src/parser/pdfium_document.cpp
src/parser/pdfium_internal.cpp
src/parser/pdfium_reflow.cpp
src/parser/pdfium_page.cpp
src/parser/pdfium_page_model.cpp
src/parser/pdfium_fonts.cpp
src/parser/pdfium_edit.cpp
src/parser/pdfium_edit_replace.cpp
src/parser/pdfium_edit_reflow.cpp
src/parser/pdfium_edit_annotations.cpp
src/parser/pdfium_edit_pages.cpp
src/parser/pdfium_edit_images.cpp
src/parser/content_stream_parser.cpp
src/parser/decoration_builder.cpp
src/text/selection.cpp
src/fonts/face/font_face.cpp
src/fonts/face/free_type_manager.cpp
src/fonts/loader/font_resolver.cpp
src/fonts/pdf_fonts/font_loader.cpp
src/fonts/shaping/hb_shaper.cpp
src/fonts/cache/glyph_bitmap.cpp
src/fonts/cache/glyph_cache.cpp
@@ -48,6 +34,7 @@ add_library(pdfengine STATIC
src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp
)
add_library(pdfengine::pdfengine ALIAS pdfengine)
set_target_properties(pdfengine PROPERTIES POSITION_INDEPENDENT_CODE ON)
target_include_directories(pdfengine
PUBLIC
@@ -69,8 +56,26 @@ target_link_libraries(pdfengine
)
if(PDFENGINE_WITH_PDFIUM)
target_sources(pdfengine PRIVATE
src/parser/pdfium_loader.cpp
src/parser/pdfium_document.cpp
src/parser/pdfium_internal.cpp
src/parser/pdfium_reflow.cpp
src/parser/pdfium_page.cpp
src/parser/pdfium_page_model.cpp
src/parser/pdfium_fonts.cpp
src/parser/pdfium_edit.cpp
src/parser/pdfium_edit_replace.cpp
src/parser/pdfium_edit_reflow.cpp
src/parser/pdfium_edit_annotations.cpp
src/parser/pdfium_edit_pages.cpp
src/parser/pdfium_edit_images.cpp
)
target_link_libraries(pdfengine PRIVATE pdfium::pdfium)
target_compile_definitions(pdfengine PRIVATE PDFENGINE_WITH_PDFIUM)
target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_PDFIUM)
# PDFium statically bundles its own libjpeg, zlib, etc. which conflicts with vcpkg.
# We use LLD, so we can safely allow multiple definitions to pick the first one.
target_link_options(pdfengine PUBLIC "-Wl,--allow-multiple-definition")
endif()
if(EMSCRIPTEN)
+5 -5
View File
@@ -32,34 +32,34 @@ struct SetTransformCommand : public Command {
struct FillRectCommand : public Command {
float x, y, width, height;
FillRectCommand(float x, float y, float w, float h) : x(x), y(y), width(w), height(h) {}
FillRectCommand(float _x, float _y, float w, float h) : x(_x), y(_y), width(w), height(h) {}
void accept(CommandVisitor& visitor) const override;
};
struct DrawTextCommand : public Command {
std::string text;
float x, y;
DrawTextCommand(std::string text, float x, float y) : text(std::move(text)), x(x), y(y) {}
DrawTextCommand(std::string _text, float _x, float _y) : text(std::move(_text)), x(_x), y(_y) {}
void accept(CommandVisitor& visitor) const override;
};
struct FillPathCommand : public Command {
Path path;
FillRule rule;
explicit FillPathCommand(Path path, FillRule rule = FillRule::NonZero) : path(std::move(path)), rule(rule) {}
explicit FillPathCommand(Path _path, FillRule _rule = FillRule::NonZero) : path(std::move(_path)), rule(_rule) {}
void accept(CommandVisitor& visitor) const override;
};
struct StrokePathCommand : public Command {
Path path;
explicit StrokePathCommand(Path path) : path(std::move(path)) {}
explicit StrokePathCommand(Path _path) : path(std::move(_path)) {}
void accept(CommandVisitor& visitor) const override;
};
struct FillStrokePathCommand : public Command {
Path path;
FillRule rule;
explicit FillStrokePathCommand(Path path, FillRule rule = FillRule::NonZero) : path(std::move(path)), rule(rule) {}
explicit FillStrokePathCommand(Path _path, FillRule _rule = FillRule::NonZero) : path(std::move(_path)), rule(_rule) {}
void accept(CommandVisitor& visitor) const override;
};
+2 -2
View File
@@ -11,8 +11,8 @@ struct Matrix {
float e = 0.0f, f = 0.0f;
Matrix() = default;
Matrix(float a, float b, float c, float d, float e, float f)
: a(a), b(b), c(c), d(d), e(e), f(f) {}
Matrix(float _a, float _b, float _c, float _d, float _e, float _f)
: a(_a), b(_b), c(_c), d(_d), e(_e), f(_f) {}
[[nodiscard]] Matrix multiply(const Matrix& other) const noexcept;
+16 -1
View File
@@ -61,6 +61,14 @@ struct DevicePoint {
int y;
};
struct InvalidatedRegion {
int pageIndex;
double x;
double y;
double width;
double height;
};
struct GlyphBounds {
std::string text;
double x;
@@ -173,6 +181,12 @@ public:
return std::unexpected(EngineError::Unknown);
}
[[nodiscard]] virtual std::expected<PageImage, EngineError>
renderTile(int dpi, double xPt, double yPt, double wPt, double hPt) const {
(void)dpi; (void)xPt; (void)yPt; (void)wPt; (void)hPt;
return std::unexpected(EngineError::Unknown);
}
[[nodiscard]] virtual std::expected<std::string, EngineError> extractText() const = 0;
[[nodiscard]] virtual std::expected<std::vector<GlyphBounds>, EngineError> extractTextWithBounds() const = 0;
@@ -259,7 +273,8 @@ public:
[[nodiscard]] virtual std::expected<std::shared_ptr<fonts::pdf_fonts::Font>, std::string>
getResolvedFont(const FontInfo& fontInfo) = 0;
virtual std::expected<void, EngineError> applyEdits(const std::string& editsJson) = 0;
virtual std::expected<std::vector<InvalidatedRegion>, EngineError> applyEdits(const std::string& editsJson) = 0;
[[nodiscard]] virtual std::string lastReflowLayout() const { return {}; }
+3 -1
View File
@@ -48,7 +48,9 @@ public:
std::expected<PageImage, EngineError> render(int dpi = 96) const override;
std::expected<PageImage, EngineError> renderRegionRaw(int dpi, double yTopPt, double heightPt) const override;
std::expected<PageImage, EngineError> renderTile(int dpi, double xPt, double yPt, double wPt, double hPt) const override;
std::expected<std::string, EngineError> extractText() const override;
std::expected<std::vector<GlyphBounds>, EngineError> extractTextWithBounds() const override;
std::expected<PageModel, EngineError> extractDocumentModel() const override;
std::expected<std::vector<FontInfo>, EngineError> getFonts() const override;
@@ -100,7 +102,7 @@ public:
void invalidateCaches();
std::expected<void, EngineError> applyEdits(const std::string& editsJson) override;
std::expected<std::vector<InvalidatedRegion>, EngineError> applyEdits(const std::string& editsJson) override;
std::expected<std::vector<uint8_t>, EngineError> saveIncremental() const override;
std::expected<std::vector<uint8_t>, EngineError> saveFull() const override;
std::expected<std::vector<uint8_t>, EngineError> saveFullForExport() const override;
+33 -3
View File
@@ -8,7 +8,7 @@
namespace pdfengine::parser {
std::expected<void, EngineError> PdfiumDocument::applyEdits(const std::string& editsJson) {
std::expected<std::vector<InvalidatedRegion>, EngineError> PdfiumDocument::applyEdits(const std::string& editsJson) {
#ifdef PDFENGINE_WITH_PDFIUM
if (!doc_) {
return std::unexpected(EngineError::Unknown);
@@ -17,7 +17,9 @@ std::expected<void, EngineError> PdfiumDocument::applyEdits(const std::string& e
lastReflowLayout_.clear();
std::vector<int> editedPages;
auto markEdited = [&editedPages](int p) {
std::vector<InvalidatedRegion> invalidatedRegions;
auto markEdited = [&](int p) {
if (std::find(editedPages.begin(), editedPages.end(), p) == editedPages.end())
editedPages.push_back(p);
};
@@ -89,6 +91,34 @@ std::expected<void, EngineError> PdfiumDocument::applyEdits(const std::string& e
if (!r) {
return std::unexpected(r.error());
}
// Calculate invalidated region
bool regionFound = false;
if (op.contains("data") && op["data"].is_object()) {
const auto& data = op["data"];
if (data.contains("x") && data.contains("y") && data.contains("width") && data.contains("height")) {
invalidatedRegions.push_back({
pageIndex,
data.value("x", 0.0),
data.value("y", 0.0),
data.value("width", 0.0),
data.value("height", 0.0)
});
regionFound = true;
}
}
if (!regionFound) {
auto pageOpt = getPage(pageIndex);
if (pageOpt) {
invalidatedRegions.push_back({
pageIndex,
0.0,
0.0,
(*pageOpt)->width(),
(*pageOpt)->height()
});
}
}
}
} catch (const nlohmann::json::parse_error& e) {
spdlog::error("JSON parse error in applyEdits: {}", e.what());
@@ -100,7 +130,7 @@ std::expected<void, EngineError> PdfiumDocument::applyEdits(const std::string& e
rebalanceEditedPages(editedPages);
invalidateCaches();
return {};
return invalidatedRegions;
#else
(void)editsJson;
return std::unexpected(EngineError::Unknown);
+43
View File
@@ -154,6 +154,49 @@ std::expected<PageImage, EngineError> PdfiumPage::renderRegionRaw(int dpi, doubl
#endif
}
std::expected<PageImage, EngineError> PdfiumPage::renderTile(int dpi, double xPt, double yPt, double wPt, double hPt) const {
#ifdef PDFENGINE_WITH_PDFIUM
if (!page_) return std::unexpected(EngineError::Unknown);
if (!limits::pageDimensionsOk(width(), height())) return std::unexpected(EngineError::RenderFailed);
if (!limits::objectCountOk(FPDFPage_CountObjects(page_))) return std::unexpected(EngineError::RenderFailed);
const double scale = dpi / 72.0;
const int w = static_cast<int>(wPt * scale);
const int h = static_cast<int>(hPt * scale);
int start_x = -static_cast<int>(xPt * scale);
int start_y = -static_cast<int>(yPt * scale);
const int fullW = static_cast<int>(width() * scale);
const int fullH = static_cast<int>(height() * scale);
if (w <= 0 || h <= 0) return std::unexpected(EngineError::RenderFailed);
if (!limits::rasterSizeOk(w, h)) return std::unexpected(EngineError::RenderFailed);
FPDF_BITMAP bitmap = FPDFBitmap_Create(w, h, 1);
if (!bitmap) return std::unexpected(EngineError::RenderFailed);
FPDFBitmap_FillRect(bitmap, 0, 0, w, h, 0xFFFFFFFF);
FPDF_RenderPageBitmap(bitmap, page_, start_x, start_y, fullW, fullH, 0, 0);
const auto* bgra = static_cast<const uint8_t*>(FPDFBitmap_GetBuffer(bitmap));
const int stride = FPDFBitmap_GetStride(bitmap);
std::vector<uint8_t> rgba(static_cast<size_t>(w) * h * 4);
for (int y = 0; y < h; ++y) {
const uint8_t* src = bgra + static_cast<size_t>(y) * stride;
uint8_t* dst = rgba.data() + static_cast<size_t>(y) * w * 4;
for (int x = 0; x < w; ++x) {
dst[x * 4 + 0] = src[x * 4 + 2];
dst[x * 4 + 1] = src[x * 4 + 1];
dst[x * 4 + 2] = src[x * 4 + 0];
dst[x * 4 + 3] = src[x * 4 + 3];
}
}
FPDFBitmap_Destroy(bitmap);
return PageImage{w, h, std::move(rgba)};
#else
(void)dpi; (void)xPt; (void)yPt; (void)wPt; (void)hPt;
return std::unexpected(EngineError::Unknown);
#endif
}
std::expected<std::string, EngineError> PdfiumPage::extractText() const {
#ifdef PDFENGINE_WITH_PDFIUM
if (!page_) {
+4
View File
@@ -38,6 +38,10 @@ if(PDFENGINE_WITH_SKIA)
target_link_libraries(pdfengine_smoke PRIVATE skia::skia)
endif()
if(PDFENGINE_WITH_PDFIUM)
target_link_libraries(pdfengine_smoke PRIVATE pdfium::pdfium)
endif()
if(PDFENGINE_WITH_QPDF)
target_link_libraries(pdfengine_smoke PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
endif()
+129
View File
@@ -1,5 +1,8 @@
#include <gtest/gtest.h>
#include <pdfengine/display_list.hpp>
#include <pdfengine/path_interpreter.hpp>
#include <pdfengine/content_object.hpp>
#include "../src/parser/content_stream_parser.hpp"
#include <string>
#include <vector>
@@ -20,6 +23,38 @@ public:
void visit(const DrawImageCommand&) override { calls.push_back("DrawImage"); }
};
class PathVerifyVisitor : public CommandVisitor {
public:
struct PathInfo {
std::string type;
size_t segmentCount = 0;
FillRule rule = FillRule::NonZero;
};
std::vector<PathInfo> paths;
std::vector<std::string> calls;
void visit(const SaveStateCommand&) override { calls.push_back("SaveState"); }
void visit(const RestoreStateCommand&) override { calls.push_back("RestoreState"); }
void visit(const SetTransformCommand& cmd) override {
calls.push_back("SetTransform(" + std::to_string(cmd.matrix.a) + "," + std::to_string(cmd.matrix.d) + ")");
}
void visit(const FillRectCommand&) override {}
void visit(const DrawTextCommand&) override {}
void visit(const FillPathCommand& cmd) override {
calls.push_back("FillPath");
paths.push_back({"Fill", cmd.path.segments().size(), cmd.rule});
}
void visit(const StrokePathCommand& cmd) override {
calls.push_back("StrokePath");
paths.push_back({"Stroke", cmd.path.segments().size(), FillRule::NonZero});
}
void visit(const FillStrokePathCommand& cmd) override {
calls.push_back("FillStrokePath");
paths.push_back({"FillStroke", cmd.path.segments().size(), cmd.rule});
}
void visit(const DrawImageCommand&) override {}
};
TEST(DisplayListTest, RecordAndReplay) {
DisplayList list;
@@ -53,3 +88,97 @@ TEST(DisplayListTest, Clear) {
list.clear();
EXPECT_EQ(list.size(), 0);
}
TEST(DisplayListTest, PathObjectInterpreterStroke) {
Path path;
path.moveTo(10.0f, 20.0f);
path.lineTo(30.0f, 40.0f);
PathObject pathObj;
pathObj.path = path;
pathObj.paintOp = PathPaintOp::Stroke;
pathObj.transform = Matrix(1.5f, 0.0f, 0.0f, 1.5f, 5.0f, 5.0f);
DisplayList list;
PathObjectInterpreter::interpret(pathObj, list);
PathVerifyVisitor visitor;
list.replay(visitor);
ASSERT_EQ(visitor.calls.size(), 4);
EXPECT_EQ(visitor.calls[0], "SaveState");
EXPECT_EQ(visitor.calls[1], "SetTransform(1.500000,1.500000)");
EXPECT_EQ(visitor.calls[2], "StrokePath");
EXPECT_EQ(visitor.calls[3], "RestoreState");
ASSERT_EQ(visitor.paths.size(), 1);
EXPECT_EQ(visitor.paths[0].type, "Stroke");
EXPECT_EQ(visitor.paths[0].segmentCount, 2);
}
TEST(DisplayListTest, PathObjectInterpreterFillEvenOdd) {
Path path;
path.addRect(0.0f, 0.0f, 10.0f, 10.0f);
PathObject pathObj;
pathObj.path = path;
pathObj.paintOp = PathPaintOp::Fill;
pathObj.fillRule = FillRule::EvenOdd;
pathObj.transform = Matrix(1.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f);
DisplayList list;
PathObjectInterpreter::interpret(pathObj, list);
PathVerifyVisitor visitor;
list.replay(visitor);
ASSERT_EQ(visitor.calls.size(), 4);
EXPECT_EQ(visitor.calls[2], "FillPath");
ASSERT_EQ(visitor.paths.size(), 1);
EXPECT_EQ(visitor.paths[0].type, "Fill");
EXPECT_EQ(visitor.paths[0].segmentCount, 5); // moveTo + 3 lineTo + close
EXPECT_EQ(visitor.paths[0].rule, FillRule::EvenOdd);
}
TEST(DisplayListTest, ContentStreamParserPaths) {
DisplayList list;
ContentStreamParser parser;
// Parse stroke path (m, l, S)
parser.parse("10 20 m 30 40 l S", list);
// Parse fill path (re, f)
parser.parse("5 6 7 8 re f", list);
// Parse fill and stroke (m, c, B)
parser.parse("1 2 m 3 4 5 6 7 8 c B", list);
PathVerifyVisitor visitor;
list.replay(visitor);
// S -> StrokePath
// f -> FillPath
// B -> FillPath, StrokePath
ASSERT_EQ(visitor.calls.size(), 4);
EXPECT_EQ(visitor.calls[0], "StrokePath");
EXPECT_EQ(visitor.calls[1], "FillPath");
EXPECT_EQ(visitor.calls[2], "FillPath");
EXPECT_EQ(visitor.calls[3], "StrokePath");
ASSERT_EQ(visitor.paths.size(), 4);
// 1st: 10 20 m 30 40 l S -> MoveTo, LineTo
EXPECT_EQ(visitor.paths[0].type, "Stroke");
EXPECT_EQ(visitor.paths[0].segmentCount, 2);
// 2nd: 5 6 7 8 re f -> MoveTo, 3xLineTo, Close
EXPECT_EQ(visitor.paths[1].type, "Fill");
EXPECT_EQ(visitor.paths[1].segmentCount, 5);
// 3rd & 4th: 1 2 m 3 4 5 6 7 8 c B -> MoveTo, CubicBezierTo
EXPECT_EQ(visitor.paths[2].type, "Fill");
EXPECT_EQ(visitor.paths[2].segmentCount, 2);
EXPECT_EQ(visitor.paths[3].type, "Stroke");
EXPECT_EQ(visitor.paths[3].segmentCount, 2);
}
+4 -1
View File
@@ -262,7 +262,8 @@ std::string decodedStream(QPDFObjectHandle contents) {
}
std::vector<std::unique_ptr<ContentObject>> buildPageObjects(QPDFObjectHandle page) {
Lexer lexer(decodedStream(page.getKey("/Contents")));
std::string decodedContent = decodedStream(page.getKey("/Contents"));
Lexer lexer(decodedContent);
auto tokens = lexer.tokenize();
ContentParser parser(tokens);
auto operations = parser.parse();
@@ -436,6 +437,8 @@ TEST(ImageXObjectVerification, JpegLogoAndPhotoDecodeAndMatchPdfiumRender) {
QPDF qpdf;
auto objects = buildFirstPageObjects(qpdf, pdf);
auto images = imageObjects(objects);
ASSERT_EQ(images.size(), 2u);
+20 -1
View File
@@ -435,7 +435,14 @@ class GatewayService {
}
}
private renderCache = new Map<string, string>();
async renderPage(params: RenderParams): Promise<string> {
const cacheKey = `${params.documentId}_${params.pageIndex}_${params.zoom}_${params.rotation}`;
if (this.renderCache.has(cacheKey)) {
return this.renderCache.get(cacheKey)!;
}
try {
const query = new URLSearchParams({
page: params.pageIndex.toString(),
@@ -452,7 +459,19 @@ class GatewayService {
if (!response.ok) throw new Error(`Page render failed: ${response.statusText}`);
const blob = await response.blob();
return URL.createObjectURL(blob);
const objectUrl = URL.createObjectURL(blob);
this.renderCache.set(cacheKey, objectUrl);
if (this.renderCache.size > 100) {
const firstKey = this.renderCache.keys().next().value;
if (firstKey) {
const oldUrl = this.renderCache.get(firstKey)!;
if (oldUrl.startsWith('blob:')) URL.revokeObjectURL(oldUrl);
this.renderCache.delete(firstKey);
}
}
return objectUrl;
} catch (err) {
return this.generateMockPage(params.pageIndex);
}
+92 -11
View File
@@ -1,3 +1,84 @@
# Stage 1: Builder
FROM python:3.11-slim AS builder
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
VCPKG_FORCE_SYSTEM_BINARIES=1
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
cmake \
ninja-build \
pkg-config \
git \
fonts-liberation \
libjpeg-dev \
zlib1g-dev \
libpng-dev \
curl \
zip \
unzip \
tar \
autoconf \
autoconf-archive \
automake \
libtool \
lld \
&& rm -rf /var/lib/apt/lists/* \
&& ln -sf /usr/bin/ld.lld /usr/bin/ld
WORKDIR /build
# Install vcpkg
RUN git clone https://github.com/microsoft/vcpkg.git /opt/vcpkg \
&& /opt/vcpkg/bootstrap-vcpkg.sh -disableMetrics
ENV VCPKG_ROOT=/opt/vcpkg
# Cache vcpkg dependencies in a separate layer
COPY vcpkg.json ./
RUN --mount=type=cache,target=/root/.cache \
--mount=type=cache,target=/opt/vcpkg/downloads \
/opt/vcpkg/vcpkg install --triplet x64-linux
# Install depot_tools globally with caching
RUN --mount=type=cache,target=/opt/depot_tools \
if [ ! -d /opt/depot_tools/.git ]; then \
git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git /opt/depot_tools; \
fi
ENV PATH="/opt/depot_tools:${PATH}"
# Build PDFium for Linux (heavily cached, keeping the huge source tree out of the image layer)
COPY third_party/pdfium/ ./third_party/pdfium/
RUN --mount=type=cache,target=/build/third_party/pdfium/checkout \
./third_party/pdfium/build_pdfium.sh
# Copy everything needed for the engine and bindings build
COPY CMakeLists.txt CMakePresets.json ./
COPY cmake/ ./cmake/
COPY engine/ ./engine/
COPY bindings/ ./bindings/
COPY gateway/ ./gateway/
COPY corpus/ ./corpus/
# Configure CMake with tests enabled
RUN --mount=type=cache,target=/root/.cache \
--mount=type=cache,target=/opt/vcpkg/downloads \
cmake --preset linux-release \
-DPDFENGINE_BUILD_TESTS=ON \
-DPDFENGINE_WITH_PDFIUM=ON \
-DPDFENGINE_WITH_SKIA=OFF \
-DPDFENGINE_WITH_QPDF=ON \
-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake
# Build all targets including tests and pdfengine_py
RUN cmake --build out/build/linux-release
# Stage 2: Tester
FROM builder AS tester
RUN ctest --test-dir out/build/linux-release --output-on-failure
# Stage 3: Runtime
FROM python:3.11-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
@@ -8,27 +89,27 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
cmake \
ninja-build \
pkg-config \
git \
libjpeg-dev \
zlib1g-dev \
libpng-dev \
libjpeg62-turbo \
zlib1g \
libpng16-16 \
fonts-liberation \
&& rm -rf /var/lib/apt/lists/*
# Install the extension globally so it's not shadowed by the volume mount ./gateway:/home/app
COPY --from=tester /build/gateway/pdfengine*.so /usr/local/lib/python3.11/site-packages/
RUN groupadd --system app \
&& useradd --system --gid app --create-home --home-dir /home/app app
WORKDIR /home/app
COPY pyproject.toml README.md ./
# Copy gateway Python code
COPY gateway/pyproject.toml gateway/README.md ./
RUN python -m pip install --upgrade pip \
&& pip install --no-cache-dir -e ".[dev]"
COPY app ./app
COPY tests ./tests
COPY gateway/app ./app
COPY gateway/tests ./tests
RUN chown -R app:app /home/app
USER app
+11
View File
@@ -26,6 +26,17 @@ class Settings(BaseSettings):
"Phase 0 default is False — routes that need the engine return 501."
),
)
render_cache_enabled: bool = Field(default=True)
render_cache_max_entries: int = Field(default=256)
render_cache_max_bytes: int = Field(
default=536_870_912,
description="Max total bytes for the tile render cache. Set to 0 to disable byte cap."
)
render_cache_max_entry_bytes: int = Field(
default=8_388_608,
description="Max bytes for a single cache entry. Oversized tiles are rendered but not cached."
)
@lru_cache(maxsize=1)
+2 -1
View File
@@ -4,7 +4,7 @@ from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app import __version__
from app.routers import documents, edits, health, info, render
from app.routers import documents, edits, health, info, internal, render
def create_app() -> FastAPI:
@@ -29,6 +29,7 @@ def create_app() -> FastAPI:
app.include_router(render.compat_router)
app.include_router(edits.router)
app.include_router(edits.compat_router)
app.include_router(internal.router)
@app.get("/")
def read_root():
+5 -2
View File
@@ -426,7 +426,7 @@ def apply_edits_impl(document_id: str, request: EditsRequest):
edits_json = json.dumps(req_dict)
doc_copy = pdfengine.PdfDocument.load_from_memory(doc_info["bytes_data"])
doc_copy.apply_edits(edits_json)
invalidated_regions = doc_copy.apply_edits(edits_json)
full_save_types = {"redaction", "replace_text", "reflow_paragraph"}
needs_full = any(op.get("type") in full_save_types for op in req_dict.get("operations", []))
@@ -438,8 +438,11 @@ def apply_edits_impl(document_id: str, request: EditsRequest):
filename=doc_info["filename"], bytes_data=new_bytes, doc_instance=new_doc,
permissions=doc_info.get("permissions"),
)
from app.services.render_cache import tile_cache
tile_cache.invalidate_doc(doc_info.get("doc_hash", ""))
return {"success": True, "newDocumentId": new_info["id"]}
return {"success": True, "newDocumentId": new_info["id"], "invalidatedRegions": invalidated_regions}
except Exception as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) from e
finally:
+20
View File
@@ -0,0 +1,20 @@
from fastapi import APIRouter
from app.services.render_cache import tile_cache
router = APIRouter(prefix="/internal", tags=["internal"])
@router.get("/cache/stats")
def cache_stats():
stats = tile_cache.stats()
return {
"hits": stats.hits,
"misses": stats.misses,
"evictions": stats.evictions,
"current_entries": stats.current_entries,
"current_bytes": stats.current_bytes,
"max_bytes": stats.max_bytes,
"hit_rate": stats.hit_rate,
"avg_lookup_ns": stats.avg_lookup_ns,
"avg_render_time_ns": stats.avg_render_time_ns,
}
+135 -5
View File
@@ -1,19 +1,30 @@
import hashlib
from typing import Annotated
from fastapi import APIRouter, HTTPException, Path, Query, Response, status
from fastapi import APIRouter, HTTPException, Path, Query, Request, Response, status
from app.schemas.font import FontInfoResponse
from app.services import engine
from app.services.font import font_info_to_response
from app.services.render_cache import RENDERER_VERSION, RenderMode, TileCacheKey, tile_cache
from app.services.store import document_store
router = APIRouter(prefix="/documents/{document_id}/pages", tags=["render"])
compat_router = APIRouter(tags=["render"])
def _etag(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()[:16]
@router.get("/{page_index}/render")
def render_page(
document_id: str, page_index: Annotated[int, Path(ge=0)], dpi: int = 96
request: Request,
document_id: str,
page_index: Annotated[int, Path(ge=0)],
dpi: int = 96,
zoom: float = 1.0,
rotation: int = 0,
render_mode: RenderMode = RenderMode.NORMAL
) -> Response:
if not engine.is_available():
raise HTTPException(
@@ -28,8 +39,127 @@ def render_page(
try:
doc = doc_info["doc_instance"]
page = doc.get_page(page_index)
key = TileCacheKey(
doc_hash=doc_info.get("doc_hash", ""),
renderer_version=RENDERER_VERSION,
page=page_index,
dpi=dpi,
zoom=zoom,
rotation=rotation,
render_mode=render_mode,
tile_x=0.0,
tile_y=0.0,
tile_w=round(page.width, 2),
tile_h=round(page.height, 2)
)
cached = tile_cache.get(key)
if cached:
etag_val = f'"{_etag(cached)}"'
if request.headers.get("if-none-match") == etag_val:
return Response(status_code=304)
return Response(
content=cached,
media_type="image/png",
headers={"ETag": etag_val, "X-Cache": "HIT", "Cache-Control": "private, max-age=300"}
)
import time
start = time.perf_counter_ns()
img = page.render(dpi)
return Response(content=img.data, media_type="image/png")
elapsed = time.perf_counter_ns() - start
tile_cache.put(key, img.data)
tile_cache.record_render_time(elapsed)
etag_val = f'"{_etag(img.data)}"'
return Response(
content=img.data,
media_type="image/png",
headers={"ETag": etag_val, "X-Cache": "MISS", "Cache-Control": "private, max-age=300"}
)
except IndexError:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Page index out of bounds"
)
except Exception as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
@router.get("/{page_index}/render-tile")
def render_page_tile(
request: Request,
document_id: str,
page_index: Annotated[int, Path(ge=0)],
x: float,
y: float,
width: float,
height: float,
dpi: int = 96,
zoom: float = 1.0,
rotation: int = 0,
render_mode: RenderMode = RenderMode.NORMAL
) -> Response:
if not engine.is_available():
raise HTTPException(
status_code=status.HTTP_501_NOT_IMPLEMENTED,
detail="Engine bridge (bindings/python) not yet available.",
)
doc_info = document_store.get_document(document_id)
if not doc_info:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
try:
doc = doc_info["doc_instance"]
page = doc.get_page(page_index)
key = TileCacheKey(
doc_hash=doc_info.get("doc_hash", ""),
renderer_version=RENDERER_VERSION,
page=page_index,
dpi=dpi,
zoom=zoom,
rotation=rotation,
render_mode=render_mode,
tile_x=round(x, 2),
tile_y=round(y, 2),
tile_w=round(width, 2),
tile_h=round(height, 2)
)
cached = tile_cache.get(key)
if cached:
etag_val = f'"{_etag(cached)}"'
if request.headers.get("if-none-match") == etag_val:
return Response(status_code=304)
return Response(
content=cached,
media_type="image/png",
headers={"ETag": etag_val, "X-Cache": "HIT", "Cache-Control": "private, max-age=300"}
)
import time
start = time.perf_counter_ns()
img_width, img_height, img_data = page.render_tile(dpi, x, y, width, height)
import io
from PIL import Image
img = Image.frombytes("RGBA", (img_width, img_height), img_data)
out_buf = io.BytesIO()
img.save(out_buf, format="PNG")
png_bytes = out_buf.getvalue()
elapsed = time.perf_counter_ns() - start
tile_cache.put(key, png_bytes)
tile_cache.record_render_time(elapsed)
etag_val = f'"{_etag(png_bytes)}"'
return Response(
content=png_bytes,
media_type="image/png",
headers={"ETag": etag_val, "X-Cache": "MISS", "Cache-Control": "private, max-age=300"}
)
except IndexError:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Page index out of bounds"
@@ -72,10 +202,10 @@ def extract_page_text(document_id: str, page_index: Annotated[int, Path(ge=0)]):
@compat_router.get("/render/{document_id}")
def render_page_compat(
document_id: str, page: int = 0, zoom: float = 1.0, rotation: int = 0
request: Request, document_id: str, page: int = 0, zoom: float = 1.0, rotation: int = 0
) -> Response:
dpi = int(96 * zoom)
return render_page(document_id, page, dpi)
return render_page(request, document_id, page, dpi, zoom, rotation, RenderMode.NORMAL)
@router.get("/{page_index}")
+222
View File
@@ -0,0 +1,222 @@
import threading
import time
from dataclasses import dataclass
from enum import Enum
from typing import Any
from app.config import get_settings
RENDERER_VERSION: int = 1
class RenderMode(str, Enum):
# Current
NORMAL = "normal"
PRINT = "print"
GRAYSCALE = "grayscale"
# Future (reserved)
ANNOTATION_ONLY = "annotation_only"
EDIT_PREVIEW = "edit_preview"
SELECTION_OVERLAY = "selection_overlay"
PROOF = "proof"
HIGH_QUALITY = "high_quality"
DRAFT = "draft"
@dataclass(frozen=True)
class TileCacheKey:
doc_hash: str
renderer_version: int
page: int
dpi: int
zoom: float
rotation: int
render_mode: str
tile_x: float
tile_y: float
tile_w: float
tile_h: float
@dataclass
class CacheStats:
hits: int = 0
misses: int = 0
evictions: int = 0
current_entries: int = 0
current_bytes: int = 0
max_bytes: int = 0
hit_rate: float = 0.0
avg_lookup_ns: float = 0.0
avg_render_time_ns: float = 0.0
class _Node:
__slots__ = ["key", "value", "size", "prev", "next"]
def __init__(self, key: TileCacheKey | None, value: bytes | None, size: int):
self.key = key
self.value = value
self.size = size
self.prev: _Node | None = None
self.next: _Node | None = None
class TileCache:
def __init__(self):
settings = get_settings()
self.enabled = settings.render_cache_enabled
self.max_entries = settings.render_cache_max_entries
self.max_bytes = settings.render_cache_max_bytes
self.max_entry_bytes = settings.render_cache_max_entry_bytes
self._lock = threading.Lock()
self._cache: dict[TileCacheKey, _Node] = {}
self._head = _Node(None, None, 0)
self._tail = _Node(None, None, 0)
self._head.next = self._tail
self._tail.prev = self._head
self._current_bytes = 0
# Stats
self._hits = 0
self._misses = 0
self._evictions = 0
self._total_lookup_ns = 0
self._total_render_time_ns = 0
self._render_time_count = 0
def _remove(self, node: _Node):
p = node.prev
n = node.next
if p and n:
p.next = n
n.prev = p
def _add_to_front(self, node: _Node):
first = self._head.next
if first:
self._head.next = node
node.prev = self._head
node.next = first
first.prev = node
def _evict(self):
last = self._tail.prev
if last and last != self._head:
self._remove(last)
if last.key:
del self._cache[last.key]
self._current_bytes -= last.size
self._evictions += 1
def get(self, key: TileCacheKey) -> bytes | None:
if not self.enabled:
return None
start_time = time.perf_counter_ns()
with self._lock:
node = self._cache.get(key)
if node:
self._hits += 1
self._remove(node)
self._add_to_front(node)
res = node.value
else:
self._misses += 1
res = None
lookup_time = time.perf_counter_ns() - start_time
self._total_lookup_ns += lookup_time
return res
def put(self, key: TileCacheKey, data: bytes) -> None:
if not self.enabled:
return
size = len(data)
if size > self.max_entry_bytes:
# Too large to cache
return
with self._lock:
if key in self._cache:
node = self._cache[key]
self._current_bytes -= node.size
node.value = data
node.size = size
self._current_bytes += size
self._remove(node)
self._add_to_front(node)
else:
new_node = _Node(key, data, size)
self._cache[key] = new_node
self._add_to_front(new_node)
self._current_bytes += size
# Evict if over limits
while len(self._cache) > self.max_entries or (self.max_bytes > 0 and self._current_bytes > self.max_bytes):
self._evict()
def invalidate_doc(self, doc_hash: str) -> None:
with self._lock:
keys_to_remove = [k for k in self._cache.keys() if k.doc_hash == doc_hash]
for k in keys_to_remove:
node = self._cache[k]
self._remove(node)
self._current_bytes -= node.size
del self._cache[k]
def invalidate_renderer_version(self, old_version: int) -> None:
with self._lock:
keys_to_remove = [k for k in self._cache.keys() if k.renderer_version == old_version]
for k in keys_to_remove:
node = self._cache[k]
self._remove(node)
self._current_bytes -= node.size
del self._cache[k]
def clear(self) -> None:
with self._lock:
self._cache.clear()
self._head.next = self._tail
self._tail.prev = self._head
self._current_bytes = 0
def stats(self) -> CacheStats:
with self._lock:
hit_rate = 0.0
total_reqs = self._hits + self._misses
if total_reqs > 0:
hit_rate = self._hits / total_reqs
avg_lookup = 0.0
if total_reqs > 0:
avg_lookup = self._total_lookup_ns / total_reqs
avg_render = 0.0
if self._render_time_count > 0:
avg_render = self._total_render_time_ns / self._render_time_count
return CacheStats(
hits=self._hits,
misses=self._misses,
evictions=self._evictions,
current_entries=len(self._cache),
current_bytes=self._current_bytes,
max_bytes=self.max_bytes,
hit_rate=hit_rate,
avg_lookup_ns=avg_lookup,
avg_render_time_ns=avg_render,
)
def record_render_time(self, elapsed_ns: int):
with self._lock:
self._total_render_time_ns += elapsed_ns
self._render_time_count += 1
tile_cache = TileCache()
+2
View File
@@ -1,3 +1,4 @@
import hashlib
import threading
import uuid
from datetime import UTC, datetime
@@ -57,6 +58,7 @@ class DocumentStore:
info = {
"id": doc_id,
"filename": filename,
"doc_hash": hashlib.sha256(bytes_data).hexdigest(),
"sizeBytes": len(bytes_data),
"totalPages": doc_instance.page_count,
"pageWidth": page_width,
+12 -8
View File
@@ -35,8 +35,8 @@ CHECKOUT_DIR="${BUILD_ROOT}/checkout"
echo ">> Build root: ${BUILD_ROOT}"
# --- 1. Read and validate the pinned revision -------------------------------
PDFIUM_REPO="$(grep -E '^PDFIUM_REPO=' "${PINNED_FILE}" | cut -d= -f2-)"
PDFIUM_COMMIT="$(grep -E '^PDFIUM_COMMIT=' "${PINNED_FILE}" | cut -d= -f2-)"
PDFIUM_REPO="$(grep -E '^PDFIUM_REPO=' "${PINNED_FILE}" | cut -d= -f2- | tr -d '\r')"
PDFIUM_COMMIT="$(grep -E '^PDFIUM_COMMIT=' "${PINNED_FILE}" | cut -d= -f2- | tr -d '\r')"
if [[ -z "${PDFIUM_COMMIT}" || "${PDFIUM_COMMIT}" == "REPLACE_WITH_PINNED_COMMIT_SHA" ]]; then
echo "ERROR: PDFium revision is not pinned. Edit pdfium.pinned first (see README.md)." >&2
exit 1
@@ -45,12 +45,14 @@ echo ">> PDFium pinned at ${PDFIUM_COMMIT}"
# --- 2. depot_tools ---------------------------------------------------------
mkdir -p "${BUILD_ROOT}"
if [[ ! -d "${DEPOT_TOOLS_DIR}" ]]; then
echo ">> Cloning depot_tools"
git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git \
"${DEPOT_TOOLS_DIR}"
if ! command -v gclient &> /dev/null; then
if [[ ! -d "${DEPOT_TOOLS_DIR}/.git" ]]; then
echo ">> Cloning depot_tools"
git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git \
"${DEPOT_TOOLS_DIR}"
fi
export PATH="${DEPOT_TOOLS_DIR}:${PATH}"
fi
export PATH="${DEPOT_TOOLS_DIR}:${PATH}"
# Do NOT set DEPOT_TOOLS_UPDATE=0 — depot_tools is designed to self-manage, and
# on first use it must bootstrap. Reproducibility comes from the pinned PDFium
# revision below, not from freezing depot_tools.
@@ -58,9 +60,11 @@ export PATH="${DEPOT_TOOLS_DIR}:${PATH}"
# Git settings injected per-process via GIT_CONFIG_* so the user's global git
# config is never touched. core.autocrlf=false avoids gclient seeing dependency
# checkouts as "uncommitted changes" on platforms where autocrlf is enabled.
export GIT_CONFIG_COUNT=2
export GIT_CONFIG_COUNT=4
export GIT_CONFIG_KEY_0=core.autocrlf GIT_CONFIG_VALUE_0=false
export GIT_CONFIG_KEY_1=core.filemode GIT_CONFIG_VALUE_1=false
export GIT_CONFIG_KEY_2=http.postBuffer GIT_CONFIG_VALUE_2=1048576000
export GIT_CONFIG_KEY_3=core.compression GIT_CONFIG_VALUE_3=0
# --- 3. Fetch / sync the PDFium tree ----------------------------------------
mkdir -p "${CHECKOUT_DIR}"