Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ffc846cf17 | ||
|
|
626156642e | ||
|
|
185cc03df7 | ||
|
|
a707fb5216 | ||
|
|
178e4df163 | ||
|
|
67b5aed642 | ||
|
|
f667db1f75 | ||
|
|
10775936de | ||
|
|
880e56b98b | ||
|
|
ef59f2ec83 | ||
|
|
72ed04f2ed |
+3
-1
@@ -73,7 +73,9 @@ find_package(freetype CONFIG REQUIRED)
|
||||
find_package(harfbuzz CONFIG REQUIRED)
|
||||
find_package(spdlog CONFIG REQUIRED)
|
||||
find_package(nlohmann_json CONFIG REQUIRED)
|
||||
find_package(qpdf CONFIG REQUIRED)
|
||||
if(PDFENGINE_WITH_QPDF)
|
||||
find_package(qpdf CONFIG REQUIRED)
|
||||
endif()
|
||||
if(WIN32 AND DEFINED VCPKG_TARGET_TRIPLET)
|
||||
link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/lib")
|
||||
link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/debug/lib")
|
||||
|
||||
@@ -374,6 +374,11 @@ PYBIND11_MODULE(pdfengine, m) {
|
||||
return py::make_tuple(img.width, img.height,
|
||||
py::bytes(reinterpret_cast<const char*>(img.data.data()), img.data.size()));
|
||||
}, py::arg("dpi"), py::arg("y_top_pt"), py::arg("height_pt") = 0.0)
|
||||
.def("render_tile", [](const pdfengine::PdfPage& self, int dpi, double xPt, double yPt, double wPt, double hPt) {
|
||||
auto img = get_or_throw(self.renderTile(dpi, xPt, yPt, wPt, hPt));
|
||||
return py::make_tuple(img.width, img.height,
|
||||
py::bytes(reinterpret_cast<const char*>(img.data.data()), img.data.size()));
|
||||
}, py::arg("dpi"), py::arg("xPt"), py::arg("yPt"), py::arg("wPt"), py::arg("hPt"))
|
||||
.def("extract_document_model", [](const pdfengine::PdfPage& self) {
|
||||
return get_or_throw(self.extractDocumentModel());
|
||||
})
|
||||
@@ -498,7 +503,18 @@ PYBIND11_MODULE(pdfengine, m) {
|
||||
return py::bytes(reinterpret_cast<const char*>(res->data()), res->size());
|
||||
}, py::arg("internal_font_id"))
|
||||
.def("apply_edits", [](pdfengine::PdfDocument& self, const std::string& editsJson) {
|
||||
get_or_throw(self.applyEdits(editsJson));
|
||||
auto regions = get_or_throw(self.applyEdits(editsJson));
|
||||
py::list py_regions;
|
||||
for (const auto& r : regions) {
|
||||
py::dict d;
|
||||
d["pageIndex"] = r.pageIndex;
|
||||
d["x"] = r.x;
|
||||
d["y"] = r.y;
|
||||
d["width"] = r.width;
|
||||
d["height"] = r.height;
|
||||
py_regions.append(d);
|
||||
}
|
||||
return py_regions;
|
||||
}, py::arg("edits_json"))
|
||||
.def("save_incremental", [](const pdfengine::PdfDocument& self) {
|
||||
std::vector<uint8_t> res = get_or_throw(self.saveIncremental());
|
||||
|
||||
+5
-4
@@ -1,13 +1,14 @@
|
||||
services:
|
||||
gateway:
|
||||
build:
|
||||
context: ./gateway
|
||||
dockerfile: Dockerfile
|
||||
context: .
|
||||
dockerfile: gateway/Dockerfile
|
||||
network: host
|
||||
image: pdf-engine-gateway:dev
|
||||
container_name: pdf-engine-gateway
|
||||
environment:
|
||||
PDFENGINE_ENVIRONMENT: dev
|
||||
PDFENGINE_ENGINE_AVAILABLE: "false"
|
||||
PDFENGINE_ENGINE_AVAILABLE: "true"
|
||||
PORT: 8000
|
||||
ports:
|
||||
- "8000:8000"
|
||||
@@ -29,7 +30,7 @@ services:
|
||||
image: pdf-engine-frontend:dev
|
||||
container_name: pdf-engine-frontend
|
||||
environment:
|
||||
VITE_GATEWAY_URL: http://gateway:8000
|
||||
VITE_GATEWAY_URL: http://localhost:8000
|
||||
ports:
|
||||
- "5173:5173"
|
||||
volumes:
|
||||
|
||||
+21
-16
@@ -5,32 +5,18 @@ configure_file(
|
||||
|
||||
find_package(PNG REQUIRED)
|
||||
|
||||
add_library(pdfengine STATIC
|
||||
add_library(pdfengine OBJECT
|
||||
src/core/engine_info.cpp
|
||||
src/core/graphics_state.cpp
|
||||
src/core/display_list.cpp
|
||||
src/core/path_interpreter.cpp
|
||||
src/core/skia_renderer.cpp
|
||||
src/parser/pdfium_loader.cpp
|
||||
src/parser/pdfium_document.cpp
|
||||
src/parser/pdfium_internal.cpp
|
||||
src/parser/pdfium_reflow.cpp
|
||||
src/parser/pdfium_page.cpp
|
||||
src/parser/pdfium_page_model.cpp
|
||||
src/parser/pdfium_fonts.cpp
|
||||
src/parser/pdfium_edit.cpp
|
||||
src/parser/pdfium_edit_replace.cpp
|
||||
src/parser/pdfium_edit_reflow.cpp
|
||||
src/parser/pdfium_edit_annotations.cpp
|
||||
src/parser/pdfium_edit_pages.cpp
|
||||
src/parser/pdfium_edit_images.cpp
|
||||
src/parser/content_stream_parser.cpp
|
||||
src/parser/decoration_builder.cpp
|
||||
src/text/selection.cpp
|
||||
src/fonts/face/font_face.cpp
|
||||
src/fonts/face/free_type_manager.cpp
|
||||
src/fonts/loader/font_resolver.cpp
|
||||
src/fonts/pdf_fonts/font_loader.cpp
|
||||
src/fonts/shaping/hb_shaper.cpp
|
||||
src/fonts/cache/glyph_bitmap.cpp
|
||||
src/fonts/cache/glyph_cache.cpp
|
||||
@@ -48,6 +34,7 @@ add_library(pdfengine STATIC
|
||||
src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp
|
||||
)
|
||||
add_library(pdfengine::pdfengine ALIAS pdfengine)
|
||||
set_target_properties(pdfengine PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
||||
|
||||
target_include_directories(pdfengine
|
||||
PUBLIC
|
||||
@@ -69,8 +56,26 @@ target_link_libraries(pdfengine
|
||||
)
|
||||
|
||||
if(PDFENGINE_WITH_PDFIUM)
|
||||
target_sources(pdfengine PRIVATE
|
||||
src/parser/pdfium_loader.cpp
|
||||
src/parser/pdfium_document.cpp
|
||||
src/parser/pdfium_internal.cpp
|
||||
src/parser/pdfium_reflow.cpp
|
||||
src/parser/pdfium_page.cpp
|
||||
src/parser/pdfium_page_model.cpp
|
||||
src/parser/pdfium_fonts.cpp
|
||||
src/parser/pdfium_edit.cpp
|
||||
src/parser/pdfium_edit_replace.cpp
|
||||
src/parser/pdfium_edit_reflow.cpp
|
||||
src/parser/pdfium_edit_annotations.cpp
|
||||
src/parser/pdfium_edit_pages.cpp
|
||||
src/parser/pdfium_edit_images.cpp
|
||||
)
|
||||
target_link_libraries(pdfengine PRIVATE pdfium::pdfium)
|
||||
target_compile_definitions(pdfengine PRIVATE PDFENGINE_WITH_PDFIUM)
|
||||
target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_PDFIUM)
|
||||
# PDFium statically bundles its own libjpeg, zlib, etc. which conflicts with vcpkg.
|
||||
# We use LLD, so we can safely allow multiple definitions to pick the first one.
|
||||
target_link_options(pdfengine PUBLIC "-Wl,--allow-multiple-definition")
|
||||
endif()
|
||||
|
||||
if(EMSCRIPTEN)
|
||||
|
||||
@@ -32,34 +32,34 @@ struct SetTransformCommand : public Command {
|
||||
|
||||
struct FillRectCommand : public Command {
|
||||
float x, y, width, height;
|
||||
FillRectCommand(float x, float y, float w, float h) : x(x), y(y), width(w), height(h) {}
|
||||
FillRectCommand(float _x, float _y, float w, float h) : x(_x), y(_y), width(w), height(h) {}
|
||||
void accept(CommandVisitor& visitor) const override;
|
||||
};
|
||||
|
||||
struct DrawTextCommand : public Command {
|
||||
std::string text;
|
||||
float x, y;
|
||||
DrawTextCommand(std::string text, float x, float y) : text(std::move(text)), x(x), y(y) {}
|
||||
DrawTextCommand(std::string _text, float _x, float _y) : text(std::move(_text)), x(_x), y(_y) {}
|
||||
void accept(CommandVisitor& visitor) const override;
|
||||
};
|
||||
|
||||
struct FillPathCommand : public Command {
|
||||
Path path;
|
||||
FillRule rule;
|
||||
explicit FillPathCommand(Path path, FillRule rule = FillRule::NonZero) : path(std::move(path)), rule(rule) {}
|
||||
explicit FillPathCommand(Path _path, FillRule _rule = FillRule::NonZero) : path(std::move(_path)), rule(_rule) {}
|
||||
void accept(CommandVisitor& visitor) const override;
|
||||
};
|
||||
|
||||
struct StrokePathCommand : public Command {
|
||||
Path path;
|
||||
explicit StrokePathCommand(Path path) : path(std::move(path)) {}
|
||||
explicit StrokePathCommand(Path _path) : path(std::move(_path)) {}
|
||||
void accept(CommandVisitor& visitor) const override;
|
||||
};
|
||||
|
||||
struct FillStrokePathCommand : public Command {
|
||||
Path path;
|
||||
FillRule rule;
|
||||
explicit FillStrokePathCommand(Path path, FillRule rule = FillRule::NonZero) : path(std::move(path)), rule(rule) {}
|
||||
explicit FillStrokePathCommand(Path _path, FillRule _rule = FillRule::NonZero) : path(std::move(_path)), rule(_rule) {}
|
||||
void accept(CommandVisitor& visitor) const override;
|
||||
};
|
||||
|
||||
|
||||
@@ -11,8 +11,8 @@ struct Matrix {
|
||||
float e = 0.0f, f = 0.0f;
|
||||
|
||||
Matrix() = default;
|
||||
Matrix(float a, float b, float c, float d, float e, float f)
|
||||
: a(a), b(b), c(c), d(d), e(e), f(f) {}
|
||||
Matrix(float _a, float _b, float _c, float _d, float _e, float _f)
|
||||
: a(_a), b(_b), c(_c), d(_d), e(_e), f(_f) {}
|
||||
|
||||
[[nodiscard]] Matrix multiply(const Matrix& other) const noexcept;
|
||||
|
||||
|
||||
@@ -61,6 +61,14 @@ struct DevicePoint {
|
||||
int y;
|
||||
};
|
||||
|
||||
struct InvalidatedRegion {
|
||||
int pageIndex;
|
||||
double x;
|
||||
double y;
|
||||
double width;
|
||||
double height;
|
||||
};
|
||||
|
||||
struct GlyphBounds {
|
||||
std::string text;
|
||||
double x;
|
||||
@@ -173,6 +181,12 @@ public:
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
}
|
||||
|
||||
[[nodiscard]] virtual std::expected<PageImage, EngineError>
|
||||
renderTile(int dpi, double xPt, double yPt, double wPt, double hPt) const {
|
||||
(void)dpi; (void)xPt; (void)yPt; (void)wPt; (void)hPt;
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
}
|
||||
|
||||
[[nodiscard]] virtual std::expected<std::string, EngineError> extractText() const = 0;
|
||||
|
||||
[[nodiscard]] virtual std::expected<std::vector<GlyphBounds>, EngineError> extractTextWithBounds() const = 0;
|
||||
@@ -259,7 +273,8 @@ public:
|
||||
[[nodiscard]] virtual std::expected<std::shared_ptr<fonts::pdf_fonts::Font>, std::string>
|
||||
getResolvedFont(const FontInfo& fontInfo) = 0;
|
||||
|
||||
virtual std::expected<void, EngineError> applyEdits(const std::string& editsJson) = 0;
|
||||
virtual std::expected<std::vector<InvalidatedRegion>, EngineError> applyEdits(const std::string& editsJson) = 0;
|
||||
|
||||
|
||||
[[nodiscard]] virtual std::string lastReflowLayout() const { return {}; }
|
||||
|
||||
|
||||
@@ -48,7 +48,9 @@ public:
|
||||
|
||||
std::expected<PageImage, EngineError> render(int dpi = 96) const override;
|
||||
std::expected<PageImage, EngineError> renderRegionRaw(int dpi, double yTopPt, double heightPt) const override;
|
||||
std::expected<PageImage, EngineError> renderTile(int dpi, double xPt, double yPt, double wPt, double hPt) const override;
|
||||
std::expected<std::string, EngineError> extractText() const override;
|
||||
|
||||
std::expected<std::vector<GlyphBounds>, EngineError> extractTextWithBounds() const override;
|
||||
std::expected<PageModel, EngineError> extractDocumentModel() const override;
|
||||
std::expected<std::vector<FontInfo>, EngineError> getFonts() const override;
|
||||
@@ -100,7 +102,7 @@ public:
|
||||
|
||||
void invalidateCaches();
|
||||
|
||||
std::expected<void, EngineError> applyEdits(const std::string& editsJson) override;
|
||||
std::expected<std::vector<InvalidatedRegion>, EngineError> applyEdits(const std::string& editsJson) override;
|
||||
std::expected<std::vector<uint8_t>, EngineError> saveIncremental() const override;
|
||||
std::expected<std::vector<uint8_t>, EngineError> saveFull() const override;
|
||||
std::expected<std::vector<uint8_t>, EngineError> saveFullForExport() const override;
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
namespace pdfengine::parser {
|
||||
|
||||
std::expected<void, EngineError> PdfiumDocument::applyEdits(const std::string& editsJson) {
|
||||
std::expected<std::vector<InvalidatedRegion>, EngineError> PdfiumDocument::applyEdits(const std::string& editsJson) {
|
||||
#ifdef PDFENGINE_WITH_PDFIUM
|
||||
if (!doc_) {
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
@@ -17,7 +17,9 @@ std::expected<void, EngineError> PdfiumDocument::applyEdits(const std::string& e
|
||||
lastReflowLayout_.clear();
|
||||
|
||||
std::vector<int> editedPages;
|
||||
auto markEdited = [&editedPages](int p) {
|
||||
std::vector<InvalidatedRegion> invalidatedRegions;
|
||||
|
||||
auto markEdited = [&](int p) {
|
||||
if (std::find(editedPages.begin(), editedPages.end(), p) == editedPages.end())
|
||||
editedPages.push_back(p);
|
||||
};
|
||||
@@ -89,6 +91,34 @@ std::expected<void, EngineError> PdfiumDocument::applyEdits(const std::string& e
|
||||
if (!r) {
|
||||
return std::unexpected(r.error());
|
||||
}
|
||||
|
||||
// Calculate invalidated region
|
||||
bool regionFound = false;
|
||||
if (op.contains("data") && op["data"].is_object()) {
|
||||
const auto& data = op["data"];
|
||||
if (data.contains("x") && data.contains("y") && data.contains("width") && data.contains("height")) {
|
||||
invalidatedRegions.push_back({
|
||||
pageIndex,
|
||||
data.value("x", 0.0),
|
||||
data.value("y", 0.0),
|
||||
data.value("width", 0.0),
|
||||
data.value("height", 0.0)
|
||||
});
|
||||
regionFound = true;
|
||||
}
|
||||
}
|
||||
if (!regionFound) {
|
||||
auto pageOpt = getPage(pageIndex);
|
||||
if (pageOpt) {
|
||||
invalidatedRegions.push_back({
|
||||
pageIndex,
|
||||
0.0,
|
||||
0.0,
|
||||
(*pageOpt)->width(),
|
||||
(*pageOpt)->height()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (const nlohmann::json::parse_error& e) {
|
||||
spdlog::error("JSON parse error in applyEdits: {}", e.what());
|
||||
@@ -100,7 +130,7 @@ std::expected<void, EngineError> PdfiumDocument::applyEdits(const std::string& e
|
||||
|
||||
rebalanceEditedPages(editedPages);
|
||||
invalidateCaches();
|
||||
return {};
|
||||
return invalidatedRegions;
|
||||
#else
|
||||
(void)editsJson;
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
|
||||
@@ -154,6 +154,49 @@ std::expected<PageImage, EngineError> PdfiumPage::renderRegionRaw(int dpi, doubl
|
||||
#endif
|
||||
}
|
||||
|
||||
std::expected<PageImage, EngineError> PdfiumPage::renderTile(int dpi, double xPt, double yPt, double wPt, double hPt) const {
|
||||
#ifdef PDFENGINE_WITH_PDFIUM
|
||||
if (!page_) return std::unexpected(EngineError::Unknown);
|
||||
if (!limits::pageDimensionsOk(width(), height())) return std::unexpected(EngineError::RenderFailed);
|
||||
if (!limits::objectCountOk(FPDFPage_CountObjects(page_))) return std::unexpected(EngineError::RenderFailed);
|
||||
|
||||
const double scale = dpi / 72.0;
|
||||
const int w = static_cast<int>(wPt * scale);
|
||||
const int h = static_cast<int>(hPt * scale);
|
||||
int start_x = -static_cast<int>(xPt * scale);
|
||||
int start_y = -static_cast<int>(yPt * scale);
|
||||
const int fullW = static_cast<int>(width() * scale);
|
||||
const int fullH = static_cast<int>(height() * scale);
|
||||
|
||||
if (w <= 0 || h <= 0) return std::unexpected(EngineError::RenderFailed);
|
||||
if (!limits::rasterSizeOk(w, h)) return std::unexpected(EngineError::RenderFailed);
|
||||
|
||||
FPDF_BITMAP bitmap = FPDFBitmap_Create(w, h, 1);
|
||||
if (!bitmap) return std::unexpected(EngineError::RenderFailed);
|
||||
FPDFBitmap_FillRect(bitmap, 0, 0, w, h, 0xFFFFFFFF);
|
||||
FPDF_RenderPageBitmap(bitmap, page_, start_x, start_y, fullW, fullH, 0, 0);
|
||||
|
||||
const auto* bgra = static_cast<const uint8_t*>(FPDFBitmap_GetBuffer(bitmap));
|
||||
const int stride = FPDFBitmap_GetStride(bitmap);
|
||||
std::vector<uint8_t> rgba(static_cast<size_t>(w) * h * 4);
|
||||
for (int y = 0; y < h; ++y) {
|
||||
const uint8_t* src = bgra + static_cast<size_t>(y) * stride;
|
||||
uint8_t* dst = rgba.data() + static_cast<size_t>(y) * w * 4;
|
||||
for (int x = 0; x < w; ++x) {
|
||||
dst[x * 4 + 0] = src[x * 4 + 2];
|
||||
dst[x * 4 + 1] = src[x * 4 + 1];
|
||||
dst[x * 4 + 2] = src[x * 4 + 0];
|
||||
dst[x * 4 + 3] = src[x * 4 + 3];
|
||||
}
|
||||
}
|
||||
FPDFBitmap_Destroy(bitmap);
|
||||
return PageImage{w, h, std::move(rgba)};
|
||||
#else
|
||||
(void)dpi; (void)xPt; (void)yPt; (void)wPt; (void)hPt;
|
||||
return std::unexpected(EngineError::Unknown);
|
||||
#endif
|
||||
}
|
||||
|
||||
std::expected<std::string, EngineError> PdfiumPage::extractText() const {
|
||||
#ifdef PDFENGINE_WITH_PDFIUM
|
||||
if (!page_) {
|
||||
|
||||
@@ -38,6 +38,10 @@ if(PDFENGINE_WITH_SKIA)
|
||||
target_link_libraries(pdfengine_smoke PRIVATE skia::skia)
|
||||
endif()
|
||||
|
||||
if(PDFENGINE_WITH_PDFIUM)
|
||||
target_link_libraries(pdfengine_smoke PRIVATE pdfium::pdfium)
|
||||
endif()
|
||||
|
||||
if(PDFENGINE_WITH_QPDF)
|
||||
target_link_libraries(pdfengine_smoke PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG)
|
||||
endif()
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <pdfengine/display_list.hpp>
|
||||
#include <pdfengine/path_interpreter.hpp>
|
||||
#include <pdfengine/content_object.hpp>
|
||||
#include "../src/parser/content_stream_parser.hpp"
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
@@ -20,6 +23,38 @@ public:
|
||||
void visit(const DrawImageCommand&) override { calls.push_back("DrawImage"); }
|
||||
};
|
||||
|
||||
class PathVerifyVisitor : public CommandVisitor {
|
||||
public:
|
||||
struct PathInfo {
|
||||
std::string type;
|
||||
size_t segmentCount = 0;
|
||||
FillRule rule = FillRule::NonZero;
|
||||
};
|
||||
std::vector<PathInfo> paths;
|
||||
std::vector<std::string> calls;
|
||||
|
||||
void visit(const SaveStateCommand&) override { calls.push_back("SaveState"); }
|
||||
void visit(const RestoreStateCommand&) override { calls.push_back("RestoreState"); }
|
||||
void visit(const SetTransformCommand& cmd) override {
|
||||
calls.push_back("SetTransform(" + std::to_string(cmd.matrix.a) + "," + std::to_string(cmd.matrix.d) + ")");
|
||||
}
|
||||
void visit(const FillRectCommand&) override {}
|
||||
void visit(const DrawTextCommand&) override {}
|
||||
void visit(const FillPathCommand& cmd) override {
|
||||
calls.push_back("FillPath");
|
||||
paths.push_back({"Fill", cmd.path.segments().size(), cmd.rule});
|
||||
}
|
||||
void visit(const StrokePathCommand& cmd) override {
|
||||
calls.push_back("StrokePath");
|
||||
paths.push_back({"Stroke", cmd.path.segments().size(), FillRule::NonZero});
|
||||
}
|
||||
void visit(const FillStrokePathCommand& cmd) override {
|
||||
calls.push_back("FillStrokePath");
|
||||
paths.push_back({"FillStroke", cmd.path.segments().size(), cmd.rule});
|
||||
}
|
||||
void visit(const DrawImageCommand&) override {}
|
||||
};
|
||||
|
||||
TEST(DisplayListTest, RecordAndReplay) {
|
||||
DisplayList list;
|
||||
|
||||
@@ -53,3 +88,97 @@ TEST(DisplayListTest, Clear) {
|
||||
list.clear();
|
||||
EXPECT_EQ(list.size(), 0);
|
||||
}
|
||||
|
||||
TEST(DisplayListTest, PathObjectInterpreterStroke) {
|
||||
Path path;
|
||||
path.moveTo(10.0f, 20.0f);
|
||||
path.lineTo(30.0f, 40.0f);
|
||||
|
||||
PathObject pathObj;
|
||||
pathObj.path = path;
|
||||
pathObj.paintOp = PathPaintOp::Stroke;
|
||||
pathObj.transform = Matrix(1.5f, 0.0f, 0.0f, 1.5f, 5.0f, 5.0f);
|
||||
|
||||
DisplayList list;
|
||||
PathObjectInterpreter::interpret(pathObj, list);
|
||||
|
||||
PathVerifyVisitor visitor;
|
||||
list.replay(visitor);
|
||||
|
||||
ASSERT_EQ(visitor.calls.size(), 4);
|
||||
EXPECT_EQ(visitor.calls[0], "SaveState");
|
||||
EXPECT_EQ(visitor.calls[1], "SetTransform(1.500000,1.500000)");
|
||||
EXPECT_EQ(visitor.calls[2], "StrokePath");
|
||||
EXPECT_EQ(visitor.calls[3], "RestoreState");
|
||||
|
||||
ASSERT_EQ(visitor.paths.size(), 1);
|
||||
EXPECT_EQ(visitor.paths[0].type, "Stroke");
|
||||
EXPECT_EQ(visitor.paths[0].segmentCount, 2);
|
||||
}
|
||||
|
||||
TEST(DisplayListTest, PathObjectInterpreterFillEvenOdd) {
|
||||
Path path;
|
||||
path.addRect(0.0f, 0.0f, 10.0f, 10.0f);
|
||||
|
||||
PathObject pathObj;
|
||||
pathObj.path = path;
|
||||
pathObj.paintOp = PathPaintOp::Fill;
|
||||
pathObj.fillRule = FillRule::EvenOdd;
|
||||
pathObj.transform = Matrix(1.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
|
||||
DisplayList list;
|
||||
PathObjectInterpreter::interpret(pathObj, list);
|
||||
|
||||
PathVerifyVisitor visitor;
|
||||
list.replay(visitor);
|
||||
|
||||
ASSERT_EQ(visitor.calls.size(), 4);
|
||||
EXPECT_EQ(visitor.calls[2], "FillPath");
|
||||
|
||||
ASSERT_EQ(visitor.paths.size(), 1);
|
||||
EXPECT_EQ(visitor.paths[0].type, "Fill");
|
||||
EXPECT_EQ(visitor.paths[0].segmentCount, 5); // moveTo + 3 lineTo + close
|
||||
EXPECT_EQ(visitor.paths[0].rule, FillRule::EvenOdd);
|
||||
}
|
||||
|
||||
TEST(DisplayListTest, ContentStreamParserPaths) {
|
||||
DisplayList list;
|
||||
ContentStreamParser parser;
|
||||
|
||||
// Parse stroke path (m, l, S)
|
||||
parser.parse("10 20 m 30 40 l S", list);
|
||||
|
||||
// Parse fill path (re, f)
|
||||
parser.parse("5 6 7 8 re f", list);
|
||||
|
||||
// Parse fill and stroke (m, c, B)
|
||||
parser.parse("1 2 m 3 4 5 6 7 8 c B", list);
|
||||
|
||||
PathVerifyVisitor visitor;
|
||||
list.replay(visitor);
|
||||
|
||||
// S -> StrokePath
|
||||
// f -> FillPath
|
||||
// B -> FillPath, StrokePath
|
||||
ASSERT_EQ(visitor.calls.size(), 4);
|
||||
EXPECT_EQ(visitor.calls[0], "StrokePath");
|
||||
EXPECT_EQ(visitor.calls[1], "FillPath");
|
||||
EXPECT_EQ(visitor.calls[2], "FillPath");
|
||||
EXPECT_EQ(visitor.calls[3], "StrokePath");
|
||||
|
||||
ASSERT_EQ(visitor.paths.size(), 4);
|
||||
|
||||
// 1st: 10 20 m 30 40 l S -> MoveTo, LineTo
|
||||
EXPECT_EQ(visitor.paths[0].type, "Stroke");
|
||||
EXPECT_EQ(visitor.paths[0].segmentCount, 2);
|
||||
|
||||
// 2nd: 5 6 7 8 re f -> MoveTo, 3xLineTo, Close
|
||||
EXPECT_EQ(visitor.paths[1].type, "Fill");
|
||||
EXPECT_EQ(visitor.paths[1].segmentCount, 5);
|
||||
|
||||
// 3rd & 4th: 1 2 m 3 4 5 6 7 8 c B -> MoveTo, CubicBezierTo
|
||||
EXPECT_EQ(visitor.paths[2].type, "Fill");
|
||||
EXPECT_EQ(visitor.paths[2].segmentCount, 2);
|
||||
EXPECT_EQ(visitor.paths[3].type, "Stroke");
|
||||
EXPECT_EQ(visitor.paths[3].segmentCount, 2);
|
||||
}
|
||||
|
||||
@@ -262,7 +262,8 @@ std::string decodedStream(QPDFObjectHandle contents) {
|
||||
}
|
||||
|
||||
std::vector<std::unique_ptr<ContentObject>> buildPageObjects(QPDFObjectHandle page) {
|
||||
Lexer lexer(decodedStream(page.getKey("/Contents")));
|
||||
std::string decodedContent = decodedStream(page.getKey("/Contents"));
|
||||
Lexer lexer(decodedContent);
|
||||
auto tokens = lexer.tokenize();
|
||||
ContentParser parser(tokens);
|
||||
auto operations = parser.parse();
|
||||
@@ -436,6 +437,8 @@ TEST(ImageXObjectVerification, JpegLogoAndPhotoDecodeAndMatchPdfiumRender) {
|
||||
|
||||
QPDF qpdf;
|
||||
auto objects = buildFirstPageObjects(qpdf, pdf);
|
||||
|
||||
|
||||
auto images = imageObjects(objects);
|
||||
ASSERT_EQ(images.size(), 2u);
|
||||
|
||||
|
||||
@@ -435,7 +435,14 @@ class GatewayService {
|
||||
}
|
||||
}
|
||||
|
||||
private renderCache = new Map<string, string>();
|
||||
|
||||
async renderPage(params: RenderParams): Promise<string> {
|
||||
const cacheKey = `${params.documentId}_${params.pageIndex}_${params.zoom}_${params.rotation}`;
|
||||
if (this.renderCache.has(cacheKey)) {
|
||||
return this.renderCache.get(cacheKey)!;
|
||||
}
|
||||
|
||||
try {
|
||||
const query = new URLSearchParams({
|
||||
page: params.pageIndex.toString(),
|
||||
@@ -452,7 +459,19 @@ class GatewayService {
|
||||
|
||||
if (!response.ok) throw new Error(`Page render failed: ${response.statusText}`);
|
||||
const blob = await response.blob();
|
||||
return URL.createObjectURL(blob);
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
|
||||
this.renderCache.set(cacheKey, objectUrl);
|
||||
if (this.renderCache.size > 100) {
|
||||
const firstKey = this.renderCache.keys().next().value;
|
||||
if (firstKey) {
|
||||
const oldUrl = this.renderCache.get(firstKey)!;
|
||||
if (oldUrl.startsWith('blob:')) URL.revokeObjectURL(oldUrl);
|
||||
this.renderCache.delete(firstKey);
|
||||
}
|
||||
}
|
||||
|
||||
return objectUrl;
|
||||
} catch (err) {
|
||||
return this.generateMockPage(params.pageIndex);
|
||||
}
|
||||
|
||||
+92
-11
@@ -1,3 +1,84 @@
|
||||
# Stage 1: Builder
|
||||
FROM python:3.11-slim AS builder
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
VCPKG_FORCE_SYSTEM_BINARIES=1
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
cmake \
|
||||
ninja-build \
|
||||
pkg-config \
|
||||
git \
|
||||
fonts-liberation \
|
||||
libjpeg-dev \
|
||||
zlib1g-dev \
|
||||
libpng-dev \
|
||||
curl \
|
||||
zip \
|
||||
unzip \
|
||||
tar \
|
||||
autoconf \
|
||||
autoconf-archive \
|
||||
automake \
|
||||
libtool \
|
||||
lld \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ln -sf /usr/bin/ld.lld /usr/bin/ld
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Install vcpkg
|
||||
RUN git clone https://github.com/microsoft/vcpkg.git /opt/vcpkg \
|
||||
&& /opt/vcpkg/bootstrap-vcpkg.sh -disableMetrics
|
||||
ENV VCPKG_ROOT=/opt/vcpkg
|
||||
|
||||
# Cache vcpkg dependencies in a separate layer
|
||||
COPY vcpkg.json ./
|
||||
RUN --mount=type=cache,target=/root/.cache \
|
||||
--mount=type=cache,target=/opt/vcpkg/downloads \
|
||||
/opt/vcpkg/vcpkg install --triplet x64-linux
|
||||
|
||||
# Install depot_tools globally with caching
|
||||
RUN --mount=type=cache,target=/opt/depot_tools \
|
||||
if [ ! -d /opt/depot_tools/.git ]; then \
|
||||
git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git /opt/depot_tools; \
|
||||
fi
|
||||
ENV PATH="/opt/depot_tools:${PATH}"
|
||||
|
||||
# Build PDFium for Linux (heavily cached, keeping the huge source tree out of the image layer)
|
||||
COPY third_party/pdfium/ ./third_party/pdfium/
|
||||
RUN --mount=type=cache,target=/build/third_party/pdfium/checkout \
|
||||
./third_party/pdfium/build_pdfium.sh
|
||||
|
||||
# Copy everything needed for the engine and bindings build
|
||||
COPY CMakeLists.txt CMakePresets.json ./
|
||||
COPY cmake/ ./cmake/
|
||||
COPY engine/ ./engine/
|
||||
COPY bindings/ ./bindings/
|
||||
COPY gateway/ ./gateway/
|
||||
COPY corpus/ ./corpus/
|
||||
|
||||
# Configure CMake with tests enabled
|
||||
RUN --mount=type=cache,target=/root/.cache \
|
||||
--mount=type=cache,target=/opt/vcpkg/downloads \
|
||||
cmake --preset linux-release \
|
||||
-DPDFENGINE_BUILD_TESTS=ON \
|
||||
-DPDFENGINE_WITH_PDFIUM=ON \
|
||||
-DPDFENGINE_WITH_SKIA=OFF \
|
||||
-DPDFENGINE_WITH_QPDF=ON \
|
||||
-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake
|
||||
|
||||
# Build all targets including tests and pdfengine_py
|
||||
RUN cmake --build out/build/linux-release
|
||||
|
||||
# Stage 2: Tester
|
||||
FROM builder AS tester
|
||||
RUN ctest --test-dir out/build/linux-release --output-on-failure
|
||||
|
||||
# Stage 3: Runtime
|
||||
FROM python:3.11-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
@@ -8,27 +89,27 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
cmake \
|
||||
ninja-build \
|
||||
pkg-config \
|
||||
git \
|
||||
libjpeg-dev \
|
||||
zlib1g-dev \
|
||||
libpng-dev \
|
||||
libjpeg62-turbo \
|
||||
zlib1g \
|
||||
libpng16-16 \
|
||||
fonts-liberation \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install the extension globally so it's not shadowed by the volume mount ./gateway:/home/app
|
||||
COPY --from=tester /build/gateway/pdfengine*.so /usr/local/lib/python3.11/site-packages/
|
||||
|
||||
RUN groupadd --system app \
|
||||
&& useradd --system --gid app --create-home --home-dir /home/app app
|
||||
|
||||
WORKDIR /home/app
|
||||
|
||||
COPY pyproject.toml README.md ./
|
||||
# Copy gateway Python code
|
||||
COPY gateway/pyproject.toml gateway/README.md ./
|
||||
RUN python -m pip install --upgrade pip \
|
||||
&& pip install --no-cache-dir -e ".[dev]"
|
||||
|
||||
COPY app ./app
|
||||
COPY tests ./tests
|
||||
COPY gateway/app ./app
|
||||
COPY gateway/tests ./tests
|
||||
|
||||
RUN chown -R app:app /home/app
|
||||
USER app
|
||||
|
||||
@@ -26,6 +26,17 @@ class Settings(BaseSettings):
|
||||
"Phase 0 default is False — routes that need the engine return 501."
|
||||
),
|
||||
)
|
||||
|
||||
render_cache_enabled: bool = Field(default=True)
|
||||
render_cache_max_entries: int = Field(default=256)
|
||||
render_cache_max_bytes: int = Field(
|
||||
default=536_870_912,
|
||||
description="Max total bytes for the tile render cache. Set to 0 to disable byte cap."
|
||||
)
|
||||
render_cache_max_entry_bytes: int = Field(
|
||||
default=8_388_608,
|
||||
description="Max bytes for a single cache entry. Oversized tiles are rendered but not cached."
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
|
||||
+2
-1
@@ -4,7 +4,7 @@ from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app import __version__
|
||||
from app.routers import documents, edits, health, info, render
|
||||
from app.routers import documents, edits, health, info, internal, render
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
@@ -29,6 +29,7 @@ def create_app() -> FastAPI:
|
||||
app.include_router(render.compat_router)
|
||||
app.include_router(edits.router)
|
||||
app.include_router(edits.compat_router)
|
||||
app.include_router(internal.router)
|
||||
|
||||
@app.get("/")
|
||||
def read_root():
|
||||
|
||||
@@ -426,7 +426,7 @@ def apply_edits_impl(document_id: str, request: EditsRequest):
|
||||
|
||||
edits_json = json.dumps(req_dict)
|
||||
doc_copy = pdfengine.PdfDocument.load_from_memory(doc_info["bytes_data"])
|
||||
doc_copy.apply_edits(edits_json)
|
||||
invalidated_regions = doc_copy.apply_edits(edits_json)
|
||||
|
||||
full_save_types = {"redaction", "replace_text", "reflow_paragraph"}
|
||||
needs_full = any(op.get("type") in full_save_types for op in req_dict.get("operations", []))
|
||||
@@ -438,8 +438,11 @@ def apply_edits_impl(document_id: str, request: EditsRequest):
|
||||
filename=doc_info["filename"], bytes_data=new_bytes, doc_instance=new_doc,
|
||||
permissions=doc_info.get("permissions"),
|
||||
)
|
||||
|
||||
from app.services.render_cache import tile_cache
|
||||
tile_cache.invalidate_doc(doc_info.get("doc_hash", ""))
|
||||
|
||||
return {"success": True, "newDocumentId": new_info["id"]}
|
||||
return {"success": True, "newDocumentId": new_info["id"], "invalidatedRegions": invalidated_regions}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) from e
|
||||
finally:
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.services.render_cache import tile_cache
|
||||
|
||||
router = APIRouter(prefix="/internal", tags=["internal"])
|
||||
|
||||
@router.get("/cache/stats")
|
||||
def cache_stats():
|
||||
stats = tile_cache.stats()
|
||||
return {
|
||||
"hits": stats.hits,
|
||||
"misses": stats.misses,
|
||||
"evictions": stats.evictions,
|
||||
"current_entries": stats.current_entries,
|
||||
"current_bytes": stats.current_bytes,
|
||||
"max_bytes": stats.max_bytes,
|
||||
"hit_rate": stats.hit_rate,
|
||||
"avg_lookup_ns": stats.avg_lookup_ns,
|
||||
"avg_render_time_ns": stats.avg_render_time_ns,
|
||||
}
|
||||
@@ -1,19 +1,30 @@
|
||||
import hashlib
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Path, Query, Response, status
|
||||
from fastapi import APIRouter, HTTPException, Path, Query, Request, Response, status
|
||||
|
||||
from app.schemas.font import FontInfoResponse
|
||||
from app.services import engine
|
||||
from app.services.font import font_info_to_response
|
||||
from app.services.render_cache import RENDERER_VERSION, RenderMode, TileCacheKey, tile_cache
|
||||
from app.services.store import document_store
|
||||
|
||||
router = APIRouter(prefix="/documents/{document_id}/pages", tags=["render"])
|
||||
compat_router = APIRouter(tags=["render"])
|
||||
|
||||
def _etag(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()[:16]
|
||||
|
||||
|
||||
@router.get("/{page_index}/render")
|
||||
def render_page(
|
||||
document_id: str, page_index: Annotated[int, Path(ge=0)], dpi: int = 96
|
||||
request: Request,
|
||||
document_id: str,
|
||||
page_index: Annotated[int, Path(ge=0)],
|
||||
dpi: int = 96,
|
||||
zoom: float = 1.0,
|
||||
rotation: int = 0,
|
||||
render_mode: RenderMode = RenderMode.NORMAL
|
||||
) -> Response:
|
||||
if not engine.is_available():
|
||||
raise HTTPException(
|
||||
@@ -28,8 +39,127 @@ def render_page(
|
||||
try:
|
||||
doc = doc_info["doc_instance"]
|
||||
page = doc.get_page(page_index)
|
||||
|
||||
key = TileCacheKey(
|
||||
doc_hash=doc_info.get("doc_hash", ""),
|
||||
renderer_version=RENDERER_VERSION,
|
||||
page=page_index,
|
||||
dpi=dpi,
|
||||
zoom=zoom,
|
||||
rotation=rotation,
|
||||
render_mode=render_mode,
|
||||
tile_x=0.0,
|
||||
tile_y=0.0,
|
||||
tile_w=round(page.width, 2),
|
||||
tile_h=round(page.height, 2)
|
||||
)
|
||||
|
||||
cached = tile_cache.get(key)
|
||||
if cached:
|
||||
etag_val = f'"{_etag(cached)}"'
|
||||
if request.headers.get("if-none-match") == etag_val:
|
||||
return Response(status_code=304)
|
||||
return Response(
|
||||
content=cached,
|
||||
media_type="image/png",
|
||||
headers={"ETag": etag_val, "X-Cache": "HIT", "Cache-Control": "private, max-age=300"}
|
||||
)
|
||||
|
||||
import time
|
||||
start = time.perf_counter_ns()
|
||||
img = page.render(dpi)
|
||||
return Response(content=img.data, media_type="image/png")
|
||||
elapsed = time.perf_counter_ns() - start
|
||||
|
||||
tile_cache.put(key, img.data)
|
||||
tile_cache.record_render_time(elapsed)
|
||||
|
||||
etag_val = f'"{_etag(img.data)}"'
|
||||
return Response(
|
||||
content=img.data,
|
||||
media_type="image/png",
|
||||
headers={"ETag": etag_val, "X-Cache": "MISS", "Cache-Control": "private, max-age=300"}
|
||||
)
|
||||
except IndexError:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="Page index out of bounds"
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/{page_index}/render-tile")
|
||||
def render_page_tile(
|
||||
request: Request,
|
||||
document_id: str,
|
||||
page_index: Annotated[int, Path(ge=0)],
|
||||
x: float,
|
||||
y: float,
|
||||
width: float,
|
||||
height: float,
|
||||
dpi: int = 96,
|
||||
zoom: float = 1.0,
|
||||
rotation: int = 0,
|
||||
render_mode: RenderMode = RenderMode.NORMAL
|
||||
) -> Response:
|
||||
if not engine.is_available():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_501_NOT_IMPLEMENTED,
|
||||
detail="Engine bridge (bindings/python) not yet available.",
|
||||
)
|
||||
|
||||
doc_info = document_store.get_document(document_id)
|
||||
if not doc_info:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found")
|
||||
|
||||
try:
|
||||
doc = doc_info["doc_instance"]
|
||||
page = doc.get_page(page_index)
|
||||
|
||||
key = TileCacheKey(
|
||||
doc_hash=doc_info.get("doc_hash", ""),
|
||||
renderer_version=RENDERER_VERSION,
|
||||
page=page_index,
|
||||
dpi=dpi,
|
||||
zoom=zoom,
|
||||
rotation=rotation,
|
||||
render_mode=render_mode,
|
||||
tile_x=round(x, 2),
|
||||
tile_y=round(y, 2),
|
||||
tile_w=round(width, 2),
|
||||
tile_h=round(height, 2)
|
||||
)
|
||||
|
||||
cached = tile_cache.get(key)
|
||||
if cached:
|
||||
etag_val = f'"{_etag(cached)}"'
|
||||
if request.headers.get("if-none-match") == etag_val:
|
||||
return Response(status_code=304)
|
||||
return Response(
|
||||
content=cached,
|
||||
media_type="image/png",
|
||||
headers={"ETag": etag_val, "X-Cache": "HIT", "Cache-Control": "private, max-age=300"}
|
||||
)
|
||||
|
||||
import time
|
||||
start = time.perf_counter_ns()
|
||||
img_width, img_height, img_data = page.render_tile(dpi, x, y, width, height)
|
||||
import io
|
||||
from PIL import Image
|
||||
img = Image.frombytes("RGBA", (img_width, img_height), img_data)
|
||||
out_buf = io.BytesIO()
|
||||
img.save(out_buf, format="PNG")
|
||||
png_bytes = out_buf.getvalue()
|
||||
elapsed = time.perf_counter_ns() - start
|
||||
|
||||
tile_cache.put(key, png_bytes)
|
||||
tile_cache.record_render_time(elapsed)
|
||||
|
||||
etag_val = f'"{_etag(png_bytes)}"'
|
||||
return Response(
|
||||
content=png_bytes,
|
||||
media_type="image/png",
|
||||
headers={"ETag": etag_val, "X-Cache": "MISS", "Cache-Control": "private, max-age=300"}
|
||||
)
|
||||
except IndexError:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND, detail="Page index out of bounds"
|
||||
@@ -72,10 +202,10 @@ def extract_page_text(document_id: str, page_index: Annotated[int, Path(ge=0)]):
|
||||
|
||||
@compat_router.get("/render/{document_id}")
|
||||
def render_page_compat(
|
||||
document_id: str, page: int = 0, zoom: float = 1.0, rotation: int = 0
|
||||
request: Request, document_id: str, page: int = 0, zoom: float = 1.0, rotation: int = 0
|
||||
) -> Response:
|
||||
dpi = int(96 * zoom)
|
||||
return render_page(document_id, page, dpi)
|
||||
return render_page(request, document_id, page, dpi, zoom, rotation, RenderMode.NORMAL)
|
||||
|
||||
|
||||
@router.get("/{page_index}")
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
RENDERER_VERSION: int = 1
|
||||
|
||||
|
||||
class RenderMode(str, Enum):
|
||||
# Current
|
||||
NORMAL = "normal"
|
||||
PRINT = "print"
|
||||
GRAYSCALE = "grayscale"
|
||||
|
||||
# Future (reserved)
|
||||
ANNOTATION_ONLY = "annotation_only"
|
||||
EDIT_PREVIEW = "edit_preview"
|
||||
SELECTION_OVERLAY = "selection_overlay"
|
||||
PROOF = "proof"
|
||||
HIGH_QUALITY = "high_quality"
|
||||
DRAFT = "draft"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TileCacheKey:
|
||||
doc_hash: str
|
||||
renderer_version: int
|
||||
page: int
|
||||
dpi: int
|
||||
zoom: float
|
||||
rotation: int
|
||||
render_mode: str
|
||||
tile_x: float
|
||||
tile_y: float
|
||||
tile_w: float
|
||||
tile_h: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class CacheStats:
|
||||
hits: int = 0
|
||||
misses: int = 0
|
||||
evictions: int = 0
|
||||
current_entries: int = 0
|
||||
current_bytes: int = 0
|
||||
max_bytes: int = 0
|
||||
hit_rate: float = 0.0
|
||||
avg_lookup_ns: float = 0.0
|
||||
avg_render_time_ns: float = 0.0
|
||||
|
||||
|
||||
class _Node:
|
||||
__slots__ = ["key", "value", "size", "prev", "next"]
|
||||
|
||||
def __init__(self, key: TileCacheKey | None, value: bytes | None, size: int):
|
||||
self.key = key
|
||||
self.value = value
|
||||
self.size = size
|
||||
self.prev: _Node | None = None
|
||||
self.next: _Node | None = None
|
||||
|
||||
|
||||
class TileCache:
|
||||
def __init__(self):
|
||||
settings = get_settings()
|
||||
self.enabled = settings.render_cache_enabled
|
||||
self.max_entries = settings.render_cache_max_entries
|
||||
self.max_bytes = settings.render_cache_max_bytes
|
||||
self.max_entry_bytes = settings.render_cache_max_entry_bytes
|
||||
|
||||
self._lock = threading.Lock()
|
||||
self._cache: dict[TileCacheKey, _Node] = {}
|
||||
self._head = _Node(None, None, 0)
|
||||
self._tail = _Node(None, None, 0)
|
||||
self._head.next = self._tail
|
||||
self._tail.prev = self._head
|
||||
|
||||
self._current_bytes = 0
|
||||
|
||||
# Stats
|
||||
self._hits = 0
|
||||
self._misses = 0
|
||||
self._evictions = 0
|
||||
self._total_lookup_ns = 0
|
||||
self._total_render_time_ns = 0
|
||||
self._render_time_count = 0
|
||||
|
||||
def _remove(self, node: _Node):
|
||||
p = node.prev
|
||||
n = node.next
|
||||
if p and n:
|
||||
p.next = n
|
||||
n.prev = p
|
||||
|
||||
def _add_to_front(self, node: _Node):
|
||||
first = self._head.next
|
||||
if first:
|
||||
self._head.next = node
|
||||
node.prev = self._head
|
||||
node.next = first
|
||||
first.prev = node
|
||||
|
||||
def _evict(self):
|
||||
last = self._tail.prev
|
||||
if last and last != self._head:
|
||||
self._remove(last)
|
||||
if last.key:
|
||||
del self._cache[last.key]
|
||||
self._current_bytes -= last.size
|
||||
self._evictions += 1
|
||||
|
||||
def get(self, key: TileCacheKey) -> bytes | None:
|
||||
if not self.enabled:
|
||||
return None
|
||||
|
||||
start_time = time.perf_counter_ns()
|
||||
with self._lock:
|
||||
node = self._cache.get(key)
|
||||
if node:
|
||||
self._hits += 1
|
||||
self._remove(node)
|
||||
self._add_to_front(node)
|
||||
res = node.value
|
||||
else:
|
||||
self._misses += 1
|
||||
res = None
|
||||
|
||||
lookup_time = time.perf_counter_ns() - start_time
|
||||
self._total_lookup_ns += lookup_time
|
||||
|
||||
return res
|
||||
|
||||
def put(self, key: TileCacheKey, data: bytes) -> None:
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
size = len(data)
|
||||
if size > self.max_entry_bytes:
|
||||
# Too large to cache
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
if key in self._cache:
|
||||
node = self._cache[key]
|
||||
self._current_bytes -= node.size
|
||||
node.value = data
|
||||
node.size = size
|
||||
self._current_bytes += size
|
||||
self._remove(node)
|
||||
self._add_to_front(node)
|
||||
else:
|
||||
new_node = _Node(key, data, size)
|
||||
self._cache[key] = new_node
|
||||
self._add_to_front(new_node)
|
||||
self._current_bytes += size
|
||||
|
||||
# Evict if over limits
|
||||
while len(self._cache) > self.max_entries or (self.max_bytes > 0 and self._current_bytes > self.max_bytes):
|
||||
self._evict()
|
||||
|
||||
def invalidate_doc(self, doc_hash: str) -> None:
|
||||
with self._lock:
|
||||
keys_to_remove = [k for k in self._cache.keys() if k.doc_hash == doc_hash]
|
||||
for k in keys_to_remove:
|
||||
node = self._cache[k]
|
||||
self._remove(node)
|
||||
self._current_bytes -= node.size
|
||||
del self._cache[k]
|
||||
|
||||
def invalidate_renderer_version(self, old_version: int) -> None:
|
||||
with self._lock:
|
||||
keys_to_remove = [k for k in self._cache.keys() if k.renderer_version == old_version]
|
||||
for k in keys_to_remove:
|
||||
node = self._cache[k]
|
||||
self._remove(node)
|
||||
self._current_bytes -= node.size
|
||||
del self._cache[k]
|
||||
|
||||
def clear(self) -> None:
|
||||
with self._lock:
|
||||
self._cache.clear()
|
||||
self._head.next = self._tail
|
||||
self._tail.prev = self._head
|
||||
self._current_bytes = 0
|
||||
|
||||
def stats(self) -> CacheStats:
|
||||
with self._lock:
|
||||
hit_rate = 0.0
|
||||
total_reqs = self._hits + self._misses
|
||||
if total_reqs > 0:
|
||||
hit_rate = self._hits / total_reqs
|
||||
|
||||
avg_lookup = 0.0
|
||||
if total_reqs > 0:
|
||||
avg_lookup = self._total_lookup_ns / total_reqs
|
||||
|
||||
avg_render = 0.0
|
||||
if self._render_time_count > 0:
|
||||
avg_render = self._total_render_time_ns / self._render_time_count
|
||||
|
||||
return CacheStats(
|
||||
hits=self._hits,
|
||||
misses=self._misses,
|
||||
evictions=self._evictions,
|
||||
current_entries=len(self._cache),
|
||||
current_bytes=self._current_bytes,
|
||||
max_bytes=self.max_bytes,
|
||||
hit_rate=hit_rate,
|
||||
avg_lookup_ns=avg_lookup,
|
||||
avg_render_time_ns=avg_render,
|
||||
)
|
||||
|
||||
def record_render_time(self, elapsed_ns: int):
|
||||
with self._lock:
|
||||
self._total_render_time_ns += elapsed_ns
|
||||
self._render_time_count += 1
|
||||
|
||||
|
||||
tile_cache = TileCache()
|
||||
@@ -1,3 +1,4 @@
|
||||
import hashlib
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
@@ -57,6 +58,7 @@ class DocumentStore:
|
||||
info = {
|
||||
"id": doc_id,
|
||||
"filename": filename,
|
||||
"doc_hash": hashlib.sha256(bytes_data).hexdigest(),
|
||||
"sizeBytes": len(bytes_data),
|
||||
"totalPages": doc_instance.page_count,
|
||||
"pageWidth": page_width,
|
||||
|
||||
Vendored
+12
-8
@@ -35,8 +35,8 @@ CHECKOUT_DIR="${BUILD_ROOT}/checkout"
|
||||
echo ">> Build root: ${BUILD_ROOT}"
|
||||
|
||||
# --- 1. Read and validate the pinned revision -------------------------------
|
||||
PDFIUM_REPO="$(grep -E '^PDFIUM_REPO=' "${PINNED_FILE}" | cut -d= -f2-)"
|
||||
PDFIUM_COMMIT="$(grep -E '^PDFIUM_COMMIT=' "${PINNED_FILE}" | cut -d= -f2-)"
|
||||
PDFIUM_REPO="$(grep -E '^PDFIUM_REPO=' "${PINNED_FILE}" | cut -d= -f2- | tr -d '\r')"
|
||||
PDFIUM_COMMIT="$(grep -E '^PDFIUM_COMMIT=' "${PINNED_FILE}" | cut -d= -f2- | tr -d '\r')"
|
||||
if [[ -z "${PDFIUM_COMMIT}" || "${PDFIUM_COMMIT}" == "REPLACE_WITH_PINNED_COMMIT_SHA" ]]; then
|
||||
echo "ERROR: PDFium revision is not pinned. Edit pdfium.pinned first (see README.md)." >&2
|
||||
exit 1
|
||||
@@ -45,12 +45,14 @@ echo ">> PDFium pinned at ${PDFIUM_COMMIT}"
|
||||
|
||||
# --- 2. depot_tools ---------------------------------------------------------
|
||||
mkdir -p "${BUILD_ROOT}"
|
||||
if [[ ! -d "${DEPOT_TOOLS_DIR}" ]]; then
|
||||
echo ">> Cloning depot_tools"
|
||||
git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git \
|
||||
"${DEPOT_TOOLS_DIR}"
|
||||
if ! command -v gclient &> /dev/null; then
|
||||
if [[ ! -d "${DEPOT_TOOLS_DIR}/.git" ]]; then
|
||||
echo ">> Cloning depot_tools"
|
||||
git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git \
|
||||
"${DEPOT_TOOLS_DIR}"
|
||||
fi
|
||||
export PATH="${DEPOT_TOOLS_DIR}:${PATH}"
|
||||
fi
|
||||
export PATH="${DEPOT_TOOLS_DIR}:${PATH}"
|
||||
# Do NOT set DEPOT_TOOLS_UPDATE=0 — depot_tools is designed to self-manage, and
|
||||
# on first use it must bootstrap. Reproducibility comes from the pinned PDFium
|
||||
# revision below, not from freezing depot_tools.
|
||||
@@ -58,9 +60,11 @@ export PATH="${DEPOT_TOOLS_DIR}:${PATH}"
|
||||
# Git settings injected per-process via GIT_CONFIG_* so the user's global git
|
||||
# config is never touched. core.autocrlf=false avoids gclient seeing dependency
|
||||
# checkouts as "uncommitted changes" on platforms where autocrlf is enabled.
|
||||
export GIT_CONFIG_COUNT=2
|
||||
export GIT_CONFIG_COUNT=4
|
||||
export GIT_CONFIG_KEY_0=core.autocrlf GIT_CONFIG_VALUE_0=false
|
||||
export GIT_CONFIG_KEY_1=core.filemode GIT_CONFIG_VALUE_1=false
|
||||
export GIT_CONFIG_KEY_2=http.postBuffer GIT_CONFIG_VALUE_2=1048576000
|
||||
export GIT_CONFIG_KEY_3=core.compression GIT_CONFIG_VALUE_3=0
|
||||
|
||||
# --- 3. Fetch / sync the PDFium tree ----------------------------------------
|
||||
mkdir -p "${CHECKOUT_DIR}"
|
||||
|
||||
Reference in New Issue
Block a user