From 7cd400e62a8c9a0b8144354a59a505a5981bb06f Mon Sep 17 00:00:00 2001 From: saqib mir Date: Tue, 16 Jun 2026 19:06:48 +0530 Subject: [PATCH] content stream --- CMakeLists.txt | 5 + bindings/CMakeLists.txt | 8 + bindings/python/pdfengine_py.cpp | 95 ++++++ engine/CMakeLists.txt | 22 ++ engine/include/pdfengine/ast.hpp | 44 +++ engine/include/pdfengine/content_object.hpp | 35 +++ engine/include/pdfengine/content_stream.hpp | 35 +++ engine/include/pdfengine/token.hpp | 35 +++ engine/src/parser/content_builder.cpp | 122 ++++++++ engine/src/parser/content_builder.hpp | 36 +++ engine/src/parser/lexer.cpp | 298 +++++++++++++++++++ engine/src/parser/lexer.hpp | 42 +++ engine/src/parser/parser.cpp | 170 +++++++++++ engine/src/parser/parser.hpp | 35 +++ engine/src/qpdf/qpdf_extractor.cpp | 194 ++++++++++++ engine/src/qpdf/qpdf_extractor.hpp | 38 +++ engine/src/qpdf/qpdf_writer.cpp | 55 ++++ engine/src/qpdf/qpdf_writer.hpp | 24 ++ engine/src/serializer/ast_serializer.cpp | 93 ++++++ engine/src/serializer/ast_serializer.hpp | 20 ++ engine/src/serializer/content_serializer.cpp | 63 ++++ engine/src/serializer/content_serializer.hpp | 20 ++ engine/tests/CMakeLists.txt | 14 + engine/tests/ast_serializer_test.cpp | 63 ++++ engine/tests/content_builder_test.cpp | 119 ++++++++ engine/tests/content_serializer_test.cpp | 47 +++ engine/tests/lexer_test.cpp | 133 +++++++++ engine/tests/parser_test.cpp | 117 ++++++++ engine/tests/qpdf_extractor_test.cpp | 94 ++++++ engine/tests/qpdf_writer_test.cpp | 88 ++++++ gateway/app/routers/documents.py | 94 ++++++ test_bindings.py | 66 ++++ vcpkg.json | 1 + 33 files changed, 2325 insertions(+) create mode 100644 engine/include/pdfengine/ast.hpp create mode 100644 engine/include/pdfengine/content_object.hpp create mode 100644 engine/include/pdfengine/content_stream.hpp create mode 100644 engine/include/pdfengine/token.hpp create mode 100644 engine/src/parser/content_builder.cpp create mode 100644 engine/src/parser/content_builder.hpp create mode 100644 engine/src/parser/lexer.cpp create mode 100644 engine/src/parser/lexer.hpp create mode 100644 engine/src/parser/parser.cpp create mode 100644 engine/src/parser/parser.hpp create mode 100644 engine/src/qpdf/qpdf_extractor.cpp create mode 100644 engine/src/qpdf/qpdf_extractor.hpp create mode 100644 engine/src/qpdf/qpdf_writer.cpp create mode 100644 engine/src/qpdf/qpdf_writer.hpp create mode 100644 engine/src/serializer/ast_serializer.cpp create mode 100644 engine/src/serializer/ast_serializer.hpp create mode 100644 engine/src/serializer/content_serializer.cpp create mode 100644 engine/src/serializer/content_serializer.hpp create mode 100644 engine/tests/ast_serializer_test.cpp create mode 100644 engine/tests/content_builder_test.cpp create mode 100644 engine/tests/content_serializer_test.cpp create mode 100644 engine/tests/lexer_test.cpp create mode 100644 engine/tests/parser_test.cpp create mode 100644 engine/tests/qpdf_extractor_test.cpp create mode 100644 engine/tests/qpdf_writer_test.cpp create mode 100644 test_bindings.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c82a32..abff300 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ option(PDFENGINE_ENABLE_SANITIZERS "Build with AddressSanitizer/UBSan" option(PDFENGINE_WARNINGS_AS_ERRORS "Treat compiler warnings as errors" OFF) option(PDFENGINE_WITH_PDFIUM "Link the PDFium static lib (build it first)" OFF) option(PDFENGINE_WITH_SKIA "Link the Skia static lib (build it first)" OFF) +option(PDFENGINE_WITH_QPDF "Link QPDF for content stream extraction" OFF) option(PDFENGINE_FUZZING "Build libFuzzer harnesses (requires Clang)" OFF) if(PDFENGINE_FUZZING) @@ -72,6 +73,10 @@ find_package(freetype CONFIG REQUIRED) find_package(harfbuzz CONFIG REQUIRED) find_package(spdlog CONFIG REQUIRED) find_package(nlohmann_json CONFIG REQUIRED) +find_package(qpdf CONFIG REQUIRED) +if(WIN32 AND DEFINED VCPKG_TARGET_TRIPLET) + link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/lib") +endif() if(PDFENGINE_BUILD_TESTS) find_package(GTest CONFIG REQUIRED) diff --git a/bindings/CMakeLists.txt b/bindings/CMakeLists.txt index 7302c5c..d95d266 100644 --- a/bindings/CMakeLists.txt +++ b/bindings/CMakeLists.txt @@ -15,6 +15,14 @@ set_target_properties(pdfengine_py PROPERTIES target_link_libraries(pdfengine_py PRIVATE pdfengine::pdfengine) +target_include_directories(pdfengine_py PRIVATE + "${CMAKE_SOURCE_DIR}/engine/src" +) + +if(MSVC) + target_link_options(pdfengine_py PRIVATE "/FORCE:MULTIPLE") +endif() + # Set warnings and sanitizers for the bindings module pdfengine_set_warnings(pdfengine_py) pdfengine_enable_sanitizers(pdfengine_py) diff --git a/bindings/python/pdfengine_py.cpp b/bindings/python/pdfengine_py.cpp index 72e20da..d84ec27 100644 --- a/bindings/python/pdfengine_py.cpp +++ b/bindings/python/pdfengine_py.cpp @@ -45,9 +45,104 @@ void get_or_throw(std::expected&& res) { } +#include +#include +#include +#include +#include +#include +#include +#include + +class StreamEditor { +public: + StreamEditor(const std::string& filepath) : filepath_(filepath) {} + + py::list extract_text_objects(int page_index) { + pdfengine::qpdf_layer::QpdfExtractor extractor; + auto stream = extractor.extractPageStream(filepath_, page_index); + if (!stream.has_value()) { + throw std::runtime_error("Failed to extract page stream"); + } + + pdfengine::Lexer lexer(stream->decodedContent); + auto tokens = lexer.tokenize(); + pdfengine::ContentParser parser(tokens); + pdfengine::ContentBuilder builder; + + auto objects = builder.build(parser.parse()); + + py::list result; + for (const auto& obj : objects) { + if (obj->getType() == pdfengine::ContentObjectType::Text) { + auto* textObj = static_cast(obj.get()); + py::dict d; + d["text"] = textObj->text; + d["fontName"] = textObj->fontName; + d["fontSize"] = textObj->fontSize; + + py::list tm; + for (int i = 0; i < 6; ++i) { + tm.append(textObj->tm[i]); + } + d["tm"] = tm; + result.append(d); + } + } + return result; + } + + bool replace_text_object(int page_index, int object_index, const std::string& new_text, const std::string& dest_path) { + pdfengine::qpdf_layer::QpdfExtractor extractor; + auto stream = extractor.extractPageStream(filepath_, page_index); + if (!stream.has_value()) return false; + + pdfengine::Lexer lexer(stream->decodedContent); + auto tokens = lexer.tokenize(); + pdfengine::ContentParser parser(tokens); + pdfengine::ContentBuilder builder; + + auto objects = builder.build(parser.parse()); + + int textCount = 0; + bool modified = false; + for (auto& obj : objects) { + if (obj->getType() == pdfengine::ContentObjectType::Text) { + if (textCount == object_index) { + auto* textObj = static_cast(obj.get()); + textObj->text = new_text; + modified = true; + break; + } + textCount++; + } + } + + if (!modified) return false; + + pdfengine::ContentSerializer cSerializer; + auto newOps = cSerializer.serialize(objects); + + pdfengine::AstSerializer astSerializer; + std::string newRawStream = astSerializer.serialize(newOps); + + pdfengine::qpdf_layer::QpdfWriter writer; + auto res = writer.replacePageStreamAndSave(filepath_, dest_path, page_index, newRawStream); + return res.has_value(); + } + +private: + std::string filepath_; +}; + PYBIND11_MODULE(pdfengine, m) { m.doc() = "Python bindings for the PdfEngine C++ Core SDK"; + py::class_(m, "StreamEditor") + .def(py::init(), py::arg("filepath")) + .def("extract_text_objects", &StreamEditor::extract_text_objects, py::arg("page_index")) + .def("replace_text_object", &StreamEditor::replace_text_object, py::arg("page_index"), py::arg("object_index"), py::arg("new_text"), py::arg("dest_path")); + m.def("engine_version", &pdfengine::engineVersion, "Get the engine version string"); m.def("engine_build_info", &pdfengine::engineBuildInfo, "Get the engine build info string"); m.def("engine_has_pdfium", &pdfengine::engineHasPdfium, "Check if the engine was built with PDFium support"); diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index bc9ce4c..f47b178 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -68,6 +68,28 @@ if(PDFENGINE_WITH_SKIA) target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_SKIA) endif() +if(PDFENGINE_WITH_QPDF) + find_package(ZLIB REQUIRED) + find_package(JPEG REQUIRED) + if(NOT TARGET zs) + add_library(zs ALIAS ZLIB::ZLIB) + endif() + if(NOT TARGET jpeg) + add_library(jpeg ALIAS JPEG::JPEG) + endif() + target_sources(pdfengine PRIVATE + src/qpdf/qpdf_extractor.cpp + src/qpdf/qpdf_writer.cpp + src/parser/lexer.cpp + src/parser/parser.cpp + src/parser/content_builder.cpp + src/serializer/content_serializer.cpp + src/serializer/ast_serializer.cpp + ) + target_link_libraries(pdfengine PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG) + target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_QPDF) +endif() + pdfengine_set_warnings(pdfengine) pdfengine_enable_sanitizers(pdfengine) diff --git a/engine/include/pdfengine/ast.hpp b/engine/include/pdfengine/ast.hpp new file mode 100644 index 0000000..00076ed --- /dev/null +++ b/engine/include/pdfengine/ast.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace pdfengine { + +enum class AstNodeType { + Number, + Name, + String, + HexString, + Boolean, + Null, + Array, + Dictionary +}; + +class AstNode { +public: + AstNodeType type; + std::string stringValue; + std::vector bytesValue; + double numberValue = 0.0; + bool boolValue = false; + + // We use a vector of shared_ptr for recursive data structures so the node is easily copyable/movable + std::vector> arrayItems; + std::unordered_map> dictItems; + + // Constructors for convenience + AstNode() = default; + explicit AstNode(AstNodeType t) : type(t) {} +}; + +struct Operation { + std::string op; + std::vector> operands; +}; + +} // namespace pdfengine diff --git a/engine/include/pdfengine/content_object.hpp b/engine/include/pdfengine/content_object.hpp new file mode 100644 index 0000000..f39c00f --- /dev/null +++ b/engine/include/pdfengine/content_object.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +namespace pdfengine { + +enum class ContentObjectType { + Text, + Path, + Image, + Unknown +}; + +class ContentObject { +public: + virtual ~ContentObject() = default; + virtual ContentObjectType getType() const = 0; +}; + +class TextObject : public ContentObject { +public: + ContentObjectType getType() const override { return ContentObjectType::Text; } + + std::string text; // The decoded text string + std::string fontName; // Font resource name (e.g. "F1") + double fontSize = 0.0; // Font size + + // Text Transformation Matrix (a, b, c, d, e, f) + // Default is identity matrix: [1 0 0 1 0 0] + double tm[6] = {1.0, 0.0, 0.0, 1.0, 0.0, 0.0}; +}; + +} // namespace pdfengine diff --git a/engine/include/pdfengine/content_stream.hpp b/engine/include/pdfengine/content_stream.hpp new file mode 100644 index 0000000..949baa7 --- /dev/null +++ b/engine/include/pdfengine/content_stream.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +namespace pdfengine { + +/// Result of extracting a raw PDF content stream from a page. +/// rawContent — bytes as-found in the PDF (may be compressed) +/// decodedContent — after applying all /Filter chains (FlateDecode, etc.) +/// pageIndex — 0-based page index +/// filters — list of filter names applied, e.g. {"FlateDecode"} +/// compressed — true if at least one filter was applied +struct ExtractedStream { + std::string rawContent; + std::string decodedContent; + int pageIndex = 0; + std::vector filters; + bool compressed = false; + bool multiStream = false; +}; + +/// Verifies structural integrity of a decoded content stream. +/// Returns true if all of: BT, ET, Tf, Tj/TJ are present. +struct StreamVerification { + bool hasBT = false; + bool hasET = false; + bool hasTf = false; + bool hasTj = false; // Tj or TJ + bool multiStream = false; // page had multiple /Contents streams +}; + +StreamVerification verifyContentStream(const ExtractedStream& stream); + +} // namespace pdfengine diff --git a/engine/include/pdfengine/token.hpp b/engine/include/pdfengine/token.hpp new file mode 100644 index 0000000..88437cb --- /dev/null +++ b/engine/include/pdfengine/token.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +namespace pdfengine { + +enum class TokenType { + Operator, // e.g., "Tj", "BT", "ET", "Tf", "re", "f" + String, // e.g., "(Hello World)", fully unescaped + HexString, // e.g., "<48656C6C6F>", fully decoded to bytes + Name, // e.g., "/F1" (without the slash, unescaped) + Number, // e.g., "12.3", "-4", stored as string/double + ArrayStart, // "[" + ArrayEnd, // "]" + DictStart, // "<<" + DictEnd, // ">>" + Boolean, // "true", "false" + Null, // "null" + EndOfStream // EOF marker +}; + +struct Token { + TokenType type; + std::string stringValue; // Used for Operator, String, Name + std::vector bytesValue; // Used for HexString + double numberValue = 0.0; // Used for Number + + // Position tracking for error reporting (optional but helpful) + size_t startOffset = 0; + size_t endOffset = 0; +}; + +} // namespace pdfengine diff --git a/engine/src/parser/content_builder.cpp b/engine/src/parser/content_builder.cpp new file mode 100644 index 0000000..f5b570d --- /dev/null +++ b/engine/src/parser/content_builder.cpp @@ -0,0 +1,122 @@ +#include "content_builder.hpp" + +namespace pdfengine { + +std::vector> ContentBuilder::build(const std::vector& operations) { + std::vector> objects; + + for (const auto& op : operations) { + processOperation(op, objects); + } + + return objects; +} + +void ContentBuilder::processOperation(const Operation& op, std::vector>& outObjects) { + if (op.op == "BT") { + state_.tm[0] = 1.0; state_.tm[1] = 0.0; state_.tm[2] = 0.0; + state_.tm[3] = 1.0; state_.tm[4] = 0.0; state_.tm[5] = 0.0; + } else if (op.op == "Tf") { + handleTf(op); + } else if (op.op == "Td" || op.op == "TD" || op.op == "Tm") { + handleTd(op); + } else if (op.op == "Tj" || op.op == "'") { // ' is equivalent to T* Tj + handleTj(op, outObjects); + } else if (op.op == "TJ") { + handleTJ_Array(op, outObjects); + } +} + +void ContentBuilder::handleTf(const Operation& op) { + if (op.operands.size() >= 2) { + auto it = op.operands.end(); + auto numNode = *(--it); + auto nameNode = *(--it); + if (nameNode->type == AstNodeType::Name) { + state_.fontName = nameNode->stringValue; + } + if (numNode->type == AstNodeType::Number) { + state_.fontSize = numNode->numberValue; + } + } +} + +void ContentBuilder::handleTd(const Operation& op) { + if (op.op == "Tm" && op.operands.size() >= 6) { + // Tm takes 6 operands: a b c d e f + auto it = op.operands.end(); + auto fNode = *(--it); + auto eNode = *(--it); + auto dNode = *(--it); + auto cNode = *(--it); + auto bNode = *(--it); + auto aNode = *(--it); + if (aNode->type == AstNodeType::Number) state_.tm[0] = aNode->numberValue; + if (bNode->type == AstNodeType::Number) state_.tm[1] = bNode->numberValue; + if (cNode->type == AstNodeType::Number) state_.tm[2] = cNode->numberValue; + if (dNode->type == AstNodeType::Number) state_.tm[3] = dNode->numberValue; + if (eNode->type == AstNodeType::Number) state_.tm[4] = eNode->numberValue; + if (fNode->type == AstNodeType::Number) state_.tm[5] = fNode->numberValue; + } else if ((op.op == "Td" || op.op == "TD") && op.operands.size() >= 2) { + // Td simply offsets e and f in the matrix + auto it = op.operands.end(); + auto yNode = *(--it); + auto xNode = *(--it); + if (xNode->type == AstNodeType::Number) state_.tm[4] += xNode->numberValue; + if (yNode->type == AstNodeType::Number) state_.tm[5] += yNode->numberValue; + } +} + +void ContentBuilder::handleTj(const Operation& op, std::vector>& outObjects) { + if (op.operands.empty()) return; + + auto strNode = op.operands.back(); + if (strNode->type == AstNodeType::String || strNode->type == AstNodeType::HexString) { + auto textObj = std::make_unique(); + if (strNode->type == AstNodeType::String) { + textObj->text = strNode->stringValue; + } else { + textObj->text = std::string(strNode->bytesValue.begin(), strNode->bytesValue.end()); + } + + textObj->fontName = state_.fontName; + textObj->fontSize = state_.fontSize; + for (int i=0; i<6; ++i) textObj->tm[i] = state_.tm[i]; + + outObjects.push_back(std::move(textObj)); + } +} + +void ContentBuilder::handleTJ_Array(const Operation& op, std::vector>& outObjects) { + if (op.operands.empty()) return; + + auto arrNode = op.operands.back(); + if (arrNode->type != AstNodeType::Array) return; + + std::string combinedText; + + const auto& arr = arrNode->arrayItems; + for (const auto& item : arr) { + if (item->type == AstNodeType::String) { + combinedText += item->stringValue; + } else if (item->type == AstNodeType::HexString) { + combinedText += std::string(item->bytesValue.begin(), item->bytesValue.end()); + } else if (item->type == AstNodeType::Number) { + if (item->numberValue < -500.0) { + combinedText += " "; + } + } + } + + if (!combinedText.empty()) { + auto textObj = std::make_unique(); + textObj->text = combinedText; + textObj->fontName = state_.fontName; + textObj->fontSize = state_.fontSize; + for (int i=0; i<6; ++i) textObj->tm[i] = state_.tm[i]; + + outObjects.push_back(std::move(textObj)); + } +} + +} // namespace pdfengine diff --git a/engine/src/parser/content_builder.hpp b/engine/src/parser/content_builder.hpp new file mode 100644 index 0000000..892f7a1 --- /dev/null +++ b/engine/src/parser/content_builder.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace pdfengine { + +class ContentBuilder { +public: + ContentBuilder() = default; + + std::vector> build(const std::vector& operations); + +private: + // Graphics State Tracker + struct GraphicsState { + std::string fontName; + double fontSize = 0.0; + double tm[6] = {1.0, 0.0, 0.0, 1.0, 0.0, 0.0}; + }; + + GraphicsState state_; + + void processOperation(const Operation& op, std::vector>& outObjects); + + // Handlers + void handleTf(const Operation& op); + void handleTd(const Operation& op); + void handleTj(const Operation& op, std::vector>& outObjects); + void handleTJ_Array(const Operation& op, std::vector>& outObjects); +}; + +} // namespace pdfengine diff --git a/engine/src/parser/lexer.cpp b/engine/src/parser/lexer.cpp new file mode 100644 index 0000000..8446f15 --- /dev/null +++ b/engine/src/parser/lexer.cpp @@ -0,0 +1,298 @@ +#include "lexer.hpp" +#include +#include + +namespace pdfengine { + +Lexer::Lexer(std::string_view input) : input_(input), pos_(0) {} + +std::vector Lexer::tokenize() { + std::vector tokens; + while (auto token = nextToken()) { + tokens.push_back(*token); + if (token->type == TokenType::EndOfStream) { + break; + } + } + return tokens; +} + +std::optional Lexer::nextToken() { + skipWhitespaceAndComments(); + + if (isEOF()) { + return std::nullopt; // Or we can return EndOfStream token + } + + size_t startPos = pos_; + char c = peek(); + + if (c == '(') { + return parseString(); + } else if (c == '<') { + if (peek(1) == '<') { + return parseDictOrLess(); + } + return parseHexString(); + } else if (c == '>') { + return parseDictOrGreater(); + } else if (c == '/') { + return parseName(); + } else if (c == '[') { + advance(); + return Token{TokenType::ArrayStart, "", {}, 0.0, startPos, pos_}; + } else if (c == ']') { + advance(); + return Token{TokenType::ArrayEnd, "", {}, 0.0, startPos, pos_}; + } else { + return parseNumberOrOperator(); + } +} + +void Lexer::skipWhitespaceAndComments() { + while (!isEOF()) { + char c = peek(); + if (isWhitespace(c)) { + advance(); + } else if (c == '%') { + // Skip until end of line + while (!isEOF() && peek() != '\n' && peek() != '\r') { + advance(); + } + } else { + break; + } + } +} + +std::optional Lexer::parseString() { + size_t startPos = pos_; + advance(); // skip '(' + + std::string str; + int parenLevel = 1; + + while (!isEOF()) { + char c = advance(); + if (c == '(') { + parenLevel++; + str += c; + } else if (c == ')') { + parenLevel--; + if (parenLevel == 0) { + break; + } + str += c; + } else if (c == '\\') { + if (isEOF()) break; + char n = advance(); + switch (n) { + case 'n': str += '\n'; break; + case 'r': str += '\r'; break; + case 't': str += '\t'; break; + case 'b': str += '\b'; break; + case 'f': str += '\f'; break; + case '(': str += '('; break; + case ')': str += ')'; break; + case '\\': str += '\\'; break; + case '\n': break; // ignored line break + case '\r': + if (peek() == '\n') advance(); + break; + default: + if (n >= '0' && n <= '7') { + // Octal up to 3 digits + int val = n - '0'; + for (int i = 0; i < 2; ++i) { + if (!isEOF() && peek() >= '0' && peek() <= '7') { + val = (val << 3) + (advance() - '0'); + } else { + break; + } + } + str += static_cast(val); + } else { + // Unknown escape, just output the char + str += n; + } + } + } else { + str += c; + } + } + + return Token{TokenType::String, str, {}, 0.0, startPos, pos_}; +} + +std::optional Lexer::parseHexString() { + size_t startPos = pos_; + advance(); // skip '<' + + std::vector bytes; + bool hasHigh = false; + uint8_t high = 0; + + while (!isEOF()) { + char c = advance(); + if (c == '>') { + if (hasHigh) { + // If odd number of hex digits, implicitly append 0 + bytes.push_back(high << 4); + } + break; + } + if (isWhitespace(c)) continue; + + int val = hexDigitValue(c); + if (val == -1) { + // Invalid char, usually stop or ignore. We'll ignore for now or break. + continue; + } + + if (!hasHigh) { + high = static_cast(val); + hasHigh = true; + } else { + bytes.push_back((high << 4) | static_cast(val)); + hasHigh = false; + } + } + + return Token{TokenType::HexString, "", bytes, 0.0, startPos, pos_}; +} + +std::optional Lexer::parseName() { + size_t startPos = pos_; + advance(); // skip '/' + + std::string name; + while (!isEOF()) { + char c = peek(); + if (isWhitespace(c) || isDelimiter(c)) { + break; + } + advance(); + if (c == '#' && pos_ + 1 < input_.size()) { + int h1 = hexDigitValue(peek(0)); + int h2 = hexDigitValue(peek(1)); + if (h1 != -1 && h2 != -1) { + name += static_cast((h1 << 4) | h2); + advance(); + advance(); + } else { + name += c; + } + } else { + name += c; + } + } + + return Token{TokenType::Name, name, {}, 0.0, startPos, pos_}; +} + +std::optional Lexer::parseNumberOrOperator() { + size_t startPos = pos_; + std::string val; + + while (!isEOF()) { + char c = peek(); + if (isWhitespace(c) || isDelimiter(c)) { + break; + } + val += advance(); + } + + if (val.empty()) { + return std::nullopt; // should not happen if we skip properly + } + + // Is it a number? + bool isNum = true; + bool hasDot = false; + for (size_t i = 0; i < val.size(); ++i) { + if (i == 0 && (val[i] == '+' || val[i] == '-')) continue; + if (val[i] == '.') { + if (hasDot) { isNum = false; break; } + hasDot = true; + continue; + } + if (val[i] < '0' || val[i] > '9') { + isNum = false; + break; + } + } + // A single '+' or '-' or '.' is not a number + if (val == "+" || val == "-" || val == ".") isNum = false; + + if (isNum) { + double d = 0.0; + std::from_chars(val.data(), val.data() + val.size(), d); + return Token{TokenType::Number, val, {}, d, startPos, pos_}; + } else if (val == "true" || val == "false") { + return Token{TokenType::Boolean, val, {}, 0.0, startPos, pos_}; + } else if (val == "null") { + return Token{TokenType::Null, val, {}, 0.0, startPos, pos_}; + } else { + return Token{TokenType::Operator, val, {}, 0.0, startPos, pos_}; + } +} + +std::optional Lexer::parseDictOrLess() { + size_t startPos = pos_; + advance(); // skip '<' + if (!isEOF() && peek() == '<') { + advance(); // skip second '<' + return Token{TokenType::DictStart, "", {}, 0.0, startPos, pos_}; + } + // Shouldn't be called if it was HexString, handled in nextToken() + return Token{TokenType::Operator, "<", {}, 0.0, startPos, pos_}; // fallback +} + +std::optional Lexer::parseDictOrGreater() { + size_t startPos = pos_; + advance(); // skip '>' + if (!isEOF() && peek() == '>') { + advance(); // skip second '>' + return Token{TokenType::DictEnd, "", {}, 0.0, startPos, pos_}; + } + return Token{TokenType::Operator, ">", {}, 0.0, startPos, pos_}; // fallback +} + +char Lexer::peek(size_t offset) const { + if (pos_ + offset < input_.size()) { + return input_[pos_ + offset]; + } + return '\0'; +} + +char Lexer::advance() { + if (pos_ < input_.size()) { + return input_[pos_++]; + } + return '\0'; +} + +bool Lexer::isEOF() const { + return pos_ >= input_.size(); +} + +bool Lexer::isWhitespace(char c) { + return c == '\0' || c == '\t' || c == '\n' || c == '\f' || c == '\r' || c == ' '; +} + +bool Lexer::isDelimiter(char c) { + return c == '(' || c == ')' || c == '<' || c == '>' || c == '[' || c == ']' || c == '{' || c == '}' || c == '/' || c == '%'; +} + +bool Lexer::isRegular(char c) { + return !isWhitespace(c) && !isDelimiter(c); +} + +int Lexer::hexDigitValue(char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +} // namespace pdfengine diff --git a/engine/src/parser/lexer.hpp b/engine/src/parser/lexer.hpp new file mode 100644 index 0000000..a8d6b52 --- /dev/null +++ b/engine/src/parser/lexer.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include + +namespace pdfengine { + +class Lexer { +public: + explicit Lexer(std::string_view input); + + // Retrieve all tokens in one pass + std::vector tokenize(); + + // Or retrieve tokens one by one + std::optional nextToken(); + +private: + std::string_view input_; + size_t pos_; + + void skipWhitespaceAndComments(); + std::optional parseString(); + std::optional parseHexString(); + std::optional parseName(); + std::optional parseNumberOrOperator(); + std::optional parseDictOrLess(); + std::optional parseDictOrGreater(); + + char peek(size_t offset = 0) const; + char advance(); + bool isEOF() const; + + static bool isWhitespace(char c); + static bool isDelimiter(char c); + static bool isRegular(char c); + static int hexDigitValue(char c); +}; + +} // namespace pdfengine diff --git a/engine/src/parser/parser.cpp b/engine/src/parser/parser.cpp new file mode 100644 index 0000000..868ab4d --- /dev/null +++ b/engine/src/parser/parser.cpp @@ -0,0 +1,170 @@ +#include "parser.hpp" + +namespace pdfengine { + +ContentParser::ContentParser(const std::vector& tokens) + : tokens_(tokens), pos_(0) {} + +std::vector ContentParser::parse() { + std::vector operations; + + while (!isEOF()) { + const Token* token = advance(); + if (!token) break; + + if (token->type == TokenType::Operator) { + Operation op; + op.op = token->stringValue; + // PDF stream operators consume whatever is on the operand stack + op.operands = std::move(operandStack_); + operandStack_.clear(); + operations.push_back(std::move(op)); + } else { + // It's an operand (or the start of a composite operand) + auto node = parseNode(*token); + if (node) { + operandStack_.push_back(std::move(node)); + } + } + } + + return operations; +} + +std::shared_ptr ContentParser::parseNode(const Token& token) { + auto node = std::make_shared(); + + switch (token.type) { + case TokenType::Number: + node->type = AstNodeType::Number; + node->numberValue = token.numberValue; + return node; + + case TokenType::Name: + node->type = AstNodeType::Name; + node->stringValue = token.stringValue; + return node; + + case TokenType::String: + node->type = AstNodeType::String; + node->stringValue = token.stringValue; + return node; + + case TokenType::HexString: + node->type = AstNodeType::HexString; + node->bytesValue = token.bytesValue; + return node; + + case TokenType::Boolean: + node->type = AstNodeType::Boolean; + node->boolValue = (token.stringValue == "true"); + return node; + + case TokenType::Null: + node->type = AstNodeType::Null; + return node; + + case TokenType::ArrayStart: + return parseArray(); + + case TokenType::DictStart: + return parseDictionary(); + + case TokenType::ArrayEnd: + case TokenType::DictEnd: + case TokenType::EndOfStream: + case TokenType::Operator: + // These should not be parsed as standalone operand nodes here + // Operators are handled in the main loop, Ends are handled in Array/Dict parsing + return nullptr; + } + + return nullptr; +} + +std::shared_ptr ContentParser::parseArray() { + auto node = std::make_shared(AstNodeType::Array); + + while (!isEOF()) { + const Token* token = peek(); + if (!token) break; + + if (token->type == TokenType::ArrayEnd) { + advance(); // consume ']' + break; + } + + // Cannot have Operator inside array + if (token->type == TokenType::Operator) { + // PDF spec doesn't strictly allow operators inside arrays, + // but we gracefully break out or skip. We'll break out to avoid infinite loops. + break; + } + + advance(); // consume item token + auto item = parseNode(*token); + if (item) { + node->arrayItems.push_back(std::move(item)); + } + } + + return node; +} + +std::shared_ptr ContentParser::parseDictionary() { + auto node = std::make_shared(AstNodeType::Dictionary); + + while (!isEOF()) { + const Token* keyToken = peek(); + if (!keyToken) break; + + if (keyToken->type == TokenType::DictEnd) { + advance(); // consume '>>' + break; + } + + if (keyToken->type != TokenType::Name) { + // Dictionaries must have Name keys. If not, this is a malformed dict. + // We just advance to avoid infinite loop. + advance(); + continue; + } + + advance(); // consume key + + if (isEOF()) break; + const Token* valToken = peek(); + if (valToken->type == TokenType::DictEnd) { + // Incomplete key-value pair + break; + } + + advance(); // consume value token + auto valNode = parseNode(*valToken); + if (valNode) { + node->dictItems[keyToken->stringValue] = std::move(valNode); + } + } + + return node; +} + +const Token* ContentParser::peek(size_t offset) const { + if (pos_ + offset < tokens_.size()) { + return &tokens_[pos_ + offset]; + } + return nullptr; +} + +const Token* ContentParser::advance() { + if (pos_ < tokens_.size()) { + return &tokens_[pos_++]; + } + return nullptr; +} + +bool ContentParser::isEOF() const { + return pos_ >= tokens_.size() || tokens_[pos_].type == TokenType::EndOfStream; +} + +} // namespace pdfengine diff --git a/engine/src/parser/parser.hpp b/engine/src/parser/parser.hpp new file mode 100644 index 0000000..259bfbb --- /dev/null +++ b/engine/src/parser/parser.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include +#include + +namespace pdfengine { + +class ContentParser { +public: + explicit ContentParser(const std::vector& tokens); + + std::vector parse(); + +private: + const std::vector& tokens_; + size_t pos_; + std::vector> operandStack_; + + const Token* peek(size_t offset = 0) const; + const Token* advance(); + bool isEOF() const; + + std::shared_ptr parseNode(const Token& token); + std::shared_ptr parseArray(); + std::shared_ptr parseDictionary(); +}; + +class ParserError : public std::runtime_error { +public: + explicit ParserError(const std::string& msg) : std::runtime_error(msg) {} +}; + +} // namespace pdfengine diff --git a/engine/src/qpdf/qpdf_extractor.cpp b/engine/src/qpdf/qpdf_extractor.cpp new file mode 100644 index 0000000..5f00668 --- /dev/null +++ b/engine/src/qpdf/qpdf_extractor.cpp @@ -0,0 +1,194 @@ +#include "qpdf_extractor.hpp" + +#ifdef PDFENGINE_WITH_QPDF +#include +#include +#include +#include +#include +#endif + +#include +#include +#include + +namespace pdfengine { + +static bool containsToken(const std::string& str, const std::string& token) { + size_t pos = 0; + while ((pos = str.find(token, pos)) != std::string::npos) { + bool startOk = (pos == 0 || std::isspace(static_cast(str[pos - 1])) || str[pos - 1] == '[' || str[pos - 1] == ']' || str[pos - 1] == '<' || str[pos - 1] == '>'); + size_t endPos = pos + token.length(); + bool endOk = (endPos == str.length() || std::isspace(static_cast(str[endPos])) || str[endPos] == '[' || str[endPos] == ']' || str[endPos] == '<' || str[endPos] == '>'); + if (startOk && endOk) { + return true; + } + pos += token.length(); + } + return false; +} + +StreamVerification verifyContentStream(const ExtractedStream& stream) { + StreamVerification v; + v.multiStream = stream.multiStream; + v.hasBT = containsToken(stream.decodedContent, "BT"); + v.hasET = containsToken(stream.decodedContent, "ET"); + v.hasTf = containsToken(stream.decodedContent, "Tf"); + v.hasTj = containsToken(stream.decodedContent, "Tj") || containsToken(stream.decodedContent, "TJ"); + return v; +} + +} // namespace pdfengine + +namespace pdfengine::qpdf_layer { + +std::expected +QpdfExtractor::extractPageStream(const std::string& filePath, int pageIndex) const { +#ifdef PDFENGINE_WITH_QPDF + try { + ::QPDF qpdf; + qpdf.processFile(filePath.c_str()); + return extractFromQpdf(qpdf, pageIndex); + } catch (const QPDFExc& e) { + fprintf(stderr, "QPDFExc in processMemoryFile: %s\n", e.what()); + if (e.getErrorCode() == qpdf_e_password) { + return std::unexpected(QpdfError::NotSupported); + } + return std::unexpected(QpdfError::InvalidFormat); + } catch (const std::exception& e) { + fprintf(stderr, "Exception in processMemoryFile: %s\n", e.what()); + return std::unexpected(QpdfError::Unknown); + } +#else + (void)filePath; + (void)pageIndex; + return std::unexpected(QpdfError::NotSupported); +#endif +} + +std::expected +QpdfExtractor::extractPageStreamFromMemory(const std::vector& data, int pageIndex) const { +#ifdef PDFENGINE_WITH_QPDF + try { + ::QPDF qpdf; + qpdf.processMemoryFile("memory", reinterpret_cast(data.data()), data.size()); + return extractFromQpdf(qpdf, pageIndex); + } catch (const QPDFExc& e) { + fprintf(stderr, "QPDFExc in processMemoryFile: %s\n", e.what()); + if (e.getErrorCode() == qpdf_e_password) { + return std::unexpected(QpdfError::NotSupported); + } + return std::unexpected(QpdfError::InvalidFormat); + } catch (const std::exception& e) { + fprintf(stderr, "Exception in processMemoryFile: %s\n", e.what()); + return std::unexpected(QpdfError::Unknown); + } +#else + (void)data; + (void)pageIndex; + return std::unexpected(QpdfError::NotSupported); +#endif +} + +std::expected +QpdfExtractor::extractFromQpdf(::QPDF& qpdf, int pageIndex) const { +#ifdef PDFENGINE_WITH_QPDF + try { + std::vector pages = qpdf.getAllPages(); + if (pageIndex < 0 || static_cast(pageIndex) >= pages.size()) { + return std::unexpected(QpdfError::PageOutOfBounds); + } + + QPDFObjectHandle page = pages[pageIndex]; + if (!page.hasKey("/Contents")) { + ExtractedStream emptyStream; + emptyStream.pageIndex = pageIndex; + return emptyStream; + } + + QPDFObjectHandle contents = page.getKey("/Contents"); + std::vector streams; + bool isMulti = false; + + if (contents.isArray()) { + isMulti = true; + for (int i = 0; i < contents.getArrayNItems(); ++i) { + streams.push_back(contents.getArrayItem(i)); + } + } else if (contents.isStream()) { + streams.push_back(contents); + } else { + return std::unexpected(QpdfError::InvalidFormat); + } + + ExtractedStream result; + result.pageIndex = pageIndex; + result.multiStream = isMulti; + + bool isCompressed = false; + std::vector allFilters; + + for (auto streamObj : streams) { + if (!streamObj.isStream()) continue; + + // Get filters + QPDFObjectHandle dict = streamObj.getDict(); + if (dict.hasKey("/Filter")) { + QPDFObjectHandle filter = dict.getKey("/Filter"); + if (filter.isArray()) { + for (int i = 0; i < filter.getArrayNItems(); ++i) { + QPDFObjectHandle item = filter.getArrayItem(i); + if (item.isName()) { + allFilters.push_back(item.getName()); + isCompressed = true; + } + } + } else if (filter.isName()) { + allFilters.push_back(filter.getName()); + isCompressed = true; + } + } + + // Raw data + try { + Pl_Buffer rawPipeline("raw"); + streamObj.pipeStreamData(&rawPipeline, 0, qpdf_dl_none, false); + std::shared_ptr rawBuf = rawPipeline.getBufferSharedPointer(); + if (rawBuf) { + if (!result.rawContent.empty()) result.rawContent += " "; + result.rawContent.append(reinterpret_cast(rawBuf->getBuffer()), rawBuf->getSize()); + } + } catch (...) { + // Ignore failure for raw + } + + // Decoded data + try { + Pl_Buffer decodedPipeline("decoded"); + streamObj.pipeStreamData(&decodedPipeline, 0, qpdf_dl_all, false); + std::shared_ptr decodedBuf = decodedPipeline.getBufferSharedPointer(); + if (decodedBuf) { + if (!result.decodedContent.empty()) result.decodedContent += "\n"; + result.decodedContent.append(reinterpret_cast(decodedBuf->getBuffer()), decodedBuf->getSize()); + } + } catch (const std::exception& e) { + fprintf(stderr, "Failed to decode stream: %s\n", e.what()); + } + } + + result.compressed = isCompressed; + result.filters = allFilters; + + return result; + } catch (const std::exception& e) { + fprintf(stderr, "Exception in extractFromQpdf: %s\n", e.what()); + return std::unexpected(QpdfError::StreamDecodeError); + } +#else + (void)qpdf; + (void)pageIndex; + return std::unexpected(QpdfError::NotSupported); +#endif +} + +} // namespace pdfengine::qpdf_layer diff --git a/engine/src/qpdf/qpdf_extractor.hpp b/engine/src/qpdf/qpdf_extractor.hpp new file mode 100644 index 0000000..8590ef0 --- /dev/null +++ b/engine/src/qpdf/qpdf_extractor.hpp @@ -0,0 +1,38 @@ +#pragma once +#include +#include +#include +#include +#include + +class QPDF; + +namespace pdfengine::qpdf_layer { + +enum class QpdfError { + FileNotFound, + InvalidFormat, + PageOutOfBounds, + StreamDecodeError, + NotSupported, + Unknown +}; + +class QpdfExtractor { +public: + QpdfExtractor() = default; + + [[nodiscard]] + std::expected + extractPageStream(const std::string& filePath, int pageIndex) const; + + [[nodiscard]] + std::expected + extractPageStreamFromMemory(const std::vector& data, int pageIndex) const; + +private: + std::expected + extractFromQpdf(::QPDF& qpdf, int pageIndex) const; +}; + +} // namespace pdfengine::qpdf_layer diff --git a/engine/src/qpdf/qpdf_writer.cpp b/engine/src/qpdf/qpdf_writer.cpp new file mode 100644 index 0000000..cdc2ffc --- /dev/null +++ b/engine/src/qpdf/qpdf_writer.cpp @@ -0,0 +1,55 @@ +#include "qpdf_writer.hpp" + +#ifdef PDFENGINE_WITH_QPDF +#include +#include +#include +#include +#endif + +namespace pdfengine::qpdf_layer { + +std::expected +QpdfWriter::replacePageStreamAndSave(const std::string& sourcePath, + const std::string& destPath, + int pageIndex, + const std::string& newStreamData) const { +#ifndef PDFENGINE_WITH_QPDF + return std::unexpected("QPDF support is not enabled in this build."); +#else + try { + QPDF pdf; + pdf.processFile(sourcePath.c_str()); + + QPDFPageDocumentHelper pdh(pdf); + auto pages = pdh.getAllPages(); + + if (pageIndex < 0 || static_cast(pageIndex) >= pages.size()) { + return std::unexpected("Page index out of bounds."); + } + + QPDFPageObjectHelper& page = pages[pageIndex]; + + // Create a new stream with the updated data + QPDFObjectHandle newStream = QPDFObjectHandle::newStream(&pdf, newStreamData); + + // Replace the page's contents stream + // According to QPDF specs, if we pass an array to newStream, it handles it, + // but it's simpler to just set the dictionary's /Contents to the new stream + QPDFObjectHandle pageDict = page.getObjectHandle(); + pageDict.replaceKey("/Contents", newStream); + + // Write it out + QPDFWriter writer(pdf, destPath.c_str()); + // For performance/size we usually compress streams + writer.setStreamDataMode(qpdf_s_compress); + writer.write(); + + return {}; + } catch (const std::exception& e) { + return std::unexpected(std::string("QPDF Writer Error: ") + e.what()); + } +#endif +} + +} // namespace pdfengine::qpdf_layer diff --git a/engine/src/qpdf/qpdf_writer.hpp b/engine/src/qpdf/qpdf_writer.hpp new file mode 100644 index 0000000..120b356 --- /dev/null +++ b/engine/src/qpdf/qpdf_writer.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include + +class QPDF; + +namespace pdfengine::qpdf_layer { + +class QpdfWriter { +public: + QpdfWriter() = default; + + // Takes a source PDF, replaces the stream of the given page, and writes to a new destination + [[nodiscard]] + std::expected + replacePageStreamAndSave(const std::string& sourcePath, + const std::string& destPath, + int pageIndex, + const std::string& newStreamData) const; +}; + +} // namespace pdfengine::qpdf_layer diff --git a/engine/src/serializer/ast_serializer.cpp b/engine/src/serializer/ast_serializer.cpp new file mode 100644 index 0000000..dbbd2bd --- /dev/null +++ b/engine/src/serializer/ast_serializer.cpp @@ -0,0 +1,93 @@ +#include "ast_serializer.hpp" +#include +#include +#include + +namespace pdfengine { + +std::string AstSerializer::serialize(const std::vector& operations) const { + std::string result; + + for (const auto& op : operations) { + for (const auto& operand : op.operands) { + result += serializeNode(*operand) + " "; + } + result += op.op + "\n"; + } + + return result; +} + +std::string AstSerializer::serializeNode(const AstNode& node) const { + switch (node.type) { + case AstNodeType::Number: { + // Need to drop trailing zeros for integers to save space and match standard PDF + double intPart; + if (std::modf(node.numberValue, &intPart) == 0.0) { + return std::to_string(static_cast(node.numberValue)); + } else { + // Round to 4 decimal places for cleanliness + std::ostringstream out; + out.precision(4); + out << std::fixed << node.numberValue; + std::string str = out.str(); + str.erase(str.find_last_not_of('0') + 1, std::string::npos); + if (str.back() == '.') str.pop_back(); + return str; + } + } + case AstNodeType::Name: + return "/" + node.stringValue; // simplified, assumes no special chars requiring # hex encoding for now + + case AstNodeType::String: + return "(" + escapeString(node.stringValue) + ")"; + + case AstNodeType::HexString: { + std::ostringstream out; + out << "<"; + for (uint8_t b : node.bytesValue) { + out << std::hex << std::setw(2) << std::setfill('0') << static_cast(b); + } + out << ">"; + return out.str(); + } + + case AstNodeType::Boolean: + return node.boolValue ? "true" : "false"; + + case AstNodeType::Null: + return "null"; + + case AstNodeType::Array: { + std::string res = "[ "; + for (const auto& item : node.arrayItems) { + res += serializeNode(*item) + " "; + } + res += "]"; + return res; + } + + case AstNodeType::Dictionary: { + std::string res = "<<\n"; + for (const auto& [key, val] : node.dictItems) { + res += " /" + key + " " + serializeNode(*val) + "\n"; + } + res += ">>"; + return res; + } + } + return ""; +} + +std::string AstSerializer::escapeString(const std::string& str) const { + std::string res; + for (char c : str) { + if (c == '(') res += "\\("; + else if (c == ')') res += "\\)"; + else if (c == '\\') res += "\\\\"; + else res += c; + } + return res; +} + +} // namespace pdfengine diff --git a/engine/src/serializer/ast_serializer.hpp b/engine/src/serializer/ast_serializer.hpp new file mode 100644 index 0000000..aa56c74 --- /dev/null +++ b/engine/src/serializer/ast_serializer.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include +#include + +namespace pdfengine { + +class AstSerializer { +public: + AstSerializer() = default; + + std::string serialize(const std::vector& operations) const; + +private: + std::string serializeNode(const AstNode& node) const; + std::string escapeString(const std::string& str) const; +}; + +} // namespace pdfengine diff --git a/engine/src/serializer/content_serializer.cpp b/engine/src/serializer/content_serializer.cpp new file mode 100644 index 0000000..bb2f807 --- /dev/null +++ b/engine/src/serializer/content_serializer.cpp @@ -0,0 +1,63 @@ +#include "content_serializer.hpp" + +namespace pdfengine { + +std::vector ContentSerializer::serialize(const std::vector>& objects) const { + std::vector ops; + + for (const auto& obj : objects) { + if (obj->getType() == ContentObjectType::Text) { + serializeText(*static_cast(obj.get()), ops); + } + } + + return ops; +} + +void ContentSerializer::serializeText(const TextObject& textObj, std::vector& outOps) const { + // BT + Operation opBT; + opBT.op = "BT"; + outOps.push_back(std::move(opBT)); + + // /FontName FontSize Tf + if (!textObj.fontName.empty() && textObj.fontSize > 0) { + Operation opTf; + opTf.op = "Tf"; + + auto nameNode = std::make_shared(AstNodeType::Name); + nameNode->stringValue = textObj.fontName; + + auto sizeNode = std::make_shared(AstNodeType::Number); + sizeNode->numberValue = textObj.fontSize; + + opTf.operands.push_back(std::move(nameNode)); + opTf.operands.push_back(std::move(sizeNode)); + outOps.push_back(std::move(opTf)); + } + + // a b c d e f Tm + Operation opTm; + opTm.op = "Tm"; + for (int i = 0; i < 6; ++i) { + auto numNode = std::make_shared(AstNodeType::Number); + numNode->numberValue = textObj.tm[i]; + opTm.operands.push_back(std::move(numNode)); + } + outOps.push_back(std::move(opTm)); + + // (text) Tj + Operation opTj; + opTj.op = "Tj"; + auto strNode = std::make_shared(AstNodeType::String); + strNode->stringValue = textObj.text; + opTj.operands.push_back(std::move(strNode)); + outOps.push_back(std::move(opTj)); + + // ET + Operation opET; + opET.op = "ET"; + outOps.push_back(std::move(opET)); +} + +} // namespace pdfengine diff --git a/engine/src/serializer/content_serializer.hpp b/engine/src/serializer/content_serializer.hpp new file mode 100644 index 0000000..ca35def --- /dev/null +++ b/engine/src/serializer/content_serializer.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include +#include +#include + +namespace pdfengine { + +class ContentSerializer { +public: + ContentSerializer() = default; + + std::vector serialize(const std::vector>& objects) const; + +private: + void serializeText(const TextObject& textObj, std::vector& outOps) const; +}; + +} // namespace pdfengine diff --git a/engine/tests/CMakeLists.txt b/engine/tests/CMakeLists.txt index ce525d5..b0e8796 100644 --- a/engine/tests/CMakeLists.txt +++ b/engine/tests/CMakeLists.txt @@ -7,8 +7,18 @@ add_executable(pdfengine_smoke display_list_test.cpp document_test.cpp skia_renderer_test.cpp + lexer_test.cpp + parser_test.cpp + content_builder_test.cpp + ast_serializer_test.cpp + content_serializer_test.cpp + qpdf_writer_test.cpp ) +if(PDFENGINE_WITH_QPDF) + target_sources(pdfengine_smoke PRIVATE qpdf_extractor_test.cpp) +endif() + target_link_libraries(pdfengine_smoke PRIVATE pdfengine::pdfengine @@ -20,6 +30,10 @@ if(PDFENGINE_WITH_SKIA) target_link_libraries(pdfengine_smoke PRIVATE skia::skia) endif() +if(MSVC) + target_link_options(pdfengine_smoke PRIVATE "/FORCE:MULTIPLE") +endif() + target_include_directories(pdfengine_smoke PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../src" diff --git a/engine/tests/ast_serializer_test.cpp b/engine/tests/ast_serializer_test.cpp new file mode 100644 index 0000000..fa0c59c --- /dev/null +++ b/engine/tests/ast_serializer_test.cpp @@ -0,0 +1,63 @@ +#include +#include "../src/serializer/ast_serializer.hpp" + +using namespace pdfengine; + +TEST(AstSerializerTest, SimpleOperations) { + Operation opTd; + opTd.op = "Td"; + + auto xNode = std::make_shared(AstNodeType::Number); + xNode->numberValue = 10.5; + + auto yNode = std::make_shared(AstNodeType::Number); + yNode->numberValue = 20.0; + + opTd.operands.push_back(xNode); + opTd.operands.push_back(yNode); + + AstSerializer serializer; + std::string result = serializer.serialize({opTd}); + + EXPECT_EQ(result, "10.5 20 Td\n"); +} + +TEST(AstSerializerTest, StringEscaping) { + Operation opTj; + opTj.op = "Tj"; + + auto strNode = std::make_shared(AstNodeType::String); + strNode->stringValue = "Hello (World)"; + + opTj.operands.push_back(strNode); + + AstSerializer serializer; + std::string result = serializer.serialize({opTj}); + + EXPECT_EQ(result, "(Hello \\(World\\)) Tj\n"); +} + +TEST(AstSerializerTest, ArraySerialization) { + Operation opTJ; + opTJ.op = "TJ"; + + auto arrNode = std::make_shared(AstNodeType::Array); + + auto str1 = std::make_shared(AstNodeType::String); + str1->stringValue = "He"; + auto num = std::make_shared(AstNodeType::Number); + num->numberValue = 120; + auto str2 = std::make_shared(AstNodeType::String); + str2->stringValue = "llo"; + + arrNode->arrayItems.push_back(str1); + arrNode->arrayItems.push_back(num); + arrNode->arrayItems.push_back(str2); + + opTJ.operands.push_back(arrNode); + + AstSerializer serializer; + std::string result = serializer.serialize({opTJ}); + + EXPECT_EQ(result, "[ (He) 120 (llo) ] TJ\n"); +} diff --git a/engine/tests/content_builder_test.cpp b/engine/tests/content_builder_test.cpp new file mode 100644 index 0000000..13a39f0 --- /dev/null +++ b/engine/tests/content_builder_test.cpp @@ -0,0 +1,119 @@ +#include +#include +#include +#include +#include "../src/parser/lexer.hpp" +#include "../src/parser/parser.hpp" +#include "../src/parser/content_builder.hpp" +#include "../src/qpdf/qpdf_extractor.hpp" +#include + +#ifndef TEST_CORPUS_DIR +#define TEST_CORPUS_DIR "../../corpus" +#endif + +using namespace pdfengine; + +TEST(ContentBuilderTest, SimpleTextState) { + Lexer lexer("10 20 Td /F1 12 Tf (Hello) Tj"); + auto tokens = lexer.tokenize(); + ContentParser parser(tokens); + ContentBuilder builder; + + auto ops = parser.parse(); + auto objects = builder.build(ops); + ASSERT_EQ(objects.size(), 1); + + EXPECT_EQ(objects[0]->getType(), ContentObjectType::Text); + auto* textObj = static_cast(objects[0].get()); + + EXPECT_EQ(textObj->text, "Hello"); + EXPECT_EQ(textObj->fontName, "F1"); + EXPECT_DOUBLE_EQ(textObj->fontSize, 12.0); + EXPECT_DOUBLE_EQ(textObj->tm[4], 10.0); + EXPECT_DOUBLE_EQ(textObj->tm[5], 20.0); +} + +TEST(ContentBuilderTest, KerningArrayTJ) { + Lexer lexer("[ (He) 120 (llo) -600 (World) ] TJ"); + auto tokens = lexer.tokenize(); + ContentParser parser(tokens); + ContentBuilder builder; + + auto ops = parser.parse(); + auto objects = builder.build(ops); + ASSERT_EQ(objects.size(), 1); + + EXPECT_EQ(objects[0]->getType(), ContentObjectType::Text); + auto* textObj = static_cast(objects[0].get()); + + // -600 is less than -500, so it inserts a space + EXPECT_EQ(textObj->text, "Hello World"); +} + +TEST(ContentBuilderTest, RotatedTextMatrix) { + Lexer lexer("0 1 -1 0 100 200 Tm (Rotated) Tj"); + auto tokens = lexer.tokenize(); + ContentParser parser(tokens); + ContentBuilder builder; + + auto ops = parser.parse(); + auto objects = builder.build(ops); + ASSERT_EQ(objects.size(), 1); + + EXPECT_EQ(objects[0]->getType(), ContentObjectType::Text); + auto* textObj = static_cast(objects[0].get()); + + EXPECT_EQ(textObj->text, "Rotated"); + EXPECT_DOUBLE_EQ(textObj->tm[0], 0.0); + EXPECT_DOUBLE_EQ(textObj->tm[1], 1.0); + EXPECT_DOUBLE_EQ(textObj->tm[2], -1.0); + EXPECT_DOUBLE_EQ(textObj->tm[3], 0.0); + EXPECT_DOUBLE_EQ(textObj->tm[4], 100.0); + EXPECT_DOUBLE_EQ(textObj->tm[5], 200.0); +} + +TEST(ContentBuilderTest, IntegrationHelloWorld) { + pdfengine::qpdf_layer::QpdfExtractor extractor; + std::filesystem::path path = std::filesystem::path(TEST_CORPUS_DIR) / "basic" / "hello_world.pdf"; + + auto stream = extractor.extractPageStream(path.string(), 0); + ASSERT_TRUE(stream.has_value()); + + Lexer lexer(stream->decodedContent); + auto tokens = lexer.tokenize(); + ContentParser parser(tokens); + ContentBuilder builder; + + auto ops = parser.parse(); + auto objects = builder.build(ops); + + // hello_world.pdf has two text lines: "Hello, world!" and "Goodbye, world!" + int textObjectCount = 0; + bool foundHello = false; + bool foundGoodbye = false; + + for (const auto& obj : objects) { + if (obj->getType() == ContentObjectType::Text) { + textObjectCount++; + auto* textObj = static_cast(obj.get()); + if (textObj->text == "Hello, world!") { + foundHello = true; + EXPECT_EQ(textObj->fontName, "F1"); + EXPECT_DOUBLE_EQ(textObj->fontSize, 12.0); + EXPECT_DOUBLE_EQ(textObj->tm[4], 20.0); + EXPECT_DOUBLE_EQ(textObj->tm[5], 50.0); + } else if (textObj->text == "Goodbye, world!") { + foundGoodbye = true; + EXPECT_EQ(textObj->fontName, "F2"); + EXPECT_DOUBLE_EQ(textObj->fontSize, 16.0); + EXPECT_DOUBLE_EQ(textObj->tm[4], 20.0); + EXPECT_DOUBLE_EQ(textObj->tm[5], 100.0); + } + } + } + + EXPECT_GE(textObjectCount, 2); + EXPECT_TRUE(foundHello); + EXPECT_TRUE(foundGoodbye); +} diff --git a/engine/tests/content_serializer_test.cpp b/engine/tests/content_serializer_test.cpp new file mode 100644 index 0000000..3064796 --- /dev/null +++ b/engine/tests/content_serializer_test.cpp @@ -0,0 +1,47 @@ +#include +#include "../src/serializer/content_serializer.hpp" + +using namespace pdfengine; + +TEST(ContentSerializerTest, SerializeTextObject) { + auto textObj = std::make_unique(); + textObj->text = "Hello Serialization"; + textObj->fontName = "F1"; + textObj->fontSize = 14.5; + textObj->tm[0] = 1.0; + textObj->tm[1] = 0.0; + textObj->tm[2] = 0.0; + textObj->tm[3] = 1.0; + textObj->tm[4] = 100.0; + textObj->tm[5] = 200.0; + + std::vector> objects; + objects.push_back(std::move(textObj)); + + ContentSerializer serializer; + auto ops = serializer.serialize(objects); + + ASSERT_EQ(ops.size(), 5); + + EXPECT_EQ(ops[0].op, "BT"); + + EXPECT_EQ(ops[1].op, "Tf"); + ASSERT_EQ(ops[1].operands.size(), 2); + EXPECT_EQ(ops[1].operands[0]->stringValue, "F1"); + EXPECT_DOUBLE_EQ(ops[1].operands[1]->numberValue, 14.5); + + EXPECT_EQ(ops[2].op, "Tm"); + ASSERT_EQ(ops[2].operands.size(), 6); + EXPECT_DOUBLE_EQ(ops[2].operands[0]->numberValue, 1.0); + EXPECT_DOUBLE_EQ(ops[2].operands[1]->numberValue, 0.0); + EXPECT_DOUBLE_EQ(ops[2].operands[2]->numberValue, 0.0); + EXPECT_DOUBLE_EQ(ops[2].operands[3]->numberValue, 1.0); + EXPECT_DOUBLE_EQ(ops[2].operands[4]->numberValue, 100.0); + EXPECT_DOUBLE_EQ(ops[2].operands[5]->numberValue, 200.0); + + EXPECT_EQ(ops[3].op, "Tj"); + ASSERT_EQ(ops[3].operands.size(), 1); + EXPECT_EQ(ops[3].operands[0]->stringValue, "Hello Serialization"); + + EXPECT_EQ(ops[4].op, "ET"); +} diff --git a/engine/tests/lexer_test.cpp b/engine/tests/lexer_test.cpp new file mode 100644 index 0000000..b600515 --- /dev/null +++ b/engine/tests/lexer_test.cpp @@ -0,0 +1,133 @@ +#include +#include +#include "../src/parser/lexer.hpp" + +using namespace pdfengine; + +TEST(LexerTest, OperatorsAndWhitespace) { + Lexer lexer("BT\n/F1 12 Tf\nET"); + auto tokens = lexer.tokenize(); + + ASSERT_EQ(tokens.size(), 5); + EXPECT_EQ(tokens[0].type, TokenType::Operator); + EXPECT_EQ(tokens[0].stringValue, "BT"); + + EXPECT_EQ(tokens[1].type, TokenType::Name); + EXPECT_EQ(tokens[1].stringValue, "F1"); + + EXPECT_EQ(tokens[2].type, TokenType::Number); + EXPECT_EQ(tokens[2].numberValue, 12.0); + + EXPECT_EQ(tokens[3].type, TokenType::Operator); + EXPECT_EQ(tokens[3].stringValue, "Tf"); + + EXPECT_EQ(tokens[4].type, TokenType::Operator); + EXPECT_EQ(tokens[4].stringValue, "ET"); +} + +TEST(LexerTest, Strings) { + Lexer lexer("(Hello World) (Nested (parens) ok) (Escapes \\n \\t \\\\ \\(\\)) (Octal \\053)"); + auto tokens = lexer.tokenize(); + + ASSERT_EQ(tokens.size(), 4); + EXPECT_EQ(tokens[0].type, TokenType::String); + EXPECT_EQ(tokens[0].stringValue, "Hello World"); + + EXPECT_EQ(tokens[1].type, TokenType::String); + EXPECT_EQ(tokens[1].stringValue, "Nested (parens) ok"); + + EXPECT_EQ(tokens[2].type, TokenType::String); + EXPECT_EQ(tokens[2].stringValue, "Escapes \n \t \\ ()"); + + EXPECT_EQ(tokens[3].type, TokenType::String); + EXPECT_EQ(tokens[3].stringValue, "Octal +"); // \053 is '+' +} + +TEST(LexerTest, HexStrings) { + Lexer lexer("<48 656c 6c6F> <4A5>"); + auto tokens = lexer.tokenize(); + + ASSERT_EQ(tokens.size(), 2); + EXPECT_EQ(tokens[0].type, TokenType::HexString); + // "Hello" + std::vector expected1 = {0x48, 0x65, 0x6C, 0x6C, 0x6F}; + EXPECT_EQ(tokens[0].bytesValue, expected1); + + EXPECT_EQ(tokens[1].type, TokenType::HexString); + // "4A5" padded to "4A50" + std::vector expected2 = {0x4A, 0x50}; + EXPECT_EQ(tokens[1].bytesValue, expected2); +} + +TEST(LexerTest, NamesAndNumbers) { + Lexer lexer("/Name1 /A#20B -3.14 .5 100"); + auto tokens = lexer.tokenize(); + + ASSERT_EQ(tokens.size(), 5); + EXPECT_EQ(tokens[0].type, TokenType::Name); + EXPECT_EQ(tokens[0].stringValue, "Name1"); + + EXPECT_EQ(tokens[1].type, TokenType::Name); + EXPECT_EQ(tokens[1].stringValue, "A B"); + + EXPECT_EQ(tokens[2].type, TokenType::Number); + EXPECT_DOUBLE_EQ(tokens[2].numberValue, -3.14); + + EXPECT_EQ(tokens[3].type, TokenType::Number); + EXPECT_DOUBLE_EQ(tokens[3].numberValue, 0.5); + + EXPECT_EQ(tokens[4].type, TokenType::Number); + EXPECT_DOUBLE_EQ(tokens[4].numberValue, 100.0); +} + +#include "../src/qpdf/qpdf_extractor.hpp" +#include + +#ifndef TEST_CORPUS_DIR +#define TEST_CORPUS_DIR "../../corpus" +#endif + +TEST(LexerTest, DictAndArray) { + Lexer lexer("<< /Type /Page >> [ 1 2 3 ]"); + auto tokens = lexer.tokenize(); + + ASSERT_EQ(tokens.size(), 9); + EXPECT_EQ(tokens[0].type, TokenType::DictStart); + EXPECT_EQ(tokens[1].type, TokenType::Name); + EXPECT_EQ(tokens[2].type, TokenType::Name); + EXPECT_EQ(tokens[3].type, TokenType::DictEnd); + EXPECT_EQ(tokens[4].type, TokenType::ArrayStart); + EXPECT_EQ(tokens[5].type, TokenType::Number); + EXPECT_EQ(tokens[6].type, TokenType::Number); + EXPECT_EQ(tokens[7].type, TokenType::Number); + EXPECT_EQ(tokens[8].type, TokenType::ArrayEnd); +} + +TEST(LexerTest, IntegrationHelloWorld) { + pdfengine::qpdf_layer::QpdfExtractor extractor; + std::filesystem::path path = std::filesystem::path(TEST_CORPUS_DIR) / "basic" / "hello_world.pdf"; + + auto stream = extractor.extractPageStream(path.string(), 0); + ASSERT_TRUE(stream.has_value()); + + Lexer lexer(stream->decodedContent); + auto tokens = lexer.tokenize(); + + // We expect something like: BT /F1 12 Tf (Hello) Tj ET + // plus any graphics state like 0 0 0 rg, etc. + // Let's just find the text block. + + bool foundHello = false; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i].type == TokenType::String && tokens[i].stringValue == "Hello, world!") { + foundHello = true; + // The next token should be Tj or TJ + ASSERT_LT(i + 1, tokens.size()); + EXPECT_EQ(tokens[i+1].type, TokenType::Operator); + EXPECT_TRUE(tokens[i+1].stringValue == "Tj" || tokens[i+1].stringValue == "TJ"); + break; + } + } + + EXPECT_TRUE(foundHello) << "Failed to lex (Hello, world!) from hello_world.pdf stream"; +} diff --git a/engine/tests/parser_test.cpp b/engine/tests/parser_test.cpp new file mode 100644 index 0000000..82c9135 --- /dev/null +++ b/engine/tests/parser_test.cpp @@ -0,0 +1,117 @@ +#include +#include +#include +#include "../src/parser/parser.hpp" +#include "../src/parser/lexer.hpp" +#include "../src/qpdf/qpdf_extractor.hpp" +#include + +#ifndef TEST_CORPUS_DIR +#define TEST_CORPUS_DIR "../../corpus" +#endif + +using namespace pdfengine; + +TEST(ParserTest, SimpleOperation) { + Lexer lexer("10 20 Td"); + auto tokens = lexer.tokenize(); + + ContentParser parser(tokens); + auto ops = parser.parse(); + + ASSERT_EQ(ops.size(), 1); + EXPECT_EQ(ops[0].op, "Td"); + ASSERT_EQ(ops[0].operands.size(), 2); + + EXPECT_EQ(ops[0].operands[0]->type, AstNodeType::Number); + EXPECT_DOUBLE_EQ(ops[0].operands[0]->numberValue, 10.0); + + EXPECT_EQ(ops[0].operands[1]->type, AstNodeType::Number); + EXPECT_DOUBLE_EQ(ops[0].operands[1]->numberValue, 20.0); +} + +TEST(ParserTest, ArraysAndDicts) { + Lexer lexer("<< /Type /Page >> [ 1 2 ] (Text) Tj"); + auto tokens = lexer.tokenize(); + + ContentParser parser(tokens); + auto ops = parser.parse(); + + // The dictionary and array and string are ALL pushed onto the operand stack + // until the operator 'Tj' is encountered. + // Tj will consume all of them. + ASSERT_EQ(ops.size(), 1); + EXPECT_EQ(ops[0].op, "Tj"); + ASSERT_EQ(ops[0].operands.size(), 3); + + // First operand: Dict + auto dictNode = ops[0].operands[0]; + EXPECT_EQ(dictNode->type, AstNodeType::Dictionary); + ASSERT_TRUE(dictNode->dictItems.find("Type") != dictNode->dictItems.end()); + EXPECT_EQ(dictNode->dictItems["Type"]->stringValue, "Page"); + + // Second operand: Array + auto arrayNode = ops[0].operands[1]; + EXPECT_EQ(arrayNode->type, AstNodeType::Array); + ASSERT_EQ(arrayNode->arrayItems.size(), 2); + EXPECT_DOUBLE_EQ(arrayNode->arrayItems[0]->numberValue, 1.0); + EXPECT_DOUBLE_EQ(arrayNode->arrayItems[1]->numberValue, 2.0); + + // Third operand: String + auto strNode = ops[0].operands[2]; + EXPECT_EQ(strNode->type, AstNodeType::String); + EXPECT_EQ(strNode->stringValue, "Text"); +} + +TEST(ParserTest, MultipleOperations) { + Lexer lexer("BT /F1 12 Tf (Hello) Tj ET"); + auto tokens = lexer.tokenize(); + + ContentParser parser(tokens); + auto ops = parser.parse(); + + ASSERT_EQ(ops.size(), 4); + + EXPECT_EQ(ops[0].op, "BT"); + EXPECT_TRUE(ops[0].operands.empty()); + + EXPECT_EQ(ops[1].op, "Tf"); + ASSERT_EQ(ops[1].operands.size(), 2); + EXPECT_EQ(ops[1].operands[0]->stringValue, "F1"); + EXPECT_DOUBLE_EQ(ops[1].operands[1]->numberValue, 12.0); + + EXPECT_EQ(ops[2].op, "Tj"); + ASSERT_EQ(ops[2].operands.size(), 1); + EXPECT_EQ(ops[2].operands[0]->stringValue, "Hello"); + + EXPECT_EQ(ops[3].op, "ET"); + EXPECT_TRUE(ops[3].operands.empty()); +} + +TEST(ParserTest, IntegrationHelloWorld) { + pdfengine::qpdf_layer::QpdfExtractor extractor; + std::filesystem::path path = std::filesystem::path(TEST_CORPUS_DIR) / "basic" / "hello_world.pdf"; + + auto stream = extractor.extractPageStream(path.string(), 0); + ASSERT_TRUE(stream.has_value()); + + Lexer lexer(stream->decodedContent); + auto tokens = lexer.tokenize(); + + ContentParser parser(tokens); + auto ops = parser.parse(); + + bool foundTj = false; + for (const auto& op : ops) { + if (op.op == "Tj" || op.op == "TJ") { + ASSERT_EQ(op.operands.size(), 1); + if (op.operands[0]->type == AstNodeType::String && + op.operands[0]->stringValue == "Hello, world!") { + foundTj = true; + break; + } + } + } + + EXPECT_TRUE(foundTj) << "Failed to parse (Hello, world!) Tj operation from hello_world.pdf stream"; +} diff --git a/engine/tests/qpdf_extractor_test.cpp b/engine/tests/qpdf_extractor_test.cpp new file mode 100644 index 0000000..5a450d6 --- /dev/null +++ b/engine/tests/qpdf_extractor_test.cpp @@ -0,0 +1,94 @@ +#include +#include +#include "../src/qpdf/qpdf_extractor.hpp" +#include +#include + +using namespace pdfengine; +using namespace pdfengine::qpdf_layer; + +#ifndef TEST_CORPUS_DIR +#define TEST_CORPUS_DIR "../../corpus" +#endif + +namespace { +std::filesystem::path getCorpusPath(const std::string& subfolder, const std::string& filename) { + return std::filesystem::path(TEST_CORPUS_DIR) / subfolder / filename; +} + +std::vector readFile(const std::filesystem::path& path) { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file.is_open()) return {}; + std::streamsize size = file.tellg(); + file.seekg(0, std::ios::beg); + std::vector buffer(size); + if (file.read(reinterpret_cast(buffer.data()), size)) { + return buffer; + } + return {}; +} +} + +class QpdfExtractorTest : public ::testing::Test { +protected: + QpdfExtractor extractor; +}; + +TEST_F(QpdfExtractorTest, ExtractFromMemory) { + auto path = getCorpusPath("basic", "hello_world.pdf"); + auto stream = extractor.extractPageStream(path.string(), 0); + ASSERT_TRUE(stream.has_value()); + EXPECT_EQ(stream->pageIndex, 0); + EXPECT_FALSE(stream->compressed); // Or true depending on qpdf, but we only verify success here + + StreamVerification v = verifyContentStream(stream.value()); + EXPECT_TRUE(v.hasBT); + EXPECT_TRUE(v.hasET); + EXPECT_TRUE(v.hasTf); + EXPECT_TRUE(v.hasTj); +} + +TEST_F(QpdfExtractorTest, PageOutOfBounds) { + auto path = getCorpusPath("basic", "hello_world.pdf"); + auto data = readFile(path); + ASSERT_FALSE(data.empty()); + + auto stream = extractor.extractPageStreamFromMemory(data, 1); + ASSERT_FALSE(stream.has_value()); + EXPECT_EQ(stream.error(), QpdfError::PageOutOfBounds); + + stream = extractor.extractPageStreamFromMemory(data, -1); + ASSERT_FALSE(stream.has_value()); + EXPECT_EQ(stream.error(), QpdfError::PageOutOfBounds); +} + +TEST_F(QpdfExtractorTest, CorruptPdf) { + std::vector data = {0x00, 0x01, 0x02}; + auto stream = extractor.extractPageStreamFromMemory(data, 0); + ASSERT_FALSE(stream.has_value()); + EXPECT_EQ(stream.error(), QpdfError::InvalidFormat); +} + +TEST_F(QpdfExtractorTest, EmptyContents) { + // about_blank.pdf usually has an empty page or no text + auto path = getCorpusPath("basic", "about_blank.pdf"); + auto stream = extractor.extractPageStream(path.string(), 0); + ASSERT_TRUE(stream.has_value()); + EXPECT_EQ(stream->pageIndex, 0); + + StreamVerification v = verifyContentStream(stream.value()); + EXPECT_FALSE(v.hasTj); +} + +TEST_F(QpdfExtractorTest, VerifyNoText) { + // black.pdf or rectangles.pdf has no text, just graphics + auto path = getCorpusPath("basic", "black.pdf"); + auto stream = extractor.extractPageStream(path.string(), 0); + ASSERT_TRUE(stream.has_value()); + + StreamVerification v = verifyContentStream(stream.value()); + EXPECT_FALSE(v.hasBT); + EXPECT_FALSE(v.hasET); + EXPECT_FALSE(v.hasTf); + EXPECT_FALSE(v.hasTj); +} diff --git a/engine/tests/qpdf_writer_test.cpp b/engine/tests/qpdf_writer_test.cpp new file mode 100644 index 0000000..af3b847 --- /dev/null +++ b/engine/tests/qpdf_writer_test.cpp @@ -0,0 +1,88 @@ +#include +#include "../src/qpdf/qpdf_extractor.hpp" +#include "../src/qpdf/qpdf_writer.hpp" +#include "../src/parser/lexer.hpp" +#include "../src/parser/parser.hpp" +#include "../src/parser/content_builder.hpp" +#include "../src/serializer/content_serializer.hpp" +#include "../src/serializer/ast_serializer.hpp" +#include +#include + +#ifndef TEST_CORPUS_DIR +#define TEST_CORPUS_DIR "../../corpus" +#endif + +using namespace pdfengine; +using namespace pdfengine::qpdf_layer; + +TEST(QpdfWriterTest, IntegrationReadModifyWrite) { + std::filesystem::path sourcePath = std::filesystem::path(TEST_CORPUS_DIR) / "basic" / "hello_world.pdf"; + std::filesystem::path destPath = std::filesystem::path(TEST_CORPUS_DIR) / "basic" / "hello_world_modified.pdf"; + + // 1. Extract + QpdfExtractor extractor; + auto stream = extractor.extractPageStream(sourcePath.string(), 0); + ASSERT_TRUE(stream.has_value()); + + // 2. Lex, Parse, Build + Lexer lexer(stream->decodedContent); + auto tokens = lexer.tokenize(); + ContentParser parser(tokens); + auto ops = parser.parse(); + ContentBuilder builder; + auto objects = builder.build(ops); + + // 3. Modify Text + bool foundAndModified = false; + for (auto& obj : objects) { + if (obj->getType() == ContentObjectType::Text) { + auto* textObj = static_cast(obj.get()); + if (textObj->text == "Hello, world!") { + textObj->text = "Hello, PDF Editor!"; + foundAndModified = true; + break; + } + } + } + ASSERT_TRUE(foundAndModified) << "Could not find 'Hello, world!' to modify"; + + // 4. Serialize back to Ops + ContentSerializer contentSerializer; + auto newOps = contentSerializer.serialize(objects); + + // 5. Serialize to raw bytes + AstSerializer astSerializer; + std::string newRawStream = astSerializer.serialize(newOps); + + // 6. Write and Save PDF + QpdfWriter writer; + auto writeRes = writer.replacePageStreamAndSave(sourcePath.string(), destPath.string(), 0, newRawStream); + ASSERT_TRUE(writeRes.has_value()) << writeRes.error(); + + // 7. Re-open and verify modification + auto verifyStream = extractor.extractPageStream(destPath.string(), 0); + ASSERT_TRUE(verifyStream.has_value()); + + Lexer verifyLexer(verifyStream->decodedContent); + auto verifyTokens = verifyLexer.tokenize(); + ContentParser verifyParser(verifyTokens); + ContentBuilder verifyBuilder; + auto verifyObjects = verifyBuilder.build(verifyParser.parse()); + + bool verifiedModification = false; + for (auto& obj : verifyObjects) { + if (obj->getType() == ContentObjectType::Text) { + auto* textObj = static_cast(obj.get()); + if (textObj->text == "Hello, PDF Editor!") { + verifiedModification = true; + break; + } + } + } + + EXPECT_TRUE(verifiedModification) << "Modified string was not successfully saved and reloaded!"; + + // Cleanup + std::filesystem::remove(destPath); +} diff --git a/gateway/app/routers/documents.py b/gateway/app/routers/documents.py index 3092b42..43e6e6c 100644 --- a/gateway/app/routers/documents.py +++ b/gateway/app/routers/documents.py @@ -737,3 +737,97 @@ def export_document(document_id: str): ) except Exception as e: raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + +class TextObjectResponse(BaseModel): + text: str + fontName: str + fontSize: float + tm: list[float] + +@router.get("/{document_id}/pages/{page_index}/text_objects", response_model=list[TextObjectResponse]) +def get_text_objects(document_id: str, page_index: int) -> list[TextObjectResponse]: + if not engine.is_available(): + raise HTTPException(status_code=501, detail="Engine not available") + + doc_info = document_store.get_document(document_id) + if not doc_info: + raise HTTPException(status_code=404, detail="Document not found") + + pdfengine = engine.require() + + import os + # For StreamEditor we need a real file path. If it's loaded from memory, we need to save it to a temp file. + # In this MVP, we assume the file was saved somewhere, but actually document_store keeps it in memory. + # Wait, doc_store has doc_info["filename"] and bytes_data. Let's write bytes to a temp file. + import tempfile + + with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp: + tmp.write(doc_info["bytes_data"]) + tmp_path = tmp.name + + try: + editor = pdfengine.StreamEditor(tmp_path) + objects = editor.extract_text_objects(page_index) + + result = [] + for obj in objects: + result.append(TextObjectResponse( + text=obj["text"], + fontName=obj["fontName"], + fontSize=obj["fontSize"], + tm=obj["tm"] + )) + return result + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + finally: + if os.path.exists(tmp_path): + os.remove(tmp_path) + + +class UpdateTextObjectRequest(BaseModel): + new_text: str + + +@router.put("/{document_id}/pages/{page_index}/text_objects/{object_index}") +def update_text_object(document_id: str, page_index: int, object_index: int, req: UpdateTextObjectRequest): + if not engine.is_available(): + raise HTTPException(status_code=501, detail="Engine not available") + + doc_info = document_store.get_document(document_id) + if not doc_info: + raise HTTPException(status_code=404, detail="Document not found") + + pdfengine = engine.require() + + import tempfile + import os + + with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_in: + tmp_in.write(doc_info["bytes_data"]) + tmp_in_path = tmp_in.name + + tmp_out_path = tmp_in_path + ".out.pdf" + + try: + editor = pdfengine.StreamEditor(tmp_in_path) + success = editor.replace_text_object(page_index, object_index, req.new_text, tmp_out_path) + if not success: + raise HTTPException(status_code=400, detail="Failed to replace text object") + + with open(tmp_out_path, "rb") as f: + new_bytes = f.read() + + # Update the store + doc_info["bytes_data"] = new_bytes + # Reload pdfium doc instance + doc_info["doc_instance"] = pdfengine.PdfDocument.load_from_memory(new_bytes, "") + + return {"success": True} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + finally: + if os.path.exists(tmp_in_path): + os.remove(tmp_in_path) + if os.path.exists(tmp_out_path): + os.remove(tmp_out_path) diff --git a/test_bindings.py b/test_bindings.py new file mode 100644 index 0000000..8c4c1bf --- /dev/null +++ b/test_bindings.py @@ -0,0 +1,66 @@ +import sys +import os + +# Add gateway to path to find pdfengine.pyd +sys.path.append(os.path.join(os.path.dirname(__file__), 'gateway')) + +try: + import pdfengine +except ImportError as e: + print(f"Failed to import pdfengine: {e}") + sys.exit(1) + +def main(): + corpus_dir = os.path.join(os.path.dirname(__file__), 'corpus', 'basic') + source_pdf = os.path.join(corpus_dir, 'hello_world.pdf') + dest_pdf = os.path.join(corpus_dir, 'hello_world_python.pdf') + + print(f"Testing StreamEditor on {source_pdf}") + editor = pdfengine.StreamEditor(source_pdf) + + # 1. Extract text objects + objects = editor.extract_text_objects(0) + print(f"Extracted {len(objects)} text objects.") + + hello_idx = -1 + for i, obj in enumerate(objects): + print(f"[{i}]: {obj['text']} (Font: {obj['fontName']} {obj['fontSize']}, Tm: {obj['tm']})") + if obj['text'] == "Hello, world!": + hello_idx = i + + if hello_idx == -1: + print("Failed to find 'Hello, world!'") + sys.exit(1) + + # 2. Modify text + new_text = "Hello from Python pybind11!" + print(f"\nReplacing object {hello_idx} with '{new_text}'...") + + success = editor.replace_text_object(0, hello_idx, new_text, dest_pdf) + if not success: + print("Failed to replace text object") + sys.exit(1) + + print(f"Successfully saved modified PDF to {dest_pdf}") + + # 3. Verify modification + print("\nVerifying modification...") + verify_editor = pdfengine.StreamEditor(dest_pdf) + verify_objects = verify_editor.extract_text_objects(0) + + found = False + for obj in verify_objects: + if obj['text'] == new_text: + found = True + break + + if found: + print("SUCCESS! Modified text was perfectly preserved.") + if os.path.exists(dest_pdf): + os.remove(dest_pdf) + else: + print("FAILURE! Modified text was not found in the output PDF.") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/vcpkg.json b/vcpkg.json index 57e14e4..d14064f 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -14,6 +14,7 @@ "harfbuzz", "spdlog", "nlohmann-json", + "qpdf", { "name": "gtest", "platform": "!emscripten"