#include #include #include "../src/parser/lexer.hpp" using namespace pdfengine; TEST(LexerTest, OperatorsAndWhitespace) { Lexer lexer("BT\n/F1 12 Tf\nET"); auto tokens = lexer.tokenize(); ASSERT_EQ(tokens.size(), 5); EXPECT_EQ(tokens[0].type, TokenType::Operator); EXPECT_EQ(tokens[0].stringValue, "BT"); EXPECT_EQ(tokens[1].type, TokenType::Name); EXPECT_EQ(tokens[1].stringValue, "F1"); EXPECT_EQ(tokens[2].type, TokenType::Number); EXPECT_EQ(tokens[2].numberValue, 12.0); EXPECT_EQ(tokens[3].type, TokenType::Operator); EXPECT_EQ(tokens[3].stringValue, "Tf"); EXPECT_EQ(tokens[4].type, TokenType::Operator); EXPECT_EQ(tokens[4].stringValue, "ET"); } TEST(LexerTest, Strings) { Lexer lexer("(Hello World) (Nested (parens) ok) (Escapes \\n \\t \\\\ \\(\\)) (Octal \\053)"); auto tokens = lexer.tokenize(); ASSERT_EQ(tokens.size(), 4); EXPECT_EQ(tokens[0].type, TokenType::String); EXPECT_EQ(tokens[0].stringValue, "Hello World"); EXPECT_EQ(tokens[1].type, TokenType::String); EXPECT_EQ(tokens[1].stringValue, "Nested (parens) ok"); EXPECT_EQ(tokens[2].type, TokenType::String); EXPECT_EQ(tokens[2].stringValue, "Escapes \n \t \\ ()"); EXPECT_EQ(tokens[3].type, TokenType::String); EXPECT_EQ(tokens[3].stringValue, "Octal +"); } TEST(LexerTest, HexStrings) { Lexer lexer("<48 656c 6c6F> <4A5>"); auto tokens = lexer.tokenize(); ASSERT_EQ(tokens.size(), 2); EXPECT_EQ(tokens[0].type, TokenType::HexString); std::vector expected1 = {0x48, 0x65, 0x6C, 0x6C, 0x6F}; EXPECT_EQ(tokens[0].bytesValue, expected1); EXPECT_EQ(tokens[1].type, TokenType::HexString); std::vector expected2 = {0x4A, 0x50}; EXPECT_EQ(tokens[1].bytesValue, expected2); } TEST(LexerTest, NamesAndNumbers) { Lexer lexer("/Name1 /A#20B -3.14 .5 100"); auto tokens = lexer.tokenize(); ASSERT_EQ(tokens.size(), 5); EXPECT_EQ(tokens[0].type, TokenType::Name); EXPECT_EQ(tokens[0].stringValue, "Name1"); EXPECT_EQ(tokens[1].type, TokenType::Name); EXPECT_EQ(tokens[1].stringValue, "A B"); EXPECT_EQ(tokens[2].type, TokenType::Number); EXPECT_DOUBLE_EQ(tokens[2].numberValue, -3.14); EXPECT_EQ(tokens[3].type, TokenType::Number); EXPECT_DOUBLE_EQ(tokens[3].numberValue, 0.5); EXPECT_EQ(tokens[4].type, TokenType::Number); EXPECT_DOUBLE_EQ(tokens[4].numberValue, 100.0); } #include "../src/qpdf/qpdf_extractor.hpp" #include #ifndef TEST_CORPUS_DIR #define TEST_CORPUS_DIR "../../corpus" #endif TEST(LexerTest, DictAndArray) { Lexer lexer("<< /Type /Page >> [ 1 2 3 ]"); auto tokens = lexer.tokenize(); ASSERT_EQ(tokens.size(), 9); EXPECT_EQ(tokens[0].type, TokenType::DictStart); EXPECT_EQ(tokens[1].type, TokenType::Name); EXPECT_EQ(tokens[2].type, TokenType::Name); EXPECT_EQ(tokens[3].type, TokenType::DictEnd); EXPECT_EQ(tokens[4].type, TokenType::ArrayStart); EXPECT_EQ(tokens[5].type, TokenType::Number); EXPECT_EQ(tokens[6].type, TokenType::Number); EXPECT_EQ(tokens[7].type, TokenType::Number); EXPECT_EQ(tokens[8].type, TokenType::ArrayEnd); } TEST(LexerTest, IntegrationHelloWorld) { pdfengine::qpdf_layer::QpdfExtractor extractor; std::filesystem::path path = std::filesystem::path(TEST_CORPUS_DIR) / "basic" / "hello_world.pdf"; auto stream = extractor.extractPageStream(path.string(), 0); ASSERT_TRUE(stream.has_value()); Lexer lexer(stream->decodedContent); auto tokens = lexer.tokenize(); bool foundHello = false; for (size_t i = 0; i < tokens.size(); ++i) { if (tokens[i].type == TokenType::String && tokens[i].stringValue == "Hello, world!") { foundHello = true; ASSERT_LT(i + 1, tokens.size()); EXPECT_EQ(tokens[i+1].type, TokenType::Operator); EXPECT_TRUE(tokens[i+1].stringValue == "Tj" || tokens[i+1].stringValue == "TJ"); break; } } EXPECT_TRUE(foundHello) << "Failed to lex (Hello, world!) from hello_world.pdf stream"; }