134 lines
4.4 KiB
C++
134 lines
4.4 KiB
C++
#include <gtest/gtest.h>
|
|
#include <pdfengine/token.hpp>
|
|
#include "../src/parser/lexer.hpp"
|
|
|
|
using namespace pdfengine;
|
|
|
|
TEST(LexerTest, OperatorsAndWhitespace) {
|
|
Lexer lexer("BT\n/F1 12 Tf\nET");
|
|
auto tokens = lexer.tokenize();
|
|
|
|
ASSERT_EQ(tokens.size(), 5);
|
|
EXPECT_EQ(tokens[0].type, TokenType::Operator);
|
|
EXPECT_EQ(tokens[0].stringValue, "BT");
|
|
|
|
EXPECT_EQ(tokens[1].type, TokenType::Name);
|
|
EXPECT_EQ(tokens[1].stringValue, "F1");
|
|
|
|
EXPECT_EQ(tokens[2].type, TokenType::Number);
|
|
EXPECT_EQ(tokens[2].numberValue, 12.0);
|
|
|
|
EXPECT_EQ(tokens[3].type, TokenType::Operator);
|
|
EXPECT_EQ(tokens[3].stringValue, "Tf");
|
|
|
|
EXPECT_EQ(tokens[4].type, TokenType::Operator);
|
|
EXPECT_EQ(tokens[4].stringValue, "ET");
|
|
}
|
|
|
|
TEST(LexerTest, Strings) {
|
|
Lexer lexer("(Hello World) (Nested (parens) ok) (Escapes \\n \\t \\\\ \\(\\)) (Octal \\053)");
|
|
auto tokens = lexer.tokenize();
|
|
|
|
ASSERT_EQ(tokens.size(), 4);
|
|
EXPECT_EQ(tokens[0].type, TokenType::String);
|
|
EXPECT_EQ(tokens[0].stringValue, "Hello World");
|
|
|
|
EXPECT_EQ(tokens[1].type, TokenType::String);
|
|
EXPECT_EQ(tokens[1].stringValue, "Nested (parens) ok");
|
|
|
|
EXPECT_EQ(tokens[2].type, TokenType::String);
|
|
EXPECT_EQ(tokens[2].stringValue, "Escapes \n \t \\ ()");
|
|
|
|
EXPECT_EQ(tokens[3].type, TokenType::String);
|
|
EXPECT_EQ(tokens[3].stringValue, "Octal +"); // \053 is '+'
|
|
}
|
|
|
|
TEST(LexerTest, HexStrings) {
|
|
Lexer lexer("<48 656c 6c6F> <4A5>");
|
|
auto tokens = lexer.tokenize();
|
|
|
|
ASSERT_EQ(tokens.size(), 2);
|
|
EXPECT_EQ(tokens[0].type, TokenType::HexString);
|
|
// "Hello"
|
|
std::vector<uint8_t> expected1 = {0x48, 0x65, 0x6C, 0x6C, 0x6F};
|
|
EXPECT_EQ(tokens[0].bytesValue, expected1);
|
|
|
|
EXPECT_EQ(tokens[1].type, TokenType::HexString);
|
|
// "4A5" padded to "4A50"
|
|
std::vector<uint8_t> expected2 = {0x4A, 0x50};
|
|
EXPECT_EQ(tokens[1].bytesValue, expected2);
|
|
}
|
|
|
|
TEST(LexerTest, NamesAndNumbers) {
|
|
Lexer lexer("/Name1 /A#20B -3.14 .5 100");
|
|
auto tokens = lexer.tokenize();
|
|
|
|
ASSERT_EQ(tokens.size(), 5);
|
|
EXPECT_EQ(tokens[0].type, TokenType::Name);
|
|
EXPECT_EQ(tokens[0].stringValue, "Name1");
|
|
|
|
EXPECT_EQ(tokens[1].type, TokenType::Name);
|
|
EXPECT_EQ(tokens[1].stringValue, "A B");
|
|
|
|
EXPECT_EQ(tokens[2].type, TokenType::Number);
|
|
EXPECT_DOUBLE_EQ(tokens[2].numberValue, -3.14);
|
|
|
|
EXPECT_EQ(tokens[3].type, TokenType::Number);
|
|
EXPECT_DOUBLE_EQ(tokens[3].numberValue, 0.5);
|
|
|
|
EXPECT_EQ(tokens[4].type, TokenType::Number);
|
|
EXPECT_DOUBLE_EQ(tokens[4].numberValue, 100.0);
|
|
}
|
|
|
|
#include "../src/qpdf/qpdf_extractor.hpp"
|
|
#include <filesystem>
|
|
|
|
#ifndef TEST_CORPUS_DIR
|
|
#define TEST_CORPUS_DIR "../../corpus"
|
|
#endif
|
|
|
|
TEST(LexerTest, DictAndArray) {
|
|
Lexer lexer("<< /Type /Page >> [ 1 2 3 ]");
|
|
auto tokens = lexer.tokenize();
|
|
|
|
ASSERT_EQ(tokens.size(), 9);
|
|
EXPECT_EQ(tokens[0].type, TokenType::DictStart);
|
|
EXPECT_EQ(tokens[1].type, TokenType::Name);
|
|
EXPECT_EQ(tokens[2].type, TokenType::Name);
|
|
EXPECT_EQ(tokens[3].type, TokenType::DictEnd);
|
|
EXPECT_EQ(tokens[4].type, TokenType::ArrayStart);
|
|
EXPECT_EQ(tokens[5].type, TokenType::Number);
|
|
EXPECT_EQ(tokens[6].type, TokenType::Number);
|
|
EXPECT_EQ(tokens[7].type, TokenType::Number);
|
|
EXPECT_EQ(tokens[8].type, TokenType::ArrayEnd);
|
|
}
|
|
|
|
TEST(LexerTest, IntegrationHelloWorld) {
|
|
pdfengine::qpdf_layer::QpdfExtractor extractor;
|
|
std::filesystem::path path = std::filesystem::path(TEST_CORPUS_DIR) / "basic" / "hello_world.pdf";
|
|
|
|
auto stream = extractor.extractPageStream(path.string(), 0);
|
|
ASSERT_TRUE(stream.has_value());
|
|
|
|
Lexer lexer(stream->decodedContent);
|
|
auto tokens = lexer.tokenize();
|
|
|
|
// We expect something like: BT /F1 12 Tf (Hello) Tj ET
|
|
// plus any graphics state like 0 0 0 rg, etc.
|
|
// Let's just find the text block.
|
|
|
|
bool foundHello = false;
|
|
for (size_t i = 0; i < tokens.size(); ++i) {
|
|
if (tokens[i].type == TokenType::String && tokens[i].stringValue == "Hello, world!") {
|
|
foundHello = true;
|
|
// The next token should be Tj or TJ
|
|
ASSERT_LT(i + 1, tokens.size());
|
|
EXPECT_EQ(tokens[i+1].type, TokenType::Operator);
|
|
EXPECT_TRUE(tokens[i+1].stringValue == "Tj" || tokens[i+1].stringValue == "TJ");
|
|
break;
|
|
}
|
|
}
|
|
|
|
EXPECT_TRUE(foundHello) << "Failed to lex (Hello, world!) from hello_world.pdf stream";
|
|
}
|