67 std::vector<bf::Token<G>> tokens;
68 auto result = parser.Parse(
"3 + 5 - 2", &tokens);
70 ASSERT_TRUE(result.has_value());
72 ASSERT_EQ(tokens.size(), 5);
74 ASSERT_EQ(tokens[0].terminal, &
NUMBER);
75 ASSERT_EQ(tokens[0].location.begin, 0);
77 ASSERT_EQ(tokens[1].terminal, &
OP_ADD);
78 ASSERT_EQ(tokens[1].location.begin, 2);
80 ASSERT_EQ(tokens[2].terminal, &
NUMBER);
81 ASSERT_EQ(tokens[2].location.begin, 4);
83 ASSERT_EQ(tokens[3].terminal, &
OP_SUB);
84 ASSERT_EQ(tokens[3].location.begin, 6);
86 ASSERT_EQ(tokens[4].terminal, &
NUMBER);
87 ASSERT_EQ(tokens[4].location.begin, 8);
90TEST(Tokenization, Permissive)
94 std::vector<bf::Token<G>> tokens;
95 auto result = parser.Parse(
"3[[[+]]]&0", &tokens);
97 ASSERT_FALSE(result.has_value());
99 ASSERT_EQ(tokens.size(), 3);
101 ASSERT_EQ(tokens[0].terminal, &
NUMBER);
102 ASSERT_EQ(tokens[0].location.begin, 0);
104 ASSERT_EQ(tokens[1].terminal, &
OP_ADD);
105 ASSERT_EQ(tokens[1].location.begin, 4);
107 ASSERT_EQ(tokens[2].terminal, &
NUMBER);
108 ASSERT_EQ(tokens[2].location.begin, 9);