diff --git a/parser/internal/pratt_parser_test.cc b/parser/internal/pratt_parser_test.cc index 6c85dd1d7..17648d27f 100644 --- a/parser/internal/pratt_parser_test.cc +++ b/parser/internal/pratt_parser_test.cc @@ -434,7 +434,7 @@ std::vector GetParserTestCases() { TestCase{ .source = "- -1", .expected_ast = R"( - 1^#3:int64# + 1^#1:int64# )", }, TestCase{ @@ -452,7 +452,7 @@ std::vector GetParserTestCases() { .source = "---a", .expected_ast = R"( -_( - a^#4:Expr.Ident# + a^#2:Expr.Ident# )^#1:Expr.Call# )", }, @@ -482,6 +482,39 @@ std::vector GetParserTestCases() { )^#1:Expr.Call# )", }, + TestCase{ + .source = "-.2.V", + .expected_ast = R"( + -0.2^#1:double#.V^#2:Expr.Select# + )", + }, + TestCase{ + .source = "!-.2.V", + .expected_ast = R"( + !_( + -0.2^#2:double#.V^#3:Expr.Select# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!-2.V", + .expected_ast = R"( + !_( + -2^#2:int64#.V^#3:Expr.Select# + )^#1:Expr.Call# + )", + }, + TestCase{ + .source = "!-.2[0]", + .expected_ast = R"( + !_( + _[_]( + -0.2^#2:double#, + 0^#4:int64# + )^#3:Expr.Call# + )^#1:Expr.Call# + )", + }, TestCase{ .source = "a + b", .expected_ast = R"( @@ -1016,8 +1049,8 @@ std::vector GetParserTestCases() { .source = "{'key': 'value', 'num': 42}", .expected_ast = R"( { - "key"^#2:string#:"value"^#4:string#^#3:Expr.CreateStruct.Entry#, - "num"^#5:string#:42^#7:int64#^#6:Expr.CreateStruct.Entry# + "key"^#3:string#:"value"^#4:string#^#2:Expr.CreateStruct.Entry#, + "num"^#6:string#:42^#7:int64#^#5:Expr.CreateStruct.Entry# }^#1:Expr.CreateMap# )", }, @@ -1025,8 +1058,8 @@ std::vector GetParserTestCases() { .source = "{?'key': 'value', 'num': 42}", .expected_ast = R"( { - ?"key"^#2:string#:"value"^#4:string#^#3:Expr.CreateStruct.Entry#, - "num"^#5:string#:42^#7:int64#^#6:Expr.CreateStruct.Entry# + ?"key"^#3:string#:"value"^#4:string#^#2:Expr.CreateStruct.Entry#, + "num"^#6:string#:42^#7:int64#^#5:Expr.CreateStruct.Entry# }^#1:Expr.CreateMap# )", .enable_optional_syntax = true, @@ -1035,8 +1068,8 @@ std::vector GetParserTestCases() { .source = "{foo: 5, bar: \"xyz\"}", .expected_ast = R"( { - foo^#2:Expr.Ident#:5^#4:int64#^#3:Expr.CreateStruct.Entry#, - bar^#5:Expr.Ident#:"xyz"^#7:string#^#6:Expr.CreateStruct.Entry# + foo^#3:Expr.Ident#:5^#4:int64#^#2:Expr.CreateStruct.Entry#, + bar^#6:Expr.Ident#:"xyz"^#7:string#^#5:Expr.CreateStruct.Entry# }^#1:Expr.CreateMap# )", }, @@ -1156,21 +1189,24 @@ std::vector GetErrorTestCases() { ErrorTestCase{ .source = "1 + 2 * 3 4", .expected_error = - "ERROR: :1:11: unexpected token after expression\n" + "ERROR: :1:11: Syntax error: unexpected token after " + "expression\n" " | 1 + 2 * 3 4\n" " | ..........^", }, ErrorTestCase{ .source = "1{}", .expected_error = - "ERROR: :1:2: unexpected token after expression\n" + "ERROR: :1:2: Syntax error: unexpected token after " + "expression\n" " | 1{}\n" " | .^", }, ErrorTestCase{ .source = "true ? 1", .expected_error = - "ERROR: :1:9: expected ':' in conditional expression\n" + "ERROR: :1:9: Syntax error: expected ':' in conditional " + "expression\n" " | true ? 1\n" " | ........^", }, @@ -1182,10 +1218,10 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "a.", - .expected_error = - "ERROR: :1:3: expected identifier after '.'\n" - " | a.\n" - " | ..^", + .expected_error = "ERROR: :1:3: Syntax error: expected " + "identifier after '.'\n" + " | a.\n" + " | ..^", }, ErrorTestCase{ .source = "a[?0]", @@ -1195,9 +1231,10 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = ". *", - .expected_error = "ERROR: :1:3: expected identifier\n" - " | . *\n" - " | ..^", + .expected_error = + "ERROR: :1:3: Syntax error: expected identifier\n" + " | . *\n" + " | ..^", }, ErrorTestCase{ .source = ".as", @@ -1208,17 +1245,19 @@ std::vector GetErrorTestCases() { ErrorTestCase{ .source = "* 2", .expected_error = - "ERROR: :1:1: unexpected token\n" + "ERROR: :1:1: Syntax error: unexpected token\n" " | * 2\n" " | ^\n" - "ERROR: :1:3: unexpected token after expression\n" + "ERROR: :1:3: Syntax error: unexpected token after " + "expression\n" " | * 2\n" " | ..^", }, ErrorTestCase{ .source = "(1 + 2", .expected_error = - "ERROR: :1:7: mismatched input expecting ')'\n" + "ERROR: :1:7: Syntax error: mismatched input " + "expecting ')'\n" " | (1 + 2\n" " | ......^", }, @@ -1230,7 +1269,7 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "[1, 2", - .expected_error = "ERROR: :1:6: expected ']'\n" + .expected_error = "ERROR: :1:6: Syntax error: expected ']'\n" " | [1, 2\n" " | .....^", }, @@ -1242,13 +1281,14 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "{'k' 'v'}", - .expected_error = "ERROR: :1:6: expected ':' in map entry\n" - " | {'k' 'v'}\n" - " | .....^", + .expected_error = + "ERROR: :1:6: Syntax error: expected ':' in map entry\n" + " | {'k' 'v'}\n" + " | .....^", }, ErrorTestCase{ .source = "{'k': 'v'", - .expected_error = "ERROR: :1:10: expected '}'\n" + .expected_error = "ERROR: :1:10: Syntax error: expected '}'\n" " | {'k': 'v'\n" " | .........^", }, @@ -1260,26 +1300,29 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "Msg{1: 2}", - .expected_error = "ERROR: :1:5: expected struct field name\n" - " | Msg{1: 2}\n" - " | ....^", + .expected_error = + "ERROR: :1:5: Syntax error: expected struct field name\n" + " | Msg{1: 2}\n" + " | ....^", }, ErrorTestCase{ .source = "Msg{f 10}", - .expected_error = "ERROR: :1:7: expected ':' in struct field\n" - " | Msg{f 10}\n" - " | ......^", + .expected_error = + "ERROR: :1:7: Syntax error: expected ':' in struct field\n" + " | Msg{f 10}\n" + " | ......^", }, ErrorTestCase{ .source = "Msg{f: 10", - .expected_error = "ERROR: :1:10: expected '}'\n" + .expected_error = "ERROR: :1:10: Syntax error: expected '}'\n" " | Msg{f: 10\n" " | .........^", }, ErrorTestCase{ .source = "f(1, 2", .expected_error = - "ERROR: :1:7: mismatched input expecting ')'\n" + "ERROR: :1:7: Syntax error: mismatched input " + "expecting ')'\n" " | f(1, 2\n" " | ......^", }, @@ -1291,35 +1334,39 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "999999999999999999999999999999999999999", - .expected_error = "ERROR: :1:1: invalid int literal\n" - " | 999999999999999999999999999999999999999\n" - " | ^", + .expected_error = + "ERROR: :1:1: Syntax error: invalid int literal\n" + " | 999999999999999999999999999999999999999\n" + " | ^", }, ErrorTestCase{ .source = "999999999999999999999999999999999999999u", - .expected_error = "ERROR: :1:1: invalid uint literal\n" - " | 999999999999999999999999999999999999999u\n" - " | ^", + .expected_error = + "ERROR: :1:1: Syntax error: invalid uint literal\n" + " | 999999999999999999999999999999999999999u\n" + " | ^", }, ErrorTestCase{ .source = "1e", .expected_error = - "ERROR: :1:1: floating point literal missing digits after " - "exponent separator\n" + "ERROR: :1:1: Syntax error: floating point literal " + "missing digits after exponent separator\n" " | 1e\n" " | ^", }, ErrorTestCase{ .source = "\"unterminated", - .expected_error = "ERROR: :1:1: unterminated string literal\n" - " | \"unterminated\n" - " | ^", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated string literal\n" + " | \"unterminated\n" + " | ^", }, ErrorTestCase{ .source = "b\"unterminated", - .expected_error = "ERROR: :1:1: unterminated bytes literal\n" - " | b\"unterminated\n" - " | ^", + .expected_error = + "ERROR: :1:1: Syntax error: unterminated bytes literal\n" + " | b\"unterminated\n" + " | ^", }, ErrorTestCase{ .source = "a.?`foo`", @@ -1374,7 +1421,8 @@ std::vector GetErrorTestCases() { ErrorTestCase{ .source = "`foo", .expected_error = - "ERROR: :1:1: unterminated quoted identifier\n" + "ERROR: :1:1: Syntax error: unterminated quoted " + "identifier\n" " | `foo\n" " | ^", .enable_quoted_identifiers = true, @@ -1382,73 +1430,82 @@ std::vector GetErrorTestCases() { ErrorTestCase{ .source = "f(*, 1e, {2 3})", .expected_error = - "ERROR: :1:3: unexpected token\n" + "ERROR: :1:3: Syntax error: unexpected token\n" " | f(*, 1e, {2 3})\n" " | ..^\n" - "ERROR: :1:6: floating point literal missing digits after " - "exponent separator\n" + "ERROR: :1:6: Syntax error: floating point literal " + "missing digits after exponent separator\n" " | f(*, 1e, {2 3})\n" " | .....^\n" - "ERROR: :1:13: expected ':' in map entry\n" + "ERROR: :1:13: Syntax error: expected ':' in map entry\n" " | f(*, 1e, {2 3})\n" " | ............^", }, ErrorTestCase{ .source = "(1 + *) + 2", - .expected_error = "ERROR: :1:6: unexpected token\n" - " | (1 + *) + 2\n" - " | .....^", + .expected_error = + "ERROR: :1:6: Syntax error: unexpected token\n" + " | (1 + *) + 2\n" + " | .....^", }, ErrorTestCase{ .source = "f(1 + *, 2)", - .expected_error = "ERROR: :1:7: unexpected token\n" - " | f(1 + *, 2)\n" - " | ......^", + .expected_error = + "ERROR: :1:7: Syntax error: unexpected token\n" + " | f(1 + *, 2)\n" + " | ......^", }, ErrorTestCase{ .source = "(a. + 1)", .expected_error = - "ERROR: :1:5: expected identifier after '.'\n" + "ERROR: :1:5: Syntax error: expected identifier after " + "'.'\n" " | (a. + 1)\n" " | ....^", }, ErrorTestCase{ .source = "f(a., 1)", .expected_error = - "ERROR: :1:5: expected identifier after '.'\n" + "ERROR: :1:5: Syntax error: expected identifier after " + "'.'\n" " | f(a., 1)\n" " | ....^", }, ErrorTestCase{ .source = "[a., 1]", .expected_error = - "ERROR: :1:4: expected identifier after '.'\n" + "ERROR: :1:4: Syntax error: expected identifier after " + "'.'\n" " | [a., 1]\n" " | ...^", }, ErrorTestCase{ .source = "-0x8000000000000001", - .expected_error = "ERROR: :1:2: invalid int literal\n" - " | -0x8000000000000001\n" - " | .^", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -0x8000000000000001\n" + " | .^", }, ErrorTestCase{ .source = "-0x10000000000000000", - .expected_error = "ERROR: :1:2: invalid int literal\n" - " | -0x10000000000000000\n" - " | .^", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -0x10000000000000000\n" + " | .^", }, ErrorTestCase{ .source = "-9223372036854775809", - .expected_error = "ERROR: :1:2: invalid int literal\n" - " | -9223372036854775809\n" - " | .^", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -9223372036854775809\n" + " | .^", }, ErrorTestCase{ .source = "-999999999999999999999999999999999999999", - .expected_error = "ERROR: :1:2: invalid int literal\n" - " | -999999999999999999999999999999999999999\n" - " | .^", + .expected_error = + "ERROR: :1:2: Syntax error: invalid int literal\n" + " | -999999999999999999999999999999999999999\n" + " | .^", }, ErrorTestCase{ .source = "-", @@ -1460,15 +1517,17 @@ std::vector GetErrorTestCases() { }, ErrorTestCase{ .source = "- *", - .expected_error = "ERROR: :1:3: unexpected token\n" - " | - *\n" - " | ..^", + .expected_error = + "ERROR: :1:3: Syntax error: unexpected token\n" + " | - *\n" + " | ..^", }, ErrorTestCase{ .source = "\"😀😀😀😀😀\" ~error", - .expected_error = "ERROR: :1:9: unexpected character\n" - " | \"😀😀😀😀😀\" ~error\n" - " | .．．．．．..^", + .expected_error = + "ERROR: :1:9: Syntax error: unexpected character\n" + " | \"😀😀😀😀😀\" ~error\n" + " | .．．．．．..^", }, }; } @@ -1826,9 +1885,9 @@ TEST(PrattParserMacroErrorTest, ReportError) { auto ast = parser->Parse(*source, &issues); EXPECT_THAT(ast, StatusIs(absl::StatusCode::kInvalidArgument)); EXPECT_EQ(FormatIssues(*source, issues), - "ERROR: :1:6: custom macro error\n" + "ERROR: :1:15: custom macro error\n" " | 42 + bad_macro(x)\n" - " | .....^"); + " | ..............^"); } TEST(PrattParserMacroErrorTest, ReportErrorAt) { @@ -1875,7 +1934,7 @@ TEST(PrattParserErrorRecoveryTest, ErrorRecoveryLimitOne) { ASSERT_OK_AND_ASSIGN(auto source, cel::NewSource("......")); EXPECT_EQ(FormatIssues(*source, issues), "ERROR: :-1:0: Error recovery limit (1) exceeded\n" - "ERROR: :1:2: expected identifier\n" + "ERROR: :1:2: Syntax error: expected identifier\n" " | ......\n" " | .^"); } diff --git a/parser/internal/pratt_parser_worker.cc b/parser/internal/pratt_parser_worker.cc index b7356528d..b01408774 100644 --- a/parser/internal/pratt_parser_worker.cc +++ b/parser/internal/pratt_parser_worker.cc @@ -133,7 +133,7 @@ Token ParserWorker::NextSignificantToken(bool report_error) { continue; } if (tok.type == TokenType::kError && report_error) { - ReportError(tok, lexer_.GetError().message); + ReportSyntaxError(tok, lexer_.GetError().message); if (is_recovery_limit_exceeded()) { return Token{.type = TokenType::kEnd, .start = 0, .end = 0}; } @@ -177,7 +177,7 @@ bool ParserWorker::Expect(TokenType type, absl::string_view msg) { } else { err_msg = std::string(msg); } - ReportError(peek_token_, err_msg); + ReportSyntaxError(peek_token_, err_msg); } SynchronizeOnDelimiter(); return false; @@ -216,6 +216,14 @@ int64_t ParserWorker::NextId(int32_t position) { int64_t ParserWorker::NextId() { return NextId(-1); } +void ParserWorker::SetPosition(int64_t id, const Token& token) { + if (token.start >= 0) { + positions_[id] = token.start; + SetNodeRange(id, token.start, + token.end > token.start ? token.end - 1 : token.start); + } +} + int64_t ParserWorker::CopyId(int64_t id) { if (id == 0) { return 0; @@ -271,4 +279,9 @@ void ParserWorker::ReportError(const SourceLocation& loc, } } +void ParserWorker::ReportSyntaxError(const Token& token, + absl::string_view msg) { + ReportError(token.start, absl::StrCat("Syntax error: ", msg)); +} + } // namespace cel::parser_internal diff --git a/parser/internal/pratt_parser_worker.h b/parser/internal/pratt_parser_worker.h index f4886aa90..5e66cd2a0 100644 --- a/parser/internal/pratt_parser_worker.h +++ b/parser/internal/pratt_parser_worker.h @@ -16,6 +16,7 @@ #define THIRD_PARTY_CEL_CPP_PARSER_INTERNAL_PRATT_PARSER_WORKER_H_ #include +#include #include #include #include @@ -83,14 +84,12 @@ class ParserWorker { int64_t NextId(int32_t position); int64_t NextId(const Token& token) { int64_t id = NextId(token.start); - if (ABSL_PREDICT_FALSE(track_node_ranges_)) { - if (token.start >= 0 && token.end > token.start) { - node_ranges_[id] = {token.start, token.end - 1}; - } - } + SetNodeRange(id, token.start, + token.end > token.start ? token.end - 1 : token.start); return id; } int64_t NextId(); + void SetPosition(int64_t id, const Token& token); int64_t CopyId(int64_t id); void EraseId(int64_t id); void SetNodeRange(int64_t id, int32_t begin, int32_t end) { @@ -110,6 +109,7 @@ class ParserWorker { void ReportError(const Token& token, absl::string_view msg) { ReportError(token.start, msg); } + void ReportSyntaxError(const Token& token, absl::string_view msg); const cel::Source& source_; cel::ParserOptions options_; @@ -149,6 +149,7 @@ template class PrattParserWorker : public ParserWorker { public: using ParserWorker::NextId; + using ParserWorker::SetPosition; explicit PrattParserWorker( const cel::Source& source, const cel::ParserOptions& options, @@ -335,7 +336,7 @@ ExprNode PrattParserWorker::Parse() { } if (peek_token_.type != TokenType::kEnd && peek_token_.type != TokenType::kError) { - ReportError(peek_token_, "unexpected token after expression"); + ReportSyntaxError(peek_token_, "unexpected token after expression"); } return expr; } @@ -357,8 +358,8 @@ ExprNode PrattParserWorker::ParseExpr() { template void PrattParserWorker::ParseTernary(ExprNode& lhs) { - NextToken(); - int64_t op_id = NextId(); + Token op_tok = NextToken(); + int64_t op_id = NextId(op_tok); ExprNode true_expr = ParseBinaryAndTernary(1); if (!Expect(TokenType::kColon, "expected ':' in conditional expression")) { return; @@ -462,7 +463,7 @@ void PrattParserWorker::ParseSelectorChainTail(ExprNode& lhs) { if (id_tok.type != TokenType::kIdent && id_tok.type != TokenType::kReservedWord) { if (id_tok.type != TokenType::kError) { - ReportError(id_tok, "expected identifier after '.'"); + ReportSyntaxError(id_tok, "expected identifier after '.'"); } SynchronizeOnDelimiter(); return; @@ -529,55 +530,61 @@ void PrattParserWorker::ParseSelectorChainTail(ExprNode& lhs) { template ExprNode PrattParserWorker::ParseUnaryOpsChain(Token first_op) { - struct UnaryOpInfo { - TokenType type; - int64_t id; + struct UnaryOp { + Token token; + int64_t id = 0; }; - std::vector ops; - ops.push_back({first_op.type, NextId(first_op)}); - + std::vector ops; + ops.push_back({first_op}); while (peek_token_.type == TokenType::kExclamation || peek_token_.type == TokenType::kMinus) { - Token op = NextToken(); - ops.push_back({op.type, NextId(op)}); + ops.push_back({NextToken()}); + } + + // Match the ANTLR parser behavior where `-(-)+` prefers to match as + // repeated negate operators instead of a negation of an int literal. + // ---9223372036854775808 will fail to parse. + const bool has_solitary_trailing_minus = + !ops.empty() && ops.back().token.type == TokenType::kMinus && + (ops.size() == 1 || ops[ops.size() - 2].token.type != TokenType::kMinus); + + if (options_.fold_unary_operators) { + size_t write = 0; + for (size_t read = 0; read < ops.size();) { + size_t next = read; + while (next < ops.size() && + ops[next].token.type == ops[read].token.type) { + next++; + } + if ((next - read) % 2 != 0) { + ops[write++] = ops[read]; + } + read = next; + } + ops.resize(write); + } + + for (auto& op : ops) { + op.id = NextId(op.token); } ExprNode operand; - if (!ops.empty() && ops.back().type == TokenType::kMinus) { - if (options_.fold_unary_operators && ops.size() > 1 && - ops[ops.size() - 2].type == TokenType::kMinus) { - // Match the ANTLR parser behavior where `-(-)+` prefers to match as - // repeated negate operators instead of a negation of an int literal. - // ---9223372036854775808 will fail to parse. - ops.pop_back(); - ops.pop_back(); - operand = ParseSelectorChain(); - } else if (peek_token_.type == TokenType::kInt) { - int64_t op_id = ops.back().id; - ops.pop_back(); - operand = ParseNegativeIntLiteral(op_id); - } else if (peek_token_.type == TokenType::kFloat) { - int64_t op_id = ops.back().id; - ops.pop_back(); - operand = ParseNegativeDoubleLiteral(op_id); - } else { - operand = ParseSelectorChain(); - } + if (has_solitary_trailing_minus && (peek_token_.type == TokenType::kInt || + peek_token_.type == TokenType::kFloat)) { + int64_t op_id = ops.back().id; + ops.pop_back(); + operand = (peek_token_.type == TokenType::kInt) + ? ParseNegativeIntLiteral(op_id) + : ParseNegativeDoubleLiteral(op_id); + ParseSelectorChainTail(operand); } else { operand = ParseSelectorChain(); } for (int i = static_cast(ops.size()) - 1; i >= 0; --i) { std::vector args; - if (options_.fold_unary_operators && i > 0) { - if (ops[i - 1].type == ops[i].type) { - i--; - continue; - } - } - args.push_back(std::move(operand)); - absl::string_view op_name = (ops[i].type == TokenType::kExclamation) + absl::string_view op_name = (ops[i].token.type == TokenType::kExclamation) ? CelOperator::LOGICAL_NOT : CelOperator::NEGATE; operand = @@ -641,7 +648,7 @@ ExprNode PrattParserWorker::ParseIdentOrCall() { if (id_tok.type != TokenType::kIdent && id_tok.type != TokenType::kReservedWord) { if (id_tok.type != TokenType::kError) { - ReportError(id_tok, "expected identifier"); + ReportSyntaxError(id_tok, "expected identifier"); } return ast_factory_.NewUnspecified(NextId(id_tok)); } @@ -653,16 +660,17 @@ ExprNode PrattParserWorker::ParseIdentOrCall() { } std::string name = leading_dot ? absl::StrCat(".", id_text) : std::string(id_text); - int64_t id = NextId(leading_dot ? first_tok : id_tok); if (peek_token_.type == TokenType::kLeftParen) { - NextToken(); + Token lparen = NextToken(); + int64_t call_id = NextId(lparen); std::vector args = ParseArguments(TokenType::kRightParen); - if (auto expanded = TryExpandMacro(id, name, nullptr, args); + if (auto expanded = TryExpandMacro(call_id, name, nullptr, args); expanded.has_value()) { return std::move(*expanded); } - return ast_factory_.NewCall(id, name, std::move(args)); + return ast_factory_.NewCall(call_id, name, std::move(args)); } + int64_t id = NextId(leading_dot ? first_tok : id_tok); return ast_factory_.NewIdent(id, std::move(name)); } @@ -715,11 +723,10 @@ ExprNode PrattParserWorker::ParsePrimary() { Token bad_tok = NextToken(); if (bad_tok.type != TokenType::kError) { if (bad_tok.type == TokenType::kEnd) { - ReportError( - bad_tok, - "Syntax error: mismatched input '' expecting expression"); + ReportSyntaxError(bad_tok, + "mismatched input '' expecting expression"); } else { - ReportError(bad_tok, "unexpected token"); + ReportSyntaxError(bad_tok, "unexpected token"); } } return ast_factory_.NewUnspecified(NextId(bad_tok)); @@ -774,12 +781,13 @@ ExprNode PrattParserWorker::ParseMap() { } key_start = peek_token_; } + int64_t entry_id = NextId(); ExprNode key = ParseExpr(); Token colon = peek_token_; if (!Expect(TokenType::kColon, "expected ':' in map entry")) { break; } - int64_t entry_id = NextId(colon); + SetPosition(entry_id, colon); builder.Add(entry_id, std::move(key), ParseExpr(), optional); if (peek_token_.type == TokenType::kComma) { NextToken(); @@ -813,7 +821,7 @@ ExprNode PrattParserWorker::ParseStruct( Token field_tok = NextToken(); if (field_tok.type != TokenType::kIdent && field_tok.type != TokenType::kReservedWord) { - ReportError(field_tok, "expected struct field name"); + ReportSyntaxError(field_tok, "expected struct field name"); SynchronizeOnDelimiter(); break; } @@ -877,7 +885,7 @@ ExprNode PrattParserWorker::ParseIntLiteral() { } else if (absl::SimpleAtoi(value, &int_val)) { return ast_factory_.NewIntConst(NextId(tok), int_val); } - ReportError(tok, "invalid int literal"); + ReportSyntaxError(tok, "invalid int literal"); return ast_factory_.NewUnspecified(NextId(tok)); } @@ -911,7 +919,7 @@ ExprNode PrattParserWorker::ParseNegativeIntLiteral(int64_t node_id) { if (success) { return ast_factory_.NewIntConst(node_id, int_val); } - ReportError(lit_tok, "invalid int literal"); + ReportSyntaxError(lit_tok, "invalid int literal"); return ast_factory_.NewUnspecified(NextId(lit_tok)); } @@ -931,7 +939,7 @@ ExprNode PrattParserWorker::ParseUintLiteral() { } else if (absl::SimpleAtoi(value, &uint_val)) { return ast_factory_.NewUintConst(NextId(tok), uint_val); } - ReportError(tok, "invalid uint literal"); + ReportSyntaxError(tok, "invalid uint literal"); return ast_factory_.NewUnspecified(NextId(tok)); } @@ -944,7 +952,7 @@ ExprNode PrattParserWorker::ParseDoubleLiteral() { if (absl::SimpleAtod(value, &double_val)) { return ast_factory_.NewDoubleConst(NextId(tok), double_val); } - ReportError(tok, "invalid double literal"); + ReportSyntaxError(tok, "invalid double literal"); return ast_factory_.NewUnspecified(NextId(tok)); } @@ -956,7 +964,7 @@ ExprNode PrattParserWorker::ParseNegativeDoubleLiteral( if (absl::SimpleAtod(GetTokenText(lit_tok), &double_val)) { return ast_factory_.NewDoubleConst(node_id, -double_val); } - ReportError(lit_tok, "invalid double literal"); + ReportSyntaxError(lit_tok, "invalid double literal"); return ast_factory_.NewUnspecified(NextId(lit_tok)); } diff --git a/parser/parser_test.cc b/parser/parser_test.cc index e47557e6b..113133769 100644 --- a/parser/parser_test.cc +++ b/parser/parser_test.cc @@ -65,15 +65,8 @@ struct TestInfo { TestInfo(const std::string& I, const std::string& P, const std::string& E = "", const std::string& L = "", const std::string& R = "", const std::string& M = "", - const std::string& P_PRATT = "", const std::string& E_PRATT = "") - : I(I), - P(P), - E(E), - L(L), - R(R), - M(M), - P_PRATT(P_PRATT), - E_PRATT(E_PRATT) {} + const std::string& E_PRATT = "") + : I(I), P(P), E(E), L(L), R(R), M(M), E_PRATT(E_PRATT) {} // I contains the input expression to be parsed. std::string I; @@ -94,9 +87,6 @@ struct TestInfo { // M contains the expected macro call output of hte expression tree. std::string M; - // P_PRATT contains alternative adorned AST string when using pratt parser. - std::string P_PRATT; - // E_PRATT contains alternative error output when using pratt parser. std::string E_PRATT; }; @@ -152,12 +142,16 @@ std::vector test_cases = { "{\n" " foo^#3:Expr.Ident#:5^#4:int64#^#2:Expr.CreateStruct.Entry#,\n" " bar^#6:Expr.Ident#:\"xyz\"^#7:string#^#5:Expr.CreateStruct.Entry#\n" - "}^#1:Expr.CreateStruct#", - "", "", "", "", - // PRATT PARSER AST + "}^#1:Expr.CreateStruct#"}, + {"{\"foo\": 5, \"bar\": \"xyz\"}", + "{\n" + " \"foo\"^#3:string#:5^#4:int64#^#2:Expr.CreateStruct.Entry#,\n" + " \"bar\"^#6:string#:\"xyz\"^#7:string#^#5:Expr.CreateStruct.Entry#\n" + "}^#1:Expr.CreateStruct#"}, + {"{'a': 1, 'b': 2}", "{\n" - " foo^#2:Expr.Ident#:5^#4:int64#^#3:Expr.CreateStruct.Entry#,\n" - " bar^#5:Expr.Ident#:\"xyz\"^#7:string#^#6:Expr.CreateStruct.Entry#\n" + " \"a\"^#3:string#:1^#4:int64#^#2:Expr.CreateStruct.Entry#,\n" + " \"b\"^#6:string#:2^#7:int64#^#5:Expr.CreateStruct.Entry#\n" "}^#1:Expr.CreateStruct#"}, {"a > 5 && a < 10", "_&&_(\n" @@ -189,9 +183,9 @@ std::vector test_cases = { "NUM_INT, " "NUM_UINT, STRING, BYTES, IDENTIFIER}\n | {\n" " | .^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:2: expected '}'\n" + "ERROR: :1:2: Syntax error: expected '}'\n" " | {\n" " | .^"}, @@ -386,12 +380,6 @@ std::vector test_cases = { "{\n" " a^#3:Expr.Ident#:b^#4:Expr.Ident#^#2:Expr.CreateStruct.Entry#,\n" " c^#6:Expr.Ident#:d^#7:Expr.Ident#^#5:Expr.CreateStruct.Entry#\n" - "}^#1:Expr.CreateStruct#", - "", "", "", "", - // PRATT PARSER AST - "{\n" - " a^#2:Expr.Ident#:b^#4:Expr.Ident#^#3:Expr.CreateStruct.Entry#,\n" - " c^#5:Expr.Ident#:d^#7:Expr.Ident#^#6:Expr.CreateStruct.Entry#\n" "}^#1:Expr.CreateStruct#"}, {"[]", "[]^#1:Expr.CreateList#"}, {"[a]", @@ -457,12 +445,12 @@ std::vector test_cases = { "ERROR: :1:7: Syntax error: extraneous input 'b' expecting \n" " | *@a | b\n" " | ......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:1: unexpected token\n" + "ERROR: :1:1: Syntax error: unexpected token\n" " | *@a | b\n" " | ^\n" - "ERROR: :1:2: unexpected character\n" + "ERROR: :1:2: Syntax error: unexpected character\n" " | *@a | b\n" " | .^"}, {"a | b", "", @@ -472,9 +460,9 @@ std::vector test_cases = { "ERROR: :1:5: Syntax error: extraneous input 'b' expecting \n" " | a | b\n" " | ....^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:3: unexpected single '|', expected '||'\n" + "ERROR: :1:3: Syntax error: unexpected single '|', expected '||'\n" " | a | b\n" " | ..^"}, {"?", "", @@ -485,9 +473,9 @@ std::vector test_cases = { "{'[', '{', '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, " "NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n | ?\n | .^\n" "ERROR: :4294967295:0: <> parsetree", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:1: unexpected token\n" + "ERROR: :1:1: Syntax error: unexpected token\n" " | ?\n" " | ^"}, {"t{>C}", "", @@ -496,9 +484,9 @@ std::vector test_cases = { ":1:5: " "Syntax error: " "mismatched input '}' expecting ':'\n | t{>C}\n | ....^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:3: expected struct field name\n" + "ERROR: :1:3: Syntax error: expected struct field name\n" " | t{>C}\n" " | ..^"}, {"foo(a,b,)", "", @@ -507,7 +495,7 @@ std::vector test_cases = { "NUM_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | foo(a,b,)\n" " | ........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:9: unexpected token\n" " | foo(a,b,)\n" @@ -657,12 +645,6 @@ std::vector test_cases = { "{\n" " 1^#3:int64#:2u^#4:uint64#^#2:Expr.CreateStruct.Entry#,\n" " 2^#6:int64#:3u^#7:uint64#^#5:Expr.CreateStruct.Entry#\n" - "}^#1:Expr.CreateStruct#", - "", "", "", "", - // PRATT PARSER AST - "{\n" - " 1^#2:int64#:2u^#4:uint64#^#3:Expr.CreateStruct.Entry#,\n" - " 2^#5:int64#:3u^#7:uint64#^#6:Expr.CreateStruct.Entry#\n" "}^#1:Expr.CreateStruct#"}, {"TestAllTypes{single_int32: 1, single_int64: 2}", "TestAllTypes{\n" @@ -673,9 +655,9 @@ std::vector test_cases = { "ERROR: :1:15: Syntax error: mismatched input '{' expecting \n" " | TestAllTypes(){single_int32: 1, single_int64: 2}\n" " | ..............^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:15: unexpected token after expression\n" + "ERROR: :1:15: Syntax error: unexpected token after expression\n" " | TestAllTypes(){single_int32: 1, single_int64: 2}\n" " | ..............^"}, {"size(x) == x.size()", @@ -695,9 +677,9 @@ std::vector test_cases = { "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | 1 + $\n" " | .....^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:5: unexpected character\n" + "ERROR: :1:5: Syntax error: unexpected character\n" " | 1 + $\n" " | ....^"}, {"1 + 2\n" @@ -706,9 +688,9 @@ std::vector test_cases = { "ERROR: :2:1: Syntax error: mismatched input '3' expecting \n" " | 3 +\n" " | ^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :2:1: unexpected token after expression\n" + "ERROR: :2:1: Syntax error: unexpected token after expression\n" " | 3 +\n" " | ^"}, {"\"\\\"\"", "\"\\\"\"^#1:string#"}, @@ -729,7 +711,7 @@ std::vector test_cases = { "ERROR: :1:9: all() variable name must be a simple identifier\n" " | [].all(.x, x)\n" " | ........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:8: all() variable name must be a simple identifier\n" " | [].all(.x, x)\n" @@ -738,7 +720,7 @@ std::vector test_cases = { "ERROR: :1:12: exists() variable name must be a simple identifier\n" " | [].exists(.x, x)\n" " | ...........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:11: exists() variable name must be a simple identifier\n" " | [].exists(.x, x)\n" @@ -748,7 +730,7 @@ std::vector test_cases = { "identifier\n" " | [].exists_one(.x, x)\n" " | ...............^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:15: exists_one() variable name must be a simple " "identifier\n" @@ -758,7 +740,7 @@ std::vector test_cases = { "ERROR: :1:9: map() variable name must be a simple identifier\n" " | [].map(.x, x, x)\n" " | ........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:8: map() variable name must be a simple identifier\n" " | [].map(.x, x, x)\n" @@ -767,7 +749,7 @@ std::vector test_cases = { "ERROR: :1:12: filter() variable name must be a simple identifier\n" " | [].filter(.x, x)\n" " | ...........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:11: filter() variable name must be a simple identifier\n" " | [].filter(.x, x)\n" @@ -827,11 +809,6 @@ std::vector test_cases = { {"---a", "-_(\n" " a^#2:Expr.Ident#\n" - ")^#1:Expr.Call#", - "", "", "", "", - // PRATT PARSER AST - "-_(\n" - " a^#4:Expr.Ident#\n" ")^#1:Expr.Call#"}, {"1 + +", "", "ERROR: :1:5: Syntax error: mismatched input '+' expecting {'[', " @@ -847,9 +824,9 @@ std::vector test_cases = { "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | 1 + +\n" " | .....^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:5: unexpected token\n" + "ERROR: :1:5: Syntax error: unexpected token\n" " | 1 + +\n" " | ....^"}, {"\"abc\" + \"def\"", @@ -862,9 +839,9 @@ std::vector test_cases = { "'.\"a\"'\n" " | {\"a\": 1}.\"a\"\n" " | .........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:10: expected identifier after '.'\n" + "ERROR: :1:10: Syntax error: expected identifier after '.'\n" " | {\"a\": 1}.\"a\"\n" " | .........^"}, {"\"\\xC3\\XBF\"", "\"ÿ\"^#1:string#"}, @@ -886,7 +863,7 @@ std::vector test_cases = { "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | \"\\xFh\"\n" " | ......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: Invalid string literal: Illegal escape sequence: Hex " "escape must be followed by 2 hex digits but saw: \\xFh\n" @@ -906,7 +883,7 @@ std::vector test_cases = { "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | \"\\a\\b\\f\\n\\r\\t\\v\\'\\\"\\\\\\? Illegal escape \\>\"\n" " | ..........................................^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: Invalid string literal: Illegal escape sequence: " "\\>\n" @@ -962,12 +939,12 @@ std::vector test_cases = { "ERROR: :2:11: Syntax error: no viable alternative at input '.'\n" " | && in.😁\n" " | .........．^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :2:7: unexpected token\n" + "ERROR: :2:7: Syntax error: unexpected token\n" " | && in.😁\n" " | ......^\n" - "ERROR: :2:10: unexpected character\n" + "ERROR: :2:10: Syntax error: unexpected character\n" " | && in.😁\n" " | .........＾"}, {"as", "", @@ -1018,9 +995,9 @@ std::vector test_cases = { "NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | in\n" " | ..^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :1:1: unexpected token\n" + "ERROR: :1:1: Syntax error: unexpected token\n" " | in\n" " | ^"}, {"let", "", @@ -1068,7 +1045,7 @@ std::vector test_cases = { "ERROR: :1:26: reserved identifier: var\n" " | [1, 2, 3].map(var, var * var)\n" " | .........................^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:15: reserved identifier: var\n" " | [1, 2, 3].map(var, var * var)\n" @@ -1106,9 +1083,9 @@ std::vector test_cases = { "{']', ','}\n" " | \r\n" " | ..^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE - "ERROR: :6:3: expected ']'\n" + "ERROR: :6:3: Syntax error: expected ']'\n" " | \r\n" " | ..^"}, @@ -1152,7 +1129,7 @@ std::vector test_cases = { "ERROR: :1:7: Syntax error: token recognition error at: '`'\n" " | a.`b c`\n" " | ......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:3: unexpected quoted identifier\n" " | a.`b c`\n" @@ -1164,7 +1141,7 @@ std::vector test_cases = { "ERROR: :1:8: Syntax error: token recognition error at: '`'\n" " | a.`@foo`\n" " | .......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:3: unexpected quoted identifier\n" " | a.`@foo`\n" @@ -1176,7 +1153,7 @@ std::vector test_cases = { "ERROR: :1:8: Syntax error: token recognition error at: '`'\n" " | a.`$foo`\n" " | .......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:3: unexpected quoted identifier\n" " | a.`$foo`\n" @@ -1189,7 +1166,7 @@ std::vector test_cases = { "BYTES, IDENTIFIER}\n" " | `a.b`\n" " | ^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: unexpected quoted identifier\n" " | `a.b`\n" @@ -1205,7 +1182,7 @@ std::vector test_cases = { "_INT, NUM_UINT, STRING, BYTES, IDENTIFIER}\n" " | `a.b`()\n" " | ......^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:1: unexpected quoted identifier\n" " | `a.b`()\n" @@ -1214,7 +1191,7 @@ std::vector test_cases = { "ERROR: :1:10: Syntax error: mismatched input '(' expecting \n" " | foo.`a.b`()\n" " | .........^", - "", "", "", "", + "", "", "", // PRATT PARSER ERROR MESSAGE "ERROR: :1:5: unexpected quoted identifier\n" " | foo.`a.b`()\n" @@ -1271,8 +1248,7 @@ std::vector test_cases = { " @result^#24:Expr.Ident#\n" " )^#25:Expr.Call#,\n" " // Result\n" - " @result^#26:Expr.Ident#)^#27:Expr.Comprehension#" - "", + " @result^#26:Expr.Ident#)^#27:Expr.Comprehension#", "", "", "", "x^#1:Expr.Ident#.filter(\n" " y^#3:Expr.Ident#,\n" @@ -1494,12 +1470,7 @@ std::vector test_cases = { {"{?'key': value}", "{\n " "?\"key\"^#3:string#:value^#4:Expr.Ident#^#2:Expr.CreateStruct.Entry#\n}^#" - "1:Expr.CreateStruct#", - "", "", "", "", - // PRATT PARSER AST - "{\n" - " ?\"key\"^#2:string#:value^#4:Expr.Ident#^#3:Expr.CreateStruct.Entry#\n" - "}^#1:Expr.CreateStruct#"}, + "1:Expr.CreateStruct#"}, {"[?a, ?b]", "[\n ?a^#2:Expr.Ident#,\n ?b^#3:Expr.Ident#\n]^#1:Expr.CreateList#"}, {"[?a[?b]]", @@ -1738,13 +1709,8 @@ TEST_P(ExpressionTest, Parse) { KindAndIdAdorner kind_and_id_adorner; ExprPrinter w(kind_and_id_adorner); std::string adorned_string = w.PrintProto(result->parsed_expr().expr()); - if (options_.enable_pratt_parser && !test_info.P_PRATT.empty()) { - EXPECT_EQ(test_info.P_PRATT, adorned_string) - << result->parsed_expr().ShortDebugString(); - } else { - EXPECT_EQ(test_info.P, adorned_string) - << result->parsed_expr().ShortDebugString(); - } + EXPECT_EQ(test_info.P, adorned_string) + << result->parsed_expr().ShortDebugString(); } if (!options_.enable_pratt_parser && !test_info.L.empty()) {