diff --git a/clients/vscode-hlasmplugin/CHANGELOG.md b/clients/vscode-hlasmplugin/CHANGELOG.md index 1469ccf53..be6635301 100644 --- a/clients/vscode-hlasmplugin/CHANGELOG.md +++ b/clients/vscode-hlasmplugin/CHANGELOG.md @@ -15,6 +15,7 @@ - Reaching ACTR limit now only generates warnings - Parsing of negative numbers in machine expressions - Empty arrays now behave similarly to other subscripted variables in macro tracer +- Forward structured macro parameters correctly ## [1.1.0](https://github.com/eclipse/che-che4z-lsp-for-hlasm/compare/1.0.0...1.1.0) (2022-03-29) diff --git a/clients/vscode-hlasmplugin/proc_grps_schema b/clients/vscode-hlasmplugin/proc_grps_schema index e3244ad25..b1a5c19f0 100644 --- a/clients/vscode-hlasmplugin/proc_grps_schema +++ b/clients/vscode-hlasmplugin/proc_grps_schema @@ -48,50 +48,50 @@ "asm_options": { "type": "object", "description": "List of assembler options", - "properties": { - "OPTABLE": { - "type": "string", - "description": "Specifies the instruction set to use.", - "enum": [ - "UNI", - "DOS", - "370", - "XA", - "ESA", - "ZOP", - "ZS1", - "YOP", - "ZS2", - "Z9", - "ZS3", - "Z10", - "ZS4", - "Z11", - "ZS5", - "Z12", - "ZS6", - "Z13", - "ZS7", - "Z14", - "ZS8", - "Z15", - "ZS9" - ] - }, - "SYSPARM": { - "type": "string", - "description": "Specifies the character string the assembler assigns to the &SYSPARM system variable symbol.", - "maxLength": 255 - }, - "PROFILE": { - "type": "string", - "description": "Profile Member to be copied into the source program." - }, - "SYSTEM_ID": { - "type": "string", - "description": "Provides the value for the SYSTEM_ID system variable. Defaults to 'z/OS 02.04.00' when omitted." + "properties": { + "OPTABLE": { + "type": "string", + "description": "Specifies the instruction set to use.", + "enum": [ + "UNI", + "DOS", + "370", + "XA", + "ESA", + "ZOP", + "ZS1", + "YOP", + "ZS2", + "Z9", + "ZS3", + "Z10", + "ZS4", + "Z11", + "ZS5", + "Z12", + "ZS6", + "Z13", + "ZS7", + "Z14", + "ZS8", + "Z15", + "ZS9" + ] + }, + "SYSPARM": { + "type": "string", + "description": "Specifies the character string the assembler assigns to the &SYSPARM system variable symbol.", + "maxLength": 255 + }, + "PROFILE": { + "type": "string", + "description": "Profile Member to be copied into the source program." + }, + "SYSTEM_ID": { + "type": "string", + "description": "Provides the value for the SYSTEM_ID system variable. Defaults to 'z/OS 02.04.00' when omitted." + } } - } }, "preprocessor": { "description": "Defines optional preprocessor pass for open code.", diff --git a/parser_library/src/debugging/set_symbol_variable.cpp b/parser_library/src/debugging/set_symbol_variable.cpp index 6de6e273e..f46cf96fc 100644 --- a/parser_library/src/debugging/set_symbol_variable.cpp +++ b/parser_library/src/debugging/set_symbol_variable.cpp @@ -93,7 +93,6 @@ std::string set_symbol_variable::get_string_array_value() const array_value.append(get_string_value(static_cast(key))); array_value.append(","); } - array_value.back() = ')'; return array_value; diff --git a/parser_library/src/parsing/error_strategy.h b/parser_library/src/parsing/error_strategy.h index e526e3b57..75c821087 100644 --- a/parser_library/src/parsing/error_strategy.h +++ b/parser_library/src/parsing/error_strategy.h @@ -29,12 +29,19 @@ enum tokens // copied from antlr4::DefaultErrorStrategy. class error_strategy final : public antlr4::DefaultErrorStrategy { + bool m_error_reported = false; + void reset(antlr4::Parser* recognizer) override + { + m_error_reported = false; + antlr4::DefaultErrorStrategy::reset(recognizer); + } void reportError(antlr4::Parser* recognizer, const antlr4::RecognitionException& e) override { - if (inErrorRecoveryMode(recognizer)) + if (m_error_reported && inErrorRecoveryMode(recognizer)) { return; // don't report spurious errors } + m_error_reported = true; // recovery strategy antlr4::misc::IntervalSet endTokens; @@ -69,6 +76,9 @@ class error_strategy final : public antlr4::DefaultErrorStrategy antlr4::Token* singleTokenDeletion(antlr4::Parser*) override { return nullptr; } bool singleTokenInsertion(antlr4::Parser*) override { return false; } + +public: + bool error_reported() const { return m_error_reported; } }; } // namespace hlasm_plugin::parser_library::parsing diff --git a/parser_library/src/parsing/grammar/deferred_operand_rules.g4 b/parser_library/src/parsing/grammar/deferred_operand_rules.g4 index fa826fa86..e4178c138 100644 --- a/parser_library/src/parsing/grammar/deferred_operand_rules.g4 +++ b/parser_library/src/parsing/grammar/deferred_operand_rules.g4 @@ -39,9 +39,11 @@ deferred_entry returns [std::vector vs] | ( {std::string name;} - AMPERSAND (ORDSYMBOL {name += $ORDSYMBOL->getText();})+ + AMPERSAND + ORDSYMBOL {name += $ORDSYMBOL->getText();} + (ORDSYMBOL {name += $ORDSYMBOL->getText();}|NUM {name += $NUM->getText();})* { - auto r = provider.get_range($AMPERSAND,$ORDSYMBOL); + auto r = provider.get_range($AMPERSAND,_input->LT(-1)); $vs.push_back(std::make_unique(hlasm_ctx->ids().add(std::move(name)), std::vector(), r)); collector.add_hl_symbol(token_info(r,hl_scopes::var_symbol)); } @@ -66,9 +68,14 @@ deferred_entry returns [std::vector vs] { name += $ORDSYMBOL->getText(); } + | + NUM + { + name += $NUM->getText(); + } )* { - auto r = provider.get_range($AMPERSAND,$ORDSYMBOL); + auto r = provider.get_range($AMPERSAND,_input->LT(-1)); $vs.push_back(std::make_unique(hlasm_ctx->ids().add(std::move(name)), std::vector(), r)); collector.add_hl_symbol(token_info(r,hl_scopes::var_symbol)); } @@ -81,19 +88,6 @@ deferred_entry returns [std::vector vs] finally {enable_ca_string();} -def_string_body - : string_ch_v - | IGNORED - | CONTINUATION; - -def_string returns [concat_chain chain] - : ap1=APOSTROPHE def_string_body*? ap2=(APOSTROPHE|ATTR) - { - collector.add_hl_symbol(token_info(provider.get_range($ap1,$ap2),hl_scopes::string)); - }; - finally - {concatenation_point::clear_concat_chain($chain);} - deferred_op_rem returns [remark_list remarks, std::vector var_list] : ( diff --git a/parser_library/src/parsing/grammar/hlasmparser.g4 b/parser_library/src/parsing/grammar/hlasmparser.g4 index 19b98a5dd..c6fbc6210 100644 --- a/parser_library/src/parsing/grammar/hlasmparser.g4 +++ b/parser_library/src/parsing/grammar/hlasmparser.g4 @@ -204,15 +204,32 @@ id_no_dot returns [id_index name = id_storage::empty_id] locals [std::string buf } ; -remark_ch: DOT|ASTERISK|MINUS|PLUS|LT|GT|COMMA|LPAR|RPAR|SLASH|EQUALS|AMPERSAND|APOSTROPHE|IDENTIFIER|NUM|VERTICAL|ORDSYMBOL|SPACE|ATTR; - remark - : remark_ch*; + : (DOT|ASTERISK|MINUS|PLUS|LT|GT|COMMA|LPAR|RPAR|SLASH|EQUALS|AMPERSAND|APOSTROPHE|IDENTIFIER|NUM|VERTICAL|ORDSYMBOL|SPACE|ATTR)*; + +remark_non_empty + : (DOT|ASTERISK|MINUS|PLUS|LT|GT|COMMA|LPAR|RPAR|SLASH|EQUALS|AMPERSAND|APOSTROPHE|IDENTIFIER|NUM|VERTICAL|ORDSYMBOL|SPACE|ATTR)+; remark_o returns [std::optional value] : SPACE remark {$value = provider.get_range( $remark.ctx);} | ; +remark_eol returns [std::optional value] + : + ( + SPACE + { + auto s = _input->LT(1); + } + l=(DOT|ASTERISK|MINUS|PLUS|LT|GT|COMMA|LPAR|RPAR|SLASH|EQUALS|AMPERSAND|APOSTROPHE|IDENTIFIER|NUM|VERTICAL|ORDSYMBOL|SPACE|ATTR)* + {$value = provider.get_range(s, $l);} + )? + ( + CONTINUATION + | + EOF + ) + ; diff --git a/parser_library/src/parsing/grammar/label_field_rules.g4 b/parser_library/src/parsing/grammar/label_field_rules.g4 index 0a118e60d..25fc158db 100644 --- a/parser_library/src/parsing/grammar/label_field_rules.g4 +++ b/parser_library/src/parsing/grammar/label_field_rules.g4 @@ -138,7 +138,6 @@ common_ch returns [std::string value] l_ch returns [std::string value] : common_ch {$value = std::move($common_ch.value);} - | EQUALS {$value = "=";} | COMMA {$value = ",";} | LPAR {$value = "(";} | RPAR {$value = ")";}; @@ -151,17 +150,21 @@ common_ch_v returns [concat_point_ptr point] | GT {$point = std::make_unique(">", provider.get_range($GT));} | SLASH {$point = std::make_unique("/", provider.get_range($SLASH));} | EQUALS {$point = std::make_unique();} - | l=AMPERSAND r=AMPERSAND {$point = std::make_unique("&&", provider.get_range($l,$r));} | VERTICAL {$point = std::make_unique("|", provider.get_range($VERTICAL));} | IDENTIFIER {$point = std::make_unique($IDENTIFIER->getText(), provider.get_range($IDENTIFIER));} | NUM {$point = std::make_unique($NUM->getText(), provider.get_range($NUM));} | ORDSYMBOL {$point = std::make_unique($ORDSYMBOL->getText(), provider.get_range($ORDSYMBOL));} | DOT {$point = std::make_unique();} - | var_symbol {$point = std::make_unique(std::move($var_symbol.vs));}; + | + ( + l=AMPERSAND r=AMPERSAND {$point = std::make_unique("&&", provider.get_range($l,$r));} + | + var_symbol {$point = std::make_unique(std::move($var_symbol.vs));} + ) + ; l_ch_v returns [concat_point_ptr point] : common_ch_v {$point = std::move($common_ch_v.point);} - | EQUALS {$point = std::make_unique("=", provider.get_range($EQUALS));} | COMMA {$point = std::make_unique(",", provider.get_range($COMMA));} | LPAR {$point = std::make_unique("(", provider.get_range($LPAR));} | RPAR {$point = std::make_unique(")", provider.get_range($RPAR));}; diff --git a/parser_library/src/parsing/grammar/macro_operand_rules.g4 b/parser_library/src/parsing/grammar/macro_operand_rules.g4 index f80a9cc84..1b0f5649b 100644 --- a/parser_library/src/parsing/grammar/macro_operand_rules.g4 +++ b/parser_library/src/parsing/grammar/macro_operand_rules.g4 @@ -16,14 +16,14 @@ parser grammar macro_operand_rules; -mac_op returns [operand_ptr op] - : mac_preproc +mac_op[int* paren_count] returns [operand_ptr op] + : mac_preproc[$paren_count] { $op = std::make_unique($mac_preproc.ctx->getText(),provider.get_range($mac_preproc.ctx)); }; mac_op_o returns [operand_ptr op] - : mac_entry? + : mac_entry[true]? { if($mac_entry.ctx) $op = std::make_unique(std::move($mac_entry.chain),provider.get_range($mac_entry.ctx)); @@ -34,114 +34,187 @@ mac_op_o returns [operand_ptr op] macro_ops returns [operand_list list] : mac_op_o {$list.push_back(std::move($mac_op_o.op));} (comma mac_op_o {$list.push_back(std::move($mac_op_o.op));})* EOF; - - -mac_preproc: mac_preproc_c+; - -mac_preproc_c returns [vs_ptr vs] - : asterisk - | minus - | plus - | LT - | GT - | slash - | equals - | VERTICAL - | IDENTIFIER {collector.add_hl_symbol(token_info(provider.get_range($IDENTIFIER), hl_scopes::operand));} - | NUM {collector.add_hl_symbol(token_info(provider.get_range($NUM), hl_scopes::operand));} - | ORDSYMBOL {collector.add_hl_symbol(token_info(provider.get_range($ORDSYMBOL), hl_scopes::operand));} - | dot - | AMPERSAND ORDSYMBOL - { - auto name = $ORDSYMBOL->getText(); - } +mac_preproc[int* paren_count] + : + ( + ASTERISK + | MINUS + | PLUS + | LT + | GT + | SLASH + | EQUALS + | VERTICAL + | IDENTIFIER + | NUM + | ORDSYMBOL + | DOT + | AMPERSAND + ( + ORDSYMBOL (ORDSYMBOL|NUM)* + | + LPAR {++*$paren_count;} + | + AMPERSAND + ) + | + LPAR {++*$paren_count;} + | + RPAR {--*$paren_count;} + | + APOSTROPHE + (~(APOSTROPHE|ATTR|CONTINUATION))* + (APOSTROPHE|ATTR) + | + ATTR + ( + {!is_previous_attribute_consuming(*$paren_count == 0, _input->LT(-2))}? + (~(APOSTROPHE|ATTR|CONTINUATION))* + (APOSTROPHE|ATTR) + )? + )+ + ; + +mac_entry [bool top_level = true] returns [concat_chain chain] + : ( - CONTINUATION IGNORED* ORDSYMBOL + asterisk {$chain.push_back(std::make_unique("*", provider.get_range($asterisk.ctx)));} + | minus {$chain.push_back(std::make_unique("-", provider.get_range($minus.ctx)));} + | plus {$chain.push_back(std::make_unique("+", provider.get_range($plus.ctx)));} + | LT {$chain.push_back(std::make_unique("<", provider.get_range($LT)));} + | GT {$chain.push_back(std::make_unique(">", provider.get_range($GT)));} + | slash {$chain.push_back(std::make_unique("/", provider.get_range($slash.ctx)));} + | equals {$chain.push_back(std::make_unique());} + | VERTICAL {$chain.push_back(std::make_unique("|", provider.get_range($VERTICAL)));} + | IDENTIFIER { - name += $ORDSYMBOL->getText(); + auto r = provider.get_range($IDENTIFIER); + $chain.push_back(std::make_unique($IDENTIFIER.text, r)); + collector.add_hl_symbol(token_info(r, hl_scopes::operand)); } - )* - { - auto r = provider.get_range($AMPERSAND,$ORDSYMBOL); - $vs = std::make_unique(hlasm_ctx->ids().add(std::move(name)), std::vector(), r); - collector.add_hl_symbol(token_info(r,hl_scopes::var_symbol)); - } - | AMPERSAND LPAR - | AMPERSAND AMPERSAND - | lpar - | rpar - | attr - | def_string; - -mac_str_ch returns [concat_point_ptr point] - : common_ch_v {$point = std::move($common_ch_v.point);} - | SPACE {$point = std::make_unique($SPACE->getText(), provider.get_range($SPACE));}//here next are for deferred - | LPAR {$point = std::make_unique("(", provider.get_range($LPAR));} - | RPAR {$point = std::make_unique(")", provider.get_range($RPAR));} - | COMMA {$point = std::make_unique(",", provider.get_range($COMMA));}; - -mac_str_b returns [concat_chain chain] - : - | tmp=mac_str_b mac_str_ch {$tmp.chain.push_back(std::move($mac_str_ch.point)); $chain = std::move($tmp.chain);}; - -mac_str returns [concat_chain chain] - : ap1=APOSTROPHE mac_str_b ap2=(APOSTROPHE|ATTR) - { - $chain.push_back(std::make_unique("'", provider.get_range($ap1))); - $chain.insert($chain.end(), std::make_move_iterator($mac_str_b.chain.begin()), std::make_move_iterator($mac_str_b.chain.end())); - $chain.push_back(std::make_unique("'", provider.get_range($ap2))); - collector.add_hl_symbol(token_info(provider.get_range($ap1,$ap2),hl_scopes::string)); - }; - -mac_ch returns [concat_chain chain] - : common_ch_v - { - $chain.push_back(std::move($common_ch_v.point)); - auto token = $common_ch_v.ctx->getStart(); - if (token->getType() == lexing::lexer::Tokens::ORDSYMBOL && $common_ch_v.ctx->getStop()->getType() == lexing::lexer::Tokens::ORDSYMBOL) - collector.add_hl_symbol(token_info(provider.get_range( $common_ch_v.ctx), hl_scopes::operand)); - } - | ATTR {$chain.push_back(std::make_unique("'", provider.get_range($ATTR)));} - | mac_str {$chain = std::move($mac_str.chain);} - | mac_sublist {$chain.push_back(std::move($mac_sublist.point));}; - -mac_ch_c returns [concat_chain chain] - : - | tmp=mac_ch_c mac_ch - { - $chain = std::move($tmp.chain); - $chain.insert($chain.end(), std::make_move_iterator($mac_ch.chain.begin()), std::make_move_iterator($mac_ch.chain.end())); - }; - -mac_entry returns [concat_chain chain] - : mac_ch {$chain = std::move($mac_ch.chain);} - | literal {$chain.push_back(std::make_unique($literal.ctx->getText(), provider.get_range($literal.ctx)));} - | ORDSYMBOL EQUALS literal - { - collector.add_hl_symbol(token_info(provider.get_range($ORDSYMBOL), hl_scopes::operand)); - $chain.push_back(std::make_unique($ORDSYMBOL->getText(), provider.get_range($ORDSYMBOL))); - $chain.push_back(std::make_unique("=", provider.get_range($EQUALS))); - $chain.push_back(std::make_unique($literal.ctx->getText(), provider.get_range($literal.ctx))); - } - | tmp=mac_entry mac_ch - { - $chain = std::move($tmp.chain); - $chain.insert($chain.end(), std::make_move_iterator($mac_ch.chain.begin()), std::make_move_iterator($mac_ch.chain.end())); - }; - finally - {concatenation_point::clear_concat_chain($chain);} - -mac_sublist_b_c returns [concat_chain chain] - : mac_ch_c {$chain = std::move($mac_ch_c.chain);} - | literal {$chain.push_back(std::make_unique($literal.ctx->getText(), provider.get_range($literal.ctx)));}; - -mac_sublist_b returns [std::vector chains] - : mac_sublist_b_c {$chains.push_back(std::move($mac_sublist_b_c.chain));} - | tmp=mac_sublist_b comma mac_sublist_b_c - { - $chains = std::move($tmp.chains); - $chains.push_back(std::move($mac_sublist_b_c.chain)); - }; - -mac_sublist returns [concat_point_ptr point] - : lpar mac_sublist_b rpar { $point = std::make_unique(std::move($mac_sublist_b.chains)); }; \ No newline at end of file + | NUM + { + auto r = provider.get_range($NUM); + $chain.push_back(std::make_unique($NUM.text, r)); + collector.add_hl_symbol(token_info(r, hl_scopes::operand)); + } + | ORDSYMBOL + { + auto r = provider.get_range($ORDSYMBOL); + $chain.push_back(std::make_unique($ORDSYMBOL.text, r)); + collector.add_hl_symbol(token_info(r, hl_scopes::operand)); + } + | dot {$chain.push_back(std::make_unique());} + | AMPERSAND + ( + vs_id tmp=subscript + { + auto id = $vs_id.name; + auto r = provider.get_range( $AMPERSAND,$tmp.ctx->getStop()); + $chain.push_back(std::make_unique(std::make_unique(id, std::move($tmp.value), r))); + collector.add_hl_symbol(token_info(provider.get_range($AMPERSAND, $vs_id.ctx->getStop()),hl_scopes::var_symbol)); + } + | + lpar (clc=created_set_body)? rpar subscript + { + $chain.push_back(std::make_unique(std::make_unique($clc.ctx ? std::move($clc.concat_list) : concat_chain{},std::move($subscript.value),provider.get_range($AMPERSAND,$subscript.ctx->getStop())))); + collector.add_hl_symbol(token_info(provider.get_range($AMPERSAND),hl_scopes::var_symbol)); + } + | + AMPERSAND + { + $chain.push_back(std::make_unique("&&", provider.get_range(_input->LT(-2),_input->LT(-1)))); + } + ) + | + lpar + { + std::vector sublist; + bool pending_empty = false; + } + ( + comma + { + sublist.emplace_back(); + pending_empty = true; + } + )* + ( + mac_entry[false] + { + sublist.push_back(std::move($mac_entry.chain)); + pending_empty = false; + } + ( + comma + ( + comma + { + sublist.emplace_back(); + } + )* + { + pending_empty = true; + } + ( + mac_entry[false] + { + sublist.push_back(std::move($mac_entry.chain)); + pending_empty = false; + } + )? + )* + )? + { + if (pending_empty) + { + sublist.emplace_back(); + } + $chain.push_back(std::make_unique(std::move(sublist))); + } + rpar + | + ap1=APOSTROPHE + { + $chain.push_back(std::make_unique("'", provider.get_range($ap1))); + } + ( + string_ch_v + { + if (auto& p = $string_ch_v.point; p) + $chain.push_back(std::move(p)); + } + )*? + ap2=(APOSTROPHE|ATTR) + { + $chain.push_back(std::make_unique("'", provider.get_range($ap2))); + collector.add_hl_symbol(token_info(provider.get_range($ap1,$ap2),hl_scopes::string)); + } + | + ap1=ATTR + { + $chain.push_back(std::make_unique("'", provider.get_range($ap1))); + bool attribute = true; + } + ( + {!is_previous_attribute_consuming($top_level, _input->LT(-2))}? + ( + string_ch_v + { + if (auto& p = $string_ch_v.point; p) + $chain.push_back(std::move(p)); + } + )*? + ap2=(APOSTROPHE|ATTR) + { + $chain.push_back(std::make_unique("'", provider.get_range($ap2))); + collector.add_hl_symbol(token_info(provider.get_range($ap1,$ap2),hl_scopes::string)); + attribute = false; + } + )? + { + if (attribute) + collector.add_hl_symbol(token_info(provider.get_range($ap1),hl_scopes::operator_symbol)); + } + )+ + ; diff --git a/parser_library/src/parsing/grammar/operand_field_rules.g4 b/parser_library/src/parsing/grammar/operand_field_rules.g4 index 8ccca528c..b3530ee33 100644 --- a/parser_library/src/parsing/grammar/operand_field_rules.g4 +++ b/parser_library/src/parsing/grammar/operand_field_rules.g4 @@ -191,23 +191,20 @@ op_rem_body_mac returns [op_rem line, range line_range] { $line = std::move($op_rem_body_alt_mac.line); $line_range = provider.get_range($op_rem_body_alt_mac.ctx); - } EOF - | remark_o - { - $line.remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; - $line_range = provider.get_range($remark_o.ctx); } EOF; op_rem_body_alt_mac returns [op_rem line] : + { + int paren_count = 0; + } ( ( - mac_op? comma + mac_op[&paren_count]? comma { if ($mac_op.ctx && $mac_op.op) $line.operands.push_back(std::move($mac_op.op)); - else - $line.operands.push_back(std::make_unique(provider.get_empty_range($comma.ctx->getStart()))); + $line.operands.push_back(std::make_unique(provider.get_range($comma.ctx->getStart()))); } )+ {enable_continuation();} @@ -220,14 +217,16 @@ op_rem_body_alt_mac returns [op_rem line] {disable_continuation();} )* ( - last_mac_op=mac_op? last_remark=remark_o + last_mac_op=mac_op[&paren_count]? last_remark=remark_o { - if ($last_mac_op.ctx) + if ($last_mac_op.ctx && $last_mac_op.op) $line.operands.push_back(std::move($last_mac_op.op)); if ($last_remark.value) $line.remarks.push_back(std::move(*$last_remark.value)); } ); + finally + {disable_continuation();} ///////////// @@ -320,10 +319,6 @@ op_rem_body_mac_r returns [op_rem line] op_rem_body_alt_mac { $line = std::move($op_rem_body_alt_mac.line); - } EOF - | remark_o - { - $line.remarks = $remark_o.value ? remark_list{*$remark_o.value} : remark_list{}; } EOF; op_rem_body_noop_r diff --git a/parser_library/src/parsing/parser_impl.cpp b/parser_library/src/parsing/parser_impl.cpp index 5f940f1e3..43003a5d7 100644 --- a/parser_library/src/parsing/parser_impl.cpp +++ b/parser_library/src/parsing/parser_impl.cpp @@ -211,6 +211,21 @@ bool parser_impl::ALIAS() return opcode.type == instruction_type::ASM && opcode.value == hlasm_ctx->ids().well_known.ALIAS; } +bool parser_impl::is_previous_attribute_consuming(bool top_level, const antlr4::Token* token) +{ + if (!token) + return false; + + auto text = token->getText(); + if (text.empty() || text.size() >= 2 && std::isalpha((unsigned char)text[text.size() - 2])) + return false; + + auto tmp = std::toupper((unsigned char)text.back()); + + // this almost looks like a bug in the original assembler + return tmp == 'O' && top_level || tmp == 'S' || tmp == 'I' || tmp == 'L' || tmp == 'T'; +} + antlr4::misc::IntervalSet parser_impl::getExpectedTokens() { if (proc_status->first.kind == processing::processing_kind::LOOKAHEAD) diff --git a/parser_library/src/parsing/parser_impl.h b/parser_library/src/parsing/parser_impl.h index b16538b5f..241449ab1 100644 --- a/parser_library/src/parsing/parser_impl.h +++ b/parser_library/src/parsing/parser_impl.h @@ -149,6 +149,7 @@ class parser_impl : public antlr4::Parser bool ASM(); bool DAT(); bool ALIAS(); + static bool is_previous_attribute_consuming(bool top_level, const antlr4::Token* token); void add_diagnostic(diagnostic_severity severity, std::string code, std::string message, range diag_range) const; void add_diagnostic(diagnostic_op d) const; diff --git a/parser_library/src/processing/instruction_sets/macro_processor.cpp b/parser_library/src/processing/instruction_sets/macro_processor.cpp index 999fb19ad..704bbf78b 100644 --- a/parser_library/src/processing/instruction_sets/macro_processor.cpp +++ b/parser_library/src/processing/instruction_sets/macro_processor.cpp @@ -41,14 +41,16 @@ void macro_processor::process(std::shared_ptropcode_ref().value, std::move(args.name_param), std::move(args.symbolic_params)); } -bool is_data_def(unsigned char c) +bool is_consuming_data_def(unsigned char c) { c = (char)toupper(c); - return c == 'L' || c == 'I' || c == 'S' || c == 'T' || c == 'D' || c == 'O' || c == 'N' || c == 'K'; + return c == 'L' || c == 'I' || c == 'S' || c == 'T'; } -std::unique_ptr find_single_macro_param(const std::string& data, size_t& start) +std::pair, bool> find_single_macro_param( + const std::string& data, size_t& start) { + // always called in nested configuration size_t begin = start; while (true) @@ -56,7 +58,7 @@ std::unique_ptr find_single_macro_param(const start = data.find_first_of(",'()", start); if (start == std::string::npos) - return nullptr; + return { nullptr, false }; if (data[start] == '(') { @@ -65,7 +67,7 @@ std::unique_ptr find_single_macro_param(const { ++start; if (start == data.size()) - return nullptr; + return { nullptr, false }; if (data[start] == '(') ++nest; @@ -76,7 +78,7 @@ std::unique_ptr find_single_macro_param(const } else if (data[start] == '\'') { - if (start > 0 && is_data_def(data[start - 1])) + if (start > 0 && is_consuming_data_def(data[start - 1])) { ++start; continue; @@ -85,7 +87,7 @@ std::unique_ptr find_single_macro_param(const start = data.find_first_of('\'', start + 1); if (start == std::string::npos) - return nullptr; + return { nullptr, false }; ++start; } @@ -94,10 +96,14 @@ std::unique_ptr find_single_macro_param(const } auto tmp_start = start; - if (data[start] == ',') + bool comma_encountered = data[start] == ','; + if (comma_encountered) ++start; - return std::make_unique(data.substr(begin, tmp_start - begin)); + return { + std::make_unique(data.substr(begin, tmp_start - begin)), + comma_encountered, + }; } context::macro_data_ptr macro_processor::string_to_macrodata(std::string data) @@ -114,6 +120,7 @@ context::macro_data_ptr macro_processor::string_to_macrodata(std::string data) nests.push(0); macro_data.emplace(); + bool empty_op_pending = false; while (true) { @@ -126,6 +133,7 @@ context::macro_data_ptr macro_processor::string_to_macrodata(std::string data) { nests.push(begin + 1); macro_data.emplace(); + empty_op_pending = false; } else if (data[begin] == ')') { @@ -135,9 +143,13 @@ context::macro_data_ptr macro_processor::string_to_macrodata(std::string data) auto vec = std::move(macro_data.top()); macro_data.pop(); + if (std::exchange(empty_op_pending, false)) + vec.push_back(std::make_unique("")); + if (begin != data.size() && data[begin] != ',' && data[begin] != ')') { - auto tmp_single = find_single_macro_param(data, begin); + auto [tmp_single, comma] = find_single_macro_param(data, begin); + empty_op_pending = comma; if (tmp_single == nullptr) return std::make_unique(std::move(data)); @@ -158,8 +170,10 @@ context::macro_data_ptr macro_processor::string_to_macrodata(std::string data) } else { - macro_data.top().push_back(find_single_macro_param(data, begin)); + auto [op, comma] = find_single_macro_param(data, begin); + macro_data.top().push_back(std::move(op)); nests.top() = begin; + empty_op_pending = comma; if (macro_data.top().back() == nullptr) return std::make_unique(std::move(data)); diff --git a/parser_library/src/processing/opencode_provider.cpp b/parser_library/src/processing/opencode_provider.cpp index 50a0bd917..a36042a6d 100644 --- a/parser_library/src/processing/opencode_provider.cpp +++ b/parser_library/src/processing/opencode_provider.cpp @@ -283,7 +283,11 @@ std::shared_ptr opencode_provider::process_ordin auto line = std::move(rule->line); auto line_range = rule->line_range; - if (line.operands.size()) + if (h.error_handler->error_reported()) + { + line.operands.clear(); + } + else if (line.operands.size()) { auto [to_parse, ranges, r] = join_operands(line.operands); @@ -751,7 +755,6 @@ const parsing::parser_holder& opencode_provider::prepare_operand_parser(const st h.stream->reset(); - h.parser->reinitialize(&hlasm_ctx, std::move(range_prov), proc_status, diags); h.parser->reset(); diff --git a/parser_library/src/processing/statement_fields_parser.cpp b/parser_library/src/processing/statement_fields_parser.cpp index 45e1103de..fd74194f0 100644 --- a/parser_library/src/processing/statement_fields_parser.cpp +++ b/parser_library/src/processing/statement_fields_parser.cpp @@ -132,7 +132,11 @@ statement_fields_parser::parse_result statement_fields_parser::parse_operand_fie line = std::move(h.parser->op_rem_body_mac_r()->line); literals = h.parser->get_collector().take_literals(); - if (line.operands.size()) + if (h.error_handler->error_reported()) + { + line.operands.clear(); + } + else if (line.operands.size()) { auto [to_parse, ranges, r] = join_operands(line.operands); diff --git a/parser_library/src/semantics/operand_impls.cpp b/parser_library/src/semantics/operand_impls.cpp index 13449cc3c..84bfed57e 100644 --- a/parser_library/src/semantics/operand_impls.cpp +++ b/parser_library/src/semantics/operand_impls.cpp @@ -757,14 +757,12 @@ join_operands_result join_operands(const operand_list& operands) result.text.reserve(string_size); - bool insert_comma = false; for (const auto& op : operands) { - if (std::exchange(insert_comma, true)) - result.text.push_back(','); - if (auto m_op = dynamic_cast(op.get())) result.text.append(m_op->value); + else + result.text.push_back(','); result.ranges.push_back(op->operand_range); } diff --git a/parser_library/src/semantics/range_provider.cpp b/parser_library/src/semantics/range_provider.cpp index 183caddb7..34b4bf93b 100644 --- a/parser_library/src/semantics/range_provider.cpp +++ b/parser_library/src/semantics/range_provider.cpp @@ -55,7 +55,13 @@ range range_provider::get_empty_range(const antlr4::Token* start) const range range_provider::adjust_range(range r) const { if (state == adjusting_state::MACRO_REPARSE) - return range(adjust_position(r.start), adjust_position(r.end)); + { + if (r.start != r.end) + return range(adjust_position(r.start, false), adjust_position(r.end, true)); + + auto adjusted = adjust_position(r.end, true); + return range(adjusted, adjusted); + } else if (state == adjusting_state::SUBSTITUTION) return original_range; else if (state == adjusting_state::NONE) @@ -64,27 +70,20 @@ range range_provider::adjust_range(range r) const return r; } -position range_provider::adjust_position(position pos) const +position range_provider::adjust_position(position pos, bool end) const { - size_t idx = 1; - auto orig_range = original_operand_ranges.empty() ? original_range : original_operand_ranges.front(); + auto [orig_range, offset] = [this, pos, end]() { + constexpr static size_t continued_code_line_width = 72 - 16; - auto offset = pos.column - orig_range.start.column; - - if (original_operand_ranges.size()) - while (true) + for (auto offset = pos.column - original_range.start.column; const auto& r : original_operand_ranges) { - auto tmp_lines = orig_range.end.line - orig_range.start.line; - auto tmp_cols = (orig_range.end.column + 72 * tmp_lines) - orig_range.start.column; - - if (tmp_cols >= offset) - break; - else - { - offset -= tmp_cols + 1; - orig_range = original_operand_ranges[idx++]; - } + auto range_len = r.end.column - r.start.column + continued_code_line_width * (r.end.line - r.start.line); + if (offset < range_len + end) + return std::pair(r, offset); + offset -= range_len; } + return std::pair(original_range, pos.column - original_range.start.column); + }(); auto column_start = orig_range.start.column; auto line_start = orig_range.start.line; @@ -92,16 +91,16 @@ position range_provider::adjust_position(position pos) const while (true) { auto rest = 71 - column_start; - if (offset > rest) + if (offset < rest + end) { - offset -= rest; - column_start = 15; - ++line_start; + column_start += offset; + break; } else { - column_start += offset; - break; + offset -= rest; + column_start = 15; + ++line_start; } } return position(line_start, column_start); @@ -113,11 +112,13 @@ range_provider::range_provider(range original_range, adjusting_state state) {} range_provider::range_provider( - range original_field_range, std::vector original_operand_ranges, adjusting_state state) + range original_field_range, std::vector original_operand_ranges_, adjusting_state state) : original_range(original_field_range) - , original_operand_ranges(std::move(original_operand_ranges)) + , original_operand_ranges(std::move(original_operand_ranges_)) , state(state) -{} +{ + assert(original_operand_ranges.empty() || original_range.start == original_operand_ranges.front().start); +} range_provider::range_provider() : original_range() diff --git a/parser_library/src/semantics/range_provider.h b/parser_library/src/semantics/range_provider.h index 8e6c021a7..ad9085dd5 100644 --- a/parser_library/src/semantics/range_provider.h +++ b/parser_library/src/semantics/range_provider.h @@ -50,7 +50,7 @@ struct range_provider range adjust_range(range r) const; private: - position adjust_position(position pos) const; + position adjust_position(position pos, bool end) const; }; } // namespace hlasm_plugin::parser_library::semantics diff --git a/parser_library/test/context/macro_test.cpp b/parser_library/test/context/macro_test.cpp index 00db40959..320aadb9b 100644 --- a/parser_library/test/context/macro_test.cpp +++ b/parser_library/test/context/macro_test.cpp @@ -889,3 +889,245 @@ ALIAS OPSYN SAM64 EXPECT_TRUE(a.diags().empty()); } + +TEST(macro, operand_string_substitution) +{ + std::string input = R"( + MACRO + MAC &PAR + GBLC &VAR +&VAR SETC '&PAR' + MEND + + GBLC &VAR + MAC 'test string &SYSPARM' +)"; + analyzer a(input, analyzer_options(asm_option { .sysparm = "ABC" })); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()); + EXPECT_EQ(get_var_value(a.hlasm_ctx(), "VAR"), "'test string ABC'"); +} + +TEST(macro, operand_string_substitution_continuation) +{ + for (int i = 0; i < 256; ++i) + { + std::string input = R"( + MACRO + MAC &PAR + GBLC &VAR +&VAR SETC '&PAR' + MEND + + GBLC &VAR +&AAA(1) SETC 'ABC' +)"; + std::string suffix_ = " MAC " + std::string(i, ' ') + "'test string &(AAA)(1)'"; + std::string_view suffix = suffix_; + size_t limit = 71; + while (suffix.size() > limit) + { + input.append(suffix.substr(0, limit)).append("X\n "); + suffix.remove_prefix(limit); + limit = 71 - 15; + } + input += std::move(suffix); + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()) << i; + EXPECT_EQ(get_var_value(a.hlasm_ctx(), "VAR"), "'test string ABC'") << i; + } +} + +TEST(macro, operand_sublist_continuation) +{ + for (int i = 0; i < 256; ++i) + { + std::string input = R"( + MACRO + MAC + MEND +)"; + std::string suffix_ = + " MAC " + std::string(i, ' ') + "(" + std::string(i / 2, 'J') + "," + std::string(i / 2, 'K') + ")"; + std::string_view suffix = suffix_; + size_t limit = 71; + while (suffix.size() > limit) + { + input.append(suffix.substr(0, limit)).append("X\n "); + suffix.remove_prefix(limit); + limit = 71 - 15; + } + input += std::move(suffix); + analyzer a(input); + a.analyze(); + a.collect_diags(); + + ASSERT_TRUE(a.diags().empty()) << i; + } +} + +TEST(macro, pass_empty_operand_components) +{ + std::string input = R"( + MACRO + MAC2 &PAR= + GBLA &VAR +&VAR SETA N'&PAR + MEND + + MACRO + MAC &PAR= + MAC2 PAR=&PAR + MEND + + GBLA &VAR + MAC PAR=(E,) +)"; + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()); + EXPECT_EQ(get_var_value(a.hlasm_ctx(), "VAR"), 2); +} + +TEST(macro, multiline_comment) +{ + std::string input = R"( + MACRO + MAC &PAR= + MEND + + MAC PAR='test' comment X + comment +)"; + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()); +} + +TEST(macro, attribute_and_multiline_comment) +{ + std::string input = R"( + MACRO + MAC &F,&L +TEST EQU L'&F-&L + MEND +LABEL DS CL80 + + MAC LABEL,L'LABEL comment X + comment +)"; + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()); + EXPECT_EQ(get_symbol_abs(a.hlasm_ctx(), "TEST"), 0); +} + +TEST(macro, attribute_string_combo) +{ + std::string input = R"( + MACRO + MAC +TEST EQU &SYSLIST(1,2) + MEND + +LABEL DS A + + MAC (0,l'LABEL),'-',(l'LABEL,0), X + '-' +)"; + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()); + EXPECT_EQ(get_symbol_abs(a.hlasm_ctx(), "TEST"), 4); +} + +TEST(macro, empty_parms) +{ + std::string input = R"( + MACRO + MAC &PAR1,&PAR2,&PAR3 + MEND + + MAC ,A,B +)"; + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()); +} + +TEST(macro, empty_parms_after_continuation) +{ + std::string input = R"( + MACRO + MAC &PAR1,&PAR2,&PAR3 + GBLB &RESULT + AIF ('&PAR2' NE '').SKIP +&RESULT SETB 1 +.SKIP ANOP , + MEND + + GBLB &RESULT + + MAC A, X + ,B +)"; + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()); + EXPECT_EQ(get_var_value(a.hlasm_ctx(), "RESULT"), true); +} + +TEST(macro, syslist_size) +{ + using namespace std::string_literals; + + std::string input = R"( + MACRO +&L MAC +&A SETA N'&SYSLIST +&L EQU &A + MEND +T0 MAC +T1 MAC +T2 MAC A +T3 MAC A, +T4 MAC A, +T5 MAC ,A +T6 MAC ,A +T7 MAC , +T8 MAC , +T9 MAC ,, +T10 MAC ,, +T11 MAC ,, A + END +)"; + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()); + + auto expected = { 0, 0, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3 }; + + for (size_t i = 0; i < expected.size(); ++i) + { + EXPECT_EQ(get_symbol_abs(a.hlasm_ctx(), std::string("T") + std::to_string(i)), expected.begin()[i]) << i; + } +} diff --git a/parser_library/test/parsing/string_test.cpp b/parser_library/test/parsing/string_test.cpp index a03d6011d..721c3a39b 100644 --- a/parser_library/test/parsing/string_test.cpp +++ b/parser_library/test/parsing/string_test.cpp @@ -69,11 +69,7 @@ INSTANTIATE_TEST_SUITE_P(parser, test_param { "literal_FS", "=FS'SYM STH'", "=FS'SYM STH'" }, test_param { "number_before_attr_L", "=4L'SYM 93'", "=4L'SYM 93'" }, test_param { "quote_before_attr_L", "\"L'SYM 93'", "\"L'SYM" }, - test_param { "quote_before_attr_D", "\"D'SYM 93'", "\"D'SYM 93'" }, - - // Invalid inputs are currently parsed as remark, therefore the variable symbol is empty. - test_param { "no_ending_apostrophe", "\"N'SYM", "" }, - test_param { "no_ending_apostrophe_2", "\"L'SYM' STH", "" }), + test_param { "quote_before_attr_D", "\"D'SYM 93'", "\"D'SYM 93'" }), stringer()); } // namespace @@ -92,11 +88,39 @@ TEST_P(parser_string_fixture, basic) a.analyze(); a.collect_diags(); - EXPECT_EQ(a.diags().size(), 0U); + EXPECT_TRUE(a.diags().empty()); - auto par_value = get_var_value(a.hlasm_ctx(), "PAR"); - ASSERT_TRUE(par_value.has_value()); - EXPECT_EQ(*par_value, GetParam().expected); + EXPECT_EQ(get_var_value(a.hlasm_ctx(), "PAR"), GetParam().expected); +} + +TEST(parser, no_ending_apostrophe) +{ + std::string input = R"( + MACRO + MAC &VAR + MEND + + MAC "N'SYM)"; + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(matches_message_codes(a.diags(), { "S0005" })); +} + +TEST(parser, no_ending_apostrophe_2) +{ + std::string input = R"( + MACRO + MAC &VAR + MEND + + MAC "L'SYM' STH)"; + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(matches_message_codes(a.diags(), { "S0005" })); } @@ -113,10 +137,36 @@ TEST(parser, incomplete_string) MAC 'A 93)"; analyzer a(input); a.analyze(); - a.collect_diags(); - EXPECT_EQ(a.diags().size(), 0U); + + EXPECT_TRUE(matches_message_codes(a.diags(), { "S0005" })); auto par_value = get_var_value(a.hlasm_ctx(), "PAR"); ASSERT_TRUE(par_value.has_value()); } + +TEST(parser, preserve_structured_parameter) +{ + std::string input = R"( + GBLC &PAR + MACRO + MAC2 + GBLC &PAR +&PAR SETC '&SYSLIST(1,1)' + MEND + + MACRO + MAC &P1 + MAC2 &P1 + MEND + + + MAC (A,O'-9') +)"; + analyzer a(input); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()); + EXPECT_EQ(get_var_value(a.hlasm_ctx(), "PAR"), "A"); +} diff --git a/parser_library/test/processing/ainsert_test.cpp b/parser_library/test/processing/ainsert_test.cpp index 8c069b53c..9fe75ef4a 100644 --- a/parser_library/test/processing/ainsert_test.cpp +++ b/parser_library/test/processing/ainsert_test.cpp @@ -442,9 +442,11 @@ TEST(ainsert, grammar_non_matching_apostrophes_by_one) analyzer a(input); a.analyze(); a.collect_diags(); - EXPECT_EQ(a.diags().size(), (size_t)7); + EXPECT_TRUE(matches_message_codes(a.diags(), { + "S0002", + "S0002", "S0003", "S0003", "A011", diff --git a/parser_library/test/semantics/highlighting_test.cpp b/parser_library/test/semantics/highlighting_test.cpp index 0165fc002..222426b93 100644 --- a/parser_library/test/semantics/highlighting_test.cpp +++ b/parser_library/test/semantics/highlighting_test.cpp @@ -205,7 +205,7 @@ TEST(highlighting, macro_alternative_continuation) token_info({ { 3, 1 }, { 3, 5 } }, hl_scopes::instruction), token_info({ { 4, 1 }, { 4, 4 } }, hl_scopes::instruction), token_info({ { 4, 5 }, { 4, 8 } }, hl_scopes::operand), - token_info({ { 4, 8 }, { 4, 71 } }, hl_scopes::operator_symbol), + token_info({ { 4, 8 }, { 4, 9 } }, hl_scopes::operator_symbol), token_info({ { 4, 10 }, { 4, 71 } }, hl_scopes::remark), token_info({ { 4, 71 }, { 4, 72 } }, hl_scopes::continuation), token_info({ { 5, 0 }, { 5, 15 } }, hl_scopes::ignored), @@ -331,4 +331,64 @@ TEST(highlighting, single_chars) } EXPECT_EQ(tokens, expected); -} \ No newline at end of file +} + +TEST(highlighting, multiline_macro_param) +{ + const std::string contents = R"( + MACRO + MAC + MEND + + MAC (L1, comment X + L2, comment X + L3, comment X + L4) comment +)"; + analyzer a(contents, analyzer_options { collect_highlighting_info::yes }); + a.analyze(); + a.collect_diags(); + + EXPECT_TRUE(a.diags().empty()); + + auto tokens = a.source_processor().semantic_tokens(); + std::sort(tokens.begin(), tokens.end(), [](const auto& l, const auto& r) { + return position::min(l.token_range.start, r.token_range.start) != r.token_range.start; + }); + semantics::lines_info expected = { + token_info({ { 1, 8 }, { 1, 13 } }, hl_scopes::instruction), + token_info({ { 2, 8 }, { 2, 11 } }, hl_scopes::instruction), + token_info({ { 3, 8 }, { 3, 12 } }, hl_scopes::instruction), + + token_info({ { 5, 8 }, { 5, 11 } }, hl_scopes::instruction), + + token_info({ { 5, 14 }, { 5, 15 } }, hl_scopes::operator_symbol), + + token_info({ { 5, 15 }, { 5, 17 } }, hl_scopes::operand), + token_info({ { 5, 17 }, { 5, 18 } }, hl_scopes::operator_symbol), + token_info({ { 5, 39 }, { 5, 71 } }, hl_scopes::remark), + + token_info({ { 5, 71 }, { 5, 72 } }, hl_scopes::continuation), + token_info({ { 6, 0 }, { 6, 15 } }, hl_scopes::ignored), + + token_info({ { 6, 15 }, { 6, 17 } }, hl_scopes::operand), + token_info({ { 6, 17 }, { 6, 18 } }, hl_scopes::operator_symbol), + token_info({ { 6, 39 }, { 6, 71 } }, hl_scopes::remark), + + token_info({ { 6, 71 }, { 6, 72 } }, hl_scopes::continuation), + token_info({ { 7, 0 }, { 7, 15 } }, hl_scopes::ignored), + + token_info({ { 7, 15 }, { 7, 17 } }, hl_scopes::operand), + token_info({ { 7, 17 }, { 7, 18 } }, hl_scopes::operator_symbol), + token_info({ { 7, 39 }, { 7, 71 } }, hl_scopes::remark), + + token_info({ { 7, 71 }, { 7, 72 } }, hl_scopes::continuation), + token_info({ { 8, 0 }, { 8, 15 } }, hl_scopes::ignored), + + token_info({ { 8, 15 }, { 8, 17 } }, hl_scopes::operand), + token_info({ { 8, 17 }, { 8, 18 } }, hl_scopes::operator_symbol), + token_info({ { 8, 39 }, { 8, 71 } }, hl_scopes::remark), + }; + + EXPECT_EQ(tokens, expected); +}