From cb1a834467f9a27c166042f0915122e57a01d90b Mon Sep 17 00:00:00 2001 From: jirimosinger <99467904+jirimosinger@users.noreply.github.com> Date: Tue, 22 Nov 2022 10:22:07 +0100 Subject: [PATCH] feat: Endevor preprocessor statements highlighting and parsing --- clients/vscode-hlasmplugin/CHANGELOG.md | 1 + language_server/src/lsp/lsp_server.h | 2 +- parser_library/src/analyzer.cpp | 31 +++- parser_library/src/analyzer.h | 6 +- parser_library/src/context/copy_member.h | 2 +- parser_library/src/context/statement_cache.h | 2 +- parser_library/src/diagnostic.cpp | 12 ++ parser_library/src/diagnostic.h | 2 + .../instruction_sets/asm_processor.cpp | 68 ++++--- .../instruction_sets/asm_processor.h | 10 +- .../src/processing/opencode_provider.cpp | 11 +- .../src/processing/opencode_provider.h | 2 + .../src/processing/preprocessor.cpp | 38 ++++ parser_library/src/processing/preprocessor.h | 46 ++++- .../preprocessors/cics_preprocessor.cpp | 12 +- .../preprocessors/db2_preprocessor.cpp | 22 ++- .../preprocessors/endevor_preprocessor.cpp | 166 +++++++++++++----- .../src/processing/processing_manager.cpp | 20 ++- .../src/processing/processing_manager.h | 2 + .../statement_analyzers/lsp_analyzer.cpp | 38 +++- .../statement_analyzers/lsp_analyzer.h | 5 +- .../lookahead_processor.cpp | 5 +- .../macrodef_processor.cpp | 9 +- .../statement_processors/ordinary_processor.h | 2 +- .../statement_processor.h | 2 +- parser_library/src/semantics/CMakeLists.txt | 1 + parser_library/src/semantics/collector.cpp | 2 - parser_library/src/semantics/collector.h | 1 - .../src/semantics/source_info_processor.h | 2 +- parser_library/src/semantics/statement.cpp | 41 +++++ parser_library/src/semantics/statement.h | 31 ++++ .../src/semantics/statement_fields.h | 5 +- parser_library/test/lsp/CMakeLists.txt | 3 +- .../lsp/lsp_context_preprocessor_test.cpp | 114 ++++++++++++ .../processing/cics_preprocessor_test.cpp | 11 +- .../test/processing/db2_preprocessor_test.cpp | 123 ++++++++----- .../processing/endevor_preprocessor_test.cpp | 131 ++++++-------- .../test/semantics/highlighting_test.cpp | 19 ++ 38 files changed, 760 insertions(+), 240 deletions(-) create mode 100644 parser_library/src/semantics/statement.cpp create mode 100644 parser_library/test/lsp/lsp_context_preprocessor_test.cpp diff --git a/clients/vscode-hlasmplugin/CHANGELOG.md b/clients/vscode-hlasmplugin/CHANGELOG.md index 3936ac6c9..455e1049c 100644 --- a/clients/vscode-hlasmplugin/CHANGELOG.md +++ b/clients/vscode-hlasmplugin/CHANGELOG.md @@ -6,6 +6,7 @@ - Command for downloading copybooks allows selections of data sets which should be downloaded - Code actions for an unknown operation code - Quick fixes for typos in instruction and macro names added to the code actions +- Endevor preprocessor statements highlighting and parsing ## [1.5.0](https://github.com/eclipse/che-che4z-lsp-for-hlasm/compare/1.4.0...1.5.0) (2022-11-02) diff --git a/language_server/src/lsp/lsp_server.h b/language_server/src/lsp/lsp_server.h index 03f972aa3..1cb5ebd39 100644 --- a/language_server/src/lsp/lsp_server.h +++ b/language_server/src/lsp/lsp_server.h @@ -54,7 +54,7 @@ class server final : public hlasm_plugin::language_server::server, void respond(const json& id, const std::string& requested_method, const json& args) override; // Sends notification to LSP client using send_message_provider. void notify(const std::string& method, const json& args) override; - // Sends errorous respond to LSP client using send_message_provider. + // Sends erroneous respond to LSP client using send_message_provider. void respond_error(const json& id, const std::string& requested_method, int err_code, diff --git a/parser_library/src/analyzer.cpp b/parser_library/src/analyzer.cpp index b1de68a09..3b864ca04 100644 --- a/parser_library/src/analyzer.cpp +++ b/parser_library/src/analyzer.cpp @@ -50,13 +50,15 @@ workspaces::parse_lib_provider& analyzer_options::get_lib_provider() const return workspaces::empty_parse_lib_provider::instance; } -std::unique_ptr analyzer_options::get_preprocessor( - processing::library_fetcher asm_lf, diagnostic_op_consumer& diag_consumer) const +std::unique_ptr analyzer_options::get_preprocessor(processing::library_fetcher asm_lf, + diagnostic_op_consumer& diag_consumer, + semantics::source_info_processor& src_proc, + context::id_storage& ids) const { - const auto transform_preprocessor = [&asm_lf, &diag_consumer](const preprocessor_options& po) { + const auto transform_preprocessor = [&asm_lf, &diag_consumer, &src_proc, &ids](const preprocessor_options& po) { return std::visit( - [&asm_lf, &diag_consumer](const auto& p) -> std::unique_ptr { - return processing::preprocessor::create(p, std::move(asm_lf), &diag_consumer); + [&asm_lf, &diag_consumer, &src_proc, &ids](const auto& p) -> std::unique_ptr { + return processing::preprocessor::create(p, std::move(asm_lf), &diag_consumer, src_proc, ids); }, po); }; @@ -65,16 +67,29 @@ std::unique_ptr analyzer_options::get_preprocessor( else if (preprocessor_args.size() == 1) return transform_preprocessor(preprocessor_args.front()); - struct combined_preprocessor : processing::preprocessor + struct combined_preprocessor final : processing::preprocessor { std::vector> pp; + document generate_replacement(document doc) override { + clear_statements(); + for (const auto& p : pp) doc = p->generate_replacement(std::move(doc)); + return doc; } + + std::vector> take_statements() override + { + for (const auto& p : pp) + set_statements(p->take_statements()); + + return preprocessor::take_statements(); + } } tmp; + std::transform( preprocessor_args.begin(), preprocessor_args.end(), std::back_inserter(tmp.pp), transform_preprocessor); @@ -103,7 +118,9 @@ analyzer::analyzer(const std::string& text, analyzer_options opts) return result; }, - *this), + *this, + src_proc_, + ctx_.hlasm_ctx->ids()), opts.parsing_opencode == file_is_opencode::yes ? processing::opencode_provider_options { true, 10 } : processing::opencode_provider_options {}, opts.vf_monitor), diff --git a/parser_library/src/analyzer.h b/parser_library/src/analyzer.h index f25b1bd54..fe3de873a 100644 --- a/parser_library/src/analyzer.h +++ b/parser_library/src/analyzer.h @@ -74,8 +74,10 @@ class analyzer_options context::hlasm_context& get_hlasm_context(); analyzing_context& get_context(); workspaces::parse_lib_provider& get_lib_provider() const; - std::unique_ptr get_preprocessor( - processing::library_fetcher, diagnostic_op_consumer&) const; + std::unique_ptr get_preprocessor(processing::library_fetcher, + diagnostic_op_consumer&, + semantics::source_info_processor&, + context::id_storage& ids) const; friend class analyzer; diff --git a/parser_library/src/context/copy_member.h b/parser_library/src/context/copy_member.h index ab11a5d01..89b9324cc 100644 --- a/parser_library/src/context/copy_member.h +++ b/parser_library/src/context/copy_member.h @@ -26,7 +26,7 @@ struct copy_member; // structure represents COPY member in HLASM macro library struct copy_member { - // member idenifier + // member identifier const id_index name; // block of statements defining the member cached_block cached_definition; diff --git a/parser_library/src/context/statement_cache.h b/parser_library/src/context/statement_cache.h index 00c3b9535..0c60c6aa6 100644 --- a/parser_library/src/context/statement_cache.h +++ b/parser_library/src/context/statement_cache.h @@ -26,7 +26,7 @@ struct complete_statement; namespace hlasm_plugin::parser_library::context { // storage used to store one deferred statement in many parsed formats -// used by macro and copy definition to avoid multiple re-parsing of a deferrend stataments +// used by macro and copy definition to avoid multiple re-parsing of a deferred statements class statement_cache { public: diff --git a/parser_library/src/diagnostic.cpp b/parser_library/src/diagnostic.cpp index 5876b64cc..cc9828068 100644 --- a/parser_library/src/diagnostic.cpp +++ b/parser_library/src/diagnostic.cpp @@ -2708,10 +2708,22 @@ diagnostic_s diagnostic_s::warning_L0006(const utils::resource::resource_locatio diagnostic_tag::none); } +diagnostic_s diagnostic_s::fade(const utils::resource::resource_location& loc, const range& range) +{ + return diagnostic_s(loc.get_uri(), + range, + diagnostic_severity::hint, + "PREPROC", + "Statement overwritten by preprocessor", + {}, + diagnostic_tag::unnecessary); // todo move and make more verbose +} + diagnostic_op diagnostic_op::error_S100(std::string_view message, const range& range) { return diagnostic_op(diagnostic_severity::error, "S100", concat("Long ordinary symbol name - ", message), range); } + std::string diagnostic_decorate_message(std::string_view field, std::string_view message) { static const std::string_view prefix = "While evaluating the result of substitution '"; diff --git a/parser_library/src/diagnostic.h b/parser_library/src/diagnostic.h index 4db70b37a..ce89c0d78 100644 --- a/parser_library/src/diagnostic.h +++ b/parser_library/src/diagnostic.h @@ -896,6 +896,8 @@ class diagnostic_s static diagnostic_s info_SUP(const utils::resource::resource_location& file_name); + static diagnostic_s fade(const utils::resource::resource_location& loc, const range& range); + /* E01x - wrong format - E010 - unknown name diff --git a/parser_library/src/processing/instruction_sets/asm_processor.cpp b/parser_library/src/processing/instruction_sets/asm_processor.cpp index 12d685ebd..3b8fecb76 100644 --- a/parser_library/src/processing/instruction_sets/asm_processor.cpp +++ b/parser_library/src/processing/instruction_sets/asm_processor.cpp @@ -350,7 +350,7 @@ void asm_processor::process_data_instruction(rebuilt_statement stmt) // Why is this so complicated? // 1. We cannot represent the individual operands because of bitfields. // 2. We cannot represent the whole area as a single dependency when the alignment requirements are growing. - // Therefore, we split the operands into chunks depending on the alignent. + // Therefore, we split the operands into chunks depending on the alignment. // Whenever the alignment requirement increases between consecutive operands, we start a new chunk. for (auto it = operands.begin(); it != operands.end();) { @@ -723,51 +723,79 @@ void asm_processor::process(std::shared_ptraccess_asm()->access_expr()->expression; - auto sym_expr = dynamic_cast(expr.get()); - - if (!sym_expr) - { - if (diagnoser) - diagnoser->add_diagnostic(diagnostic_op::error_E058(stmt.operands_ref().value.front()->operand_range)); - return false; - } - - auto tmp = ctx.hlasm_ctx->copy_members().find(sym_expr->value); + auto tmp = ctx.hlasm_ctx->copy_members().find(copy_member_id); if (tmp == ctx.hlasm_ctx->copy_members().end()) { bool result = lib_provider.parse_library( - sym_expr->value.to_string(), ctx, workspaces::library_data { processing_kind::COPY, sym_expr->value }); + copy_member_id.to_string(), ctx, workspaces::library_data { processing_kind::COPY, copy_member_id }); if (!result) { if (diagnoser) - diagnoser->add_diagnostic(diagnostic_op::error_E058(stmt.operands_ref().value.front()->operand_range)); + diagnoser->add_diagnostic(diagnostic_op::error_E058(operand_range)); return false; } } + auto whole_copy_stack = ctx.hlasm_ctx->whole_copy_stack(); - auto cycle_tmp = std::find(whole_copy_stack.begin(), whole_copy_stack.end(), sym_expr->value); + auto cycle_tmp = std::find(whole_copy_stack.begin(), whole_copy_stack.end(), copy_member_id); if (cycle_tmp != whole_copy_stack.end()) { if (diagnoser) - diagnoser->add_diagnostic(diagnostic_op::error_E062(stmt.stmt_range_ref())); + diagnoser->add_diagnostic(diagnostic_op::error_E062(stmt_range)); return false; } - ctx.hlasm_ctx->enter_copy_member(sym_expr->value); - return true; } +bool asm_processor::process_copy(const semantics::complete_statement& stmt, + analyzing_context ctx, + workspaces::parse_lib_provider& lib_provider, + diagnosable_ctx* diagnoser, + bool enter_copy) +{ + if (stmt.operands_ref().value.size() != 1 || !stmt.operands_ref().value.front()->access_asm() + || !stmt.operands_ref().value.front()->access_asm()->access_expr()) + { + if (diagnoser) + diagnoser->add_diagnostic(diagnostic_op::error_E058(stmt.operands_ref().field_range)); + return false; + } + + auto& expr = stmt.operands_ref().value.front()->access_asm()->access_expr()->expression; + auto sym_expr = dynamic_cast(expr.get()); + + if (!sym_expr) + { + if (diagnoser) + diagnoser->add_diagnostic(diagnostic_op::error_E058(stmt.operands_ref().value.front()->operand_range)); + return false; + } + + auto result = parse_copy(ctx, + lib_provider, + sym_expr->value, + stmt.operands_ref().value.front()->operand_range, + stmt.stmt_range_ref(), + diagnoser); + + if (result && enter_copy) + ctx.hlasm_ctx->enter_copy_member(sym_expr->value); + + return result; +} + asm_processor::process_table_t asm_processor::create_table() { process_table_t table; diff --git a/parser_library/src/processing/instruction_sets/asm_processor.h b/parser_library/src/processing/instruction_sets/asm_processor.h index 9a167d408..167949d53 100644 --- a/parser_library/src/processing/instruction_sets/asm_processor.h +++ b/parser_library/src/processing/instruction_sets/asm_processor.h @@ -39,10 +39,18 @@ class asm_processor : public low_language_processor void process(std::shared_ptr stmt) override; + static bool parse_copy(analyzing_context ctx, + workspaces::parse_lib_provider& lib_provider, + context::id_index copy_member_id, + const range& operand_range, + const range& stmt_range, + diagnosable_ctx* diagnoser); + static bool process_copy(const semantics::complete_statement& stmt, analyzing_context ctx, workspaces::parse_lib_provider& lib_provider, - diagnosable_ctx* diagnoser); + diagnosable_ctx* diagnoser, + bool enter_copy = true); private: opencode_provider* open_code_; diff --git a/parser_library/src/processing/opencode_provider.cpp b/parser_library/src/processing/opencode_provider.cpp index d7b23b2b4..6e03bc62d 100644 --- a/parser_library/src/processing/opencode_provider.cpp +++ b/parser_library/src/processing/opencode_provider.cpp @@ -212,7 +212,7 @@ std::shared_ptr opencode_provider::process_looka const range& op_range) { m_ctx->hlasm_ctx->set_source_position(collector.current_instruction().field_range.start); - auto proc_status = proc.get_processing_status(collector.peek_instruction()); + auto proc_status = proc.get_processing_status(collector.current_instruction()); if (op_text && proc_status.first.form != processing_form::IGNORED @@ -265,7 +265,7 @@ std::shared_ptr opencode_provider::process_ordin return nullptr; m_ctx->hlasm_ctx->set_source_position(collector.current_instruction().field_range.start); - auto proc_status = proc.get_processing_status(collector.peek_instruction()); + auto proc_status = proc.get_processing_status(collector.current_instruction()); if (op_text) { @@ -597,6 +597,13 @@ bool opencode_provider::finished() const return std::none_of(o.begin(), o.end(), [](const auto& c) { return c.suspended(); }); } +std::vector> +opencode_provider::get_preprocessor_statements() const +{ + return m_preprocessor ? m_preprocessor->take_statements() + : std::vector>(); +} + parsing::hlasmparser_multiline& opencode_provider::parser() { if (!m_line_fed) diff --git a/parser_library/src/processing/opencode_provider.h b/parser_library/src/processing/opencode_provider.h index d61e12fb4..86bcd9d0c 100644 --- a/parser_library/src/processing/opencode_provider.h +++ b/parser_library/src/processing/opencode_provider.h @@ -139,6 +139,8 @@ class opencode_provider final : public statement_provider bool finished() const override; + std::vector> get_preprocessor_statements() const; + private: void feed_line(const parsing::parser_holder& p, bool is_process); bool is_comment(); diff --git a/parser_library/src/processing/preprocessor.cpp b/parser_library/src/processing/preprocessor.cpp index bdc010142..61f843388 100644 --- a/parser_library/src/processing/preprocessor.cpp +++ b/parser_library/src/processing/preprocessor.cpp @@ -15,6 +15,8 @@ #include "preprocessor.h" #include "lexing/logical_line.h" +#include "semantics/source_info_processor.h" +#include "semantics/statement.h" #include "utils/unicode_text.h" namespace hlasm_plugin::parser_library::processing { @@ -42,4 +44,40 @@ bool preprocessor::is_continued(std::string_view s) return !cont.empty() && cont != " "; } +void preprocessor::clear_statements() { m_statements.clear(); } + +void preprocessor::set_statement(std::shared_ptr stmt) +{ + m_statements.emplace_back(std::move(stmt)); +} + +void preprocessor::set_statements(std::vector> stmts) +{ + m_statements.insert( + m_statements.end(), std::make_move_iterator(stmts.begin()), std::make_move_iterator(stmts.end())); +} + +std::vector> preprocessor::take_statements() +{ + return std::move(m_statements); +} + +void preprocessor::do_highlighting( + const semantics::preprocessor_statement_si& stmt, semantics::source_info_processor& src_proc) +{ + if (stmt.label_ref().type != semantics::label_si_type::EMPTY) + src_proc.add_hl_symbol(token_info(stmt.label_ref().field_range, semantics::hl_scopes::label)); + + src_proc.add_hl_symbol(token_info(stmt.instruction_ref().field_range, semantics::hl_scopes::instruction)); + + for (const auto& op : stmt.operands_ref().value) + { + if (op) + src_proc.add_hl_symbol(token_info(op->operand_range, semantics::hl_scopes::operand)); + } + + if (stmt.remarks_ref().value.size()) + src_proc.add_hl_symbol(token_info(stmt.remarks_ref().field_range, semantics::hl_scopes::remark)); +} + } // namespace hlasm_plugin::parser_library::processing diff --git a/parser_library/src/processing/preprocessor.h b/parser_library/src/processing/preprocessor.h index 13f11bec9..fb8dcb9c6 100644 --- a/parser_library/src/processing/preprocessor.h +++ b/parser_library/src/processing/preprocessor.h @@ -37,6 +37,15 @@ struct logical_line; struct logical_line_extractor_args; } // namespace lexing +namespace semantics { +class source_info_processor; +struct preprocessor_statement_si; +} // namespace semantics + +namespace context { +class id_storage; +} + } // namespace hlasm_plugin::parser_library namespace hlasm_plugin::parser_library::processing { @@ -50,16 +59,31 @@ class preprocessor virtual document generate_replacement(document doc) = 0; - static std::unique_ptr create( - const cics_preprocessor_options&, library_fetcher, diagnostic_op_consumer*); + static std::unique_ptr create(const cics_preprocessor_options&, + library_fetcher, + diagnostic_op_consumer*, + semantics::source_info_processor&, + context::id_storage&); - static std::unique_ptr create( - const db2_preprocessor_options&, library_fetcher, diagnostic_op_consumer*); + static std::unique_ptr create(const db2_preprocessor_options&, + library_fetcher, + diagnostic_op_consumer*, + semantics::source_info_processor&, + context::id_storage&); - static std::unique_ptr create( - const endevor_preprocessor_options&, library_fetcher, diagnostic_op_consumer*); + static std::unique_ptr create(const endevor_preprocessor_options&, + library_fetcher, + diagnostic_op_consumer*, + semantics::source_info_processor&, + context::id_storage&); + + virtual std::vector> take_statements(); protected: + preprocessor() = default; + preprocessor(const preprocessor&) = default; + preprocessor(preprocessor&&) = default; + using line_iterator = std::vector::const_iterator; static line_iterator extract_nonempty_logical_line(lexing::logical_line& out, @@ -67,7 +91,17 @@ class preprocessor line_iterator end, const lexing::logical_line_extractor_args& opts); + void clear_statements(); + void set_statement(std::shared_ptr stmt); + void set_statements(std::vector> stmts); + static bool is_continued(std::string_view s); + + static void do_highlighting( + const semantics::preprocessor_statement_si& stmt, semantics::source_info_processor& src_proc); + +private: + std::vector> m_statements; }; } // namespace hlasm_plugin::parser_library::processing diff --git a/parser_library/src/processing/preprocessors/cics_preprocessor.cpp b/parser_library/src/processing/preprocessors/cics_preprocessor.cpp index 363151302..6cff38e25 100644 --- a/parser_library/src/processing/preprocessors/cics_preprocessor.cpp +++ b/parser_library/src/processing/preprocessors/cics_preprocessor.cpp @@ -21,6 +21,8 @@ #include "lexing/logical_line.h" #include "preprocessor_options.h" #include "processing/preprocessor.h" +#include "semantics/source_info_processor.h" +#include "semantics/statement.h" #include "utils/concat.h" #include "utils/unicode_text.h" #include "workspaces/parse_lib_provider.h" @@ -830,7 +832,7 @@ class mini_parser } }; -class cics_preprocessor : public preprocessor +class cics_preprocessor final : public preprocessor { lexing::logical_line m_logical_line; std::string m_operands; @@ -1124,6 +1126,7 @@ class cics_preprocessor : public preprocessor // Inherited via preprocessor document generate_replacement(document doc) override { + clear_statements(); m_result.clear(); m_result.reserve(doc.size()); @@ -1274,8 +1277,11 @@ class cics_preprocessor : public preprocessor } // namespace -std::unique_ptr preprocessor::create( - const cics_preprocessor_options& options, library_fetcher libs, diagnostic_op_consumer* diags) +std::unique_ptr preprocessor::create(const cics_preprocessor_options& options, + library_fetcher libs, + diagnostic_op_consumer* diags, + semantics::source_info_processor& src_proc, + context::id_storage& ids) { return std::make_unique(options, std::move(libs), diags); } diff --git a/parser_library/src/processing/preprocessors/db2_preprocessor.cpp b/parser_library/src/processing/preprocessors/db2_preprocessor.cpp index f6f52633a..191974897 100644 --- a/parser_library/src/processing/preprocessors/db2_preprocessor.cpp +++ b/parser_library/src/processing/preprocessors/db2_preprocessor.cpp @@ -24,6 +24,8 @@ #include "lexing/logical_line.h" #include "preprocessor_options.h" #include "processing/preprocessor.h" +#include "semantics/source_info_processor.h" +#include "semantics/statement.h" #include "utils/concat.h" #include "workspaces/parse_lib_provider.h" @@ -43,7 +45,7 @@ namespace hlasm_plugin::parser_library::processing { namespace { using utils::concat; -class db2_preprocessor : public preprocessor +class db2_preprocessor final : public preprocessor { lexing::logical_line m_logical_line; std::string m_operands; @@ -232,12 +234,14 @@ class db2_preprocessor : public preprocessor void process_include(std::string_view operands, size_t lineno) { - if (operands == "SQLCA") + auto operands_upper = context::to_upper_copy(std::string(operands)); + + if (operands_upper == "SQLCA") { inject_SQLCA(); return; } - if (operands == "SQLDA") + if (operands_upper == "SQLDA") { inject_SQLDA(); return; @@ -246,7 +250,7 @@ class db2_preprocessor : public preprocessor std::optional include_text; if (m_libs) - include_text = m_libs(operands); + include_text = m_libs(operands_upper); if (!include_text.has_value()) { if (m_diags) @@ -445,7 +449,7 @@ class db2_preprocessor : public preprocessor add_ds_line(label, "_DATA", li.prefix + std::to_string(len <= li.limit ? len : li.limit), false); if (len > li.limit) m_result.emplace_back(replaced_line { concat(" ORG *+(", - // there seems be this strage artifical limit + // there seems be this strange artificial limit std::min(len - li.limit, 1073676289ULL), ")\n") }); break; @@ -788,6 +792,7 @@ class db2_preprocessor : public preprocessor // Inherited via preprocessor document generate_replacement(document doc) override { + clear_statements(); m_source_translated = false; m_result.clear(); m_result.reserve(doc.size()); @@ -817,8 +822,11 @@ class db2_preprocessor : public preprocessor }; } // namespace -std::unique_ptr preprocessor::create( - const db2_preprocessor_options& opts, library_fetcher libs, diagnostic_op_consumer* diags) +std::unique_ptr preprocessor::create(const db2_preprocessor_options& opts, + library_fetcher libs, + diagnostic_op_consumer* diags, + semantics::source_info_processor& src_proc, + context::id_storage& ids) { return std::make_unique(opts, std::move(libs), diags); } diff --git a/parser_library/src/processing/preprocessors/endevor_preprocessor.cpp b/parser_library/src/processing/preprocessors/endevor_preprocessor.cpp index 4b3552878..601070aa9 100644 --- a/parser_library/src/processing/preprocessors/endevor_preprocessor.cpp +++ b/parser_library/src/processing/preprocessors/endevor_preprocessor.cpp @@ -18,55 +18,141 @@ #include "preprocessor_options.h" #include "processing/preprocessor.h" +#include "semantics/collector.h" +#include "semantics/source_info_processor.h" +#include "semantics/statement.h" namespace hlasm_plugin::parser_library::processing { -class endevor_preprocessor : public preprocessor +namespace { +struct stack_entry +{ + std::string name; + document doc; + document::iterator current; + stack_entry(std::string name, document doc_) + : name(std::move(name)) + , doc(std::move(doc_)) + , current(doc.begin()) + {} + + void next() { ++current; } + bool end() const { return current == doc.end(); } +}; + +std::string_view get_copy_member(const std::match_results& matches) +{ + if (matches.size() != 4) + return ""; + + return std::string_view(std::to_address(matches[2].first), matches[2].length()); +} + +std::pair get_stmt_part_pair( + const std::match_results& matches, size_t index, size_t line_no) +{ + std::string_view name(std::to_address(matches[index].first), matches[index].length()); + auto r = range(position(line_no, std::distance(matches[0].first, matches[index].first)), + position(line_no, std::distance(matches[0].first, matches[index].second))); + + return { name, std::move(r) }; +} + +std::shared_ptr get_preproc_statement( + const std::match_results& matches, size_t line_no, context::id_storage& ids) +{ + if (matches.size() != 4) + return nullptr; + + auto stmt_r = range({ line_no, 0 }, { line_no, matches[0].str().length() }); + + auto inc_range = get_stmt_part_pair(matches, 1, line_no).second; + auto [member, member_range] = get_stmt_part_pair(matches, 2, line_no); + + auto remarks_r = range(); + std::vector rems; + if (matches[3].length()) + { + remarks_r = get_stmt_part_pair(matches, 3, line_no).second; + rems.emplace_back(remarks_r); + } + + auto remarks_si = semantics::remarks_si(std::move(remarks_r), std::move(rems)); + + return std::make_shared( + std::move(stmt_r), std::move(inc_range), member, std::move(member_range), std::move(remarks_si), ids); +} +} // namespace + +class endevor_preprocessor final : public preprocessor { library_fetcher m_libs; diagnostic_op_consumer* m_diags = nullptr; endevor_preprocessor_options m_options; + semantics::source_info_processor& m_src_proc; + context::id_storage& m_ids; + + bool process_member(std::string_view member, std::vector& stack) const + { + std::string member_upper = context::to_upper_copy(std::string(member)); + + if (std::any_of(stack.begin(), stack.end(), [member = std::string_view(member_upper)](const auto& e) { + return e.name == member; + })) + { + if (m_diags) + m_diags->add_diagnostic(diagnostic_op::error_END002( + range(position(std::prev(stack.front().current)->lineno().value_or(0), 0)), member)); + return false; + } + + if (auto lib = m_libs(member_upper); !lib.has_value()) + { + if (m_diags) + m_diags->add_diagnostic(diagnostic_op::error_END001( + range(position(std::prev(stack.front().current)->lineno().value_or(0), 0)), member)); + + // just continue... + } + else + { + document member_doc(lib.value()); + member_doc.convert_to_replaced(); + stack.emplace_back(std::move(member_upper), std::move(member_doc)); + } + + return true; + } public: - endevor_preprocessor( - const endevor_preprocessor_options& options, library_fetcher libs, diagnostic_op_consumer* diags) + endevor_preprocessor(const endevor_preprocessor_options& options, + library_fetcher libs, + diagnostic_op_consumer* diags, + semantics::source_info_processor& src_proc, + context::id_storage& ids) : m_libs(std::move(libs)) , m_diags(diags) , m_options(options) + , m_src_proc(src_proc) + , m_ids(ids) {} // Inherited via preprocessor document generate_replacement(document doc) override { + clear_statements(); + if (std::none_of(doc.begin(), doc.end(), [](const auto& l) { auto text = l.text(); return text.starts_with("-INC ") || text.starts_with("++INCLUDE "); })) return doc; - static std::regex include_regex(R"(^(?:-INC|\+\+INCLUDE)\s+(\S+))"); + static std::regex include_regex(R"(^(-INC|\+\+INCLUDE)\s+(\S+)(?:\s+(.*))?)"); std::vector result; result.reserve(doc.size()); - struct stack_entry - { - std::string name; - document doc; - document::iterator current; - stack_entry(std::string name, document doc_) - : name(std::move(name)) - , doc(std::move(doc_)) - , current(doc.begin()) - {} - - stack_entry(stack_entry&&) = default; - stack_entry& operator=(stack_entry&&) = default; - - void next() { ++current; } - bool end() const { return current == doc.end(); } - }; - std::vector stack; stack.emplace_back(std::string(), std::move(doc)); @@ -89,28 +175,17 @@ class endevor_preprocessor : public preprocessor continue; } - std::string_view member(std::to_address(matches[1].first), matches[1].length()); - if (std::any_of(stack.begin(), stack.end(), [&member](const auto& e) { return e.name == member; })) - { - if (m_diags) - m_diags->add_diagnostic(diagnostic_op::error_END002( - range(position(std::prev(stack.front().current)->lineno().value_or(0), 0)), member)); + auto copy_member = get_copy_member(matches); + auto line_no = std::prev(stack.back().current)->lineno(); + + if (!process_member(copy_member, stack)) break; - } - auto lib = m_libs(member); - if (!lib.has_value()) - { - if (m_diags) - m_diags->add_diagnostic(diagnostic_op::error_END001( - range(position(std::prev(stack.front().current)->lineno().value_or(0), 0)), member)); - // just continue... - } - else + if (line_no) { - document member_doc(lib.value()); - member_doc.convert_to_replaced(); - stack.emplace_back(std::string(member), std::move(member_doc)); + auto stmt = get_preproc_statement(matches, *line_no, m_ids); + do_highlighting(*stmt, m_src_proc); + set_statement(std::move(stmt)); } } @@ -118,9 +193,12 @@ class endevor_preprocessor : public preprocessor } }; -std::unique_ptr preprocessor::create( - const endevor_preprocessor_options& opts, library_fetcher lf, diagnostic_op_consumer* diags) +std::unique_ptr preprocessor::create(const endevor_preprocessor_options& opts, + library_fetcher lf, + diagnostic_op_consumer* diags, + semantics::source_info_processor& src_proc, + context::id_storage& ids) { - return std::make_unique(opts, std::move(lf), diags); + return std::make_unique(opts, std::move(lf), diags, src_proc, ids); } } // namespace hlasm_plugin::parser_library::processing diff --git a/parser_library/src/processing/processing_manager.cpp b/parser_library/src/processing/processing_manager.cpp index 633eca449..bfb67cc9a 100644 --- a/parser_library/src/processing/processing_manager.cpp +++ b/parser_library/src/processing/processing_manager.cpp @@ -42,6 +42,7 @@ processing_manager::processing_manager(std::unique_ptr base_p , opencode_prov_(*base_provider) , lsp_analyzer_(*ctx_.hlasm_ctx, *ctx_.lsp_ctx, file_text) , stms_analyzers_({ &lsp_analyzer_ }) + , file_loc_(file_loc) { switch (data.proc_kind) { @@ -249,7 +250,24 @@ void processing_manager::finish_copy_member(copy_processing_result result) lsp_analyzer_.copydef_finished(member, std::move(result)); } -void processing_manager::finish_opencode() { lsp_analyzer_.opencode_finished(); } +void processing_manager::finish_opencode() +{ + for (const auto& stmt : opencode_prov_.get_preprocessor_statements()) + { + if (!stmt) + continue; + + lsp_analyzer_.analyze(*stmt); + + if (stmt->m_resemblence == context::id_storage::well_known::COPY) + asm_processor::process_copy(*stmt, ctx_, lib_provider_, nullptr, false); + + // diagnosable_impl::add_diagnostic(diagnostic_s::fade(file_loc_, stmt->stmt_range_ref())); // todo create + // separate 'fade' container + } + + lsp_analyzer_.opencode_finished(); +} void processing_manager::start_macro_definition( macrodef_start_data start, std::optional file_loc) diff --git a/parser_library/src/processing/processing_manager.h b/parser_library/src/processing/processing_manager.h index 52aa7f4ff..da5f576d7 100644 --- a/parser_library/src/processing/processing_manager.h +++ b/parser_library/src/processing/processing_manager.h @@ -63,6 +63,8 @@ class processing_manager final : public processing_state_listener, public branch lsp_analyzer lsp_analyzer_; std::vector stms_analyzers_; + const utils::resource::resource_location file_loc_; + context::source_snapshot lookahead_stop_; size_t lookahead_stop_ainsert_id = 0; enum class pending_seq_redifinition_state diff --git a/parser_library/src/processing/statement_analyzers/lsp_analyzer.cpp b/parser_library/src/processing/statement_analyzers/lsp_analyzer.cpp index ef95d1f42..a56a94163 100644 --- a/parser_library/src/processing/statement_analyzers/lsp_analyzer.cpp +++ b/parser_library/src/processing/statement_analyzers/lsp_analyzer.cpp @@ -87,7 +87,25 @@ void lsp_analyzer::analyze( break; } - assign_statement_occurences(); + assign_statement_occurences(hlasm_ctx_.current_statement_location().resource_loc); +} + +void lsp_analyzer::analyze(const semantics::preprocessor_statement_si& statement) +{ + collect_occurences(lsp::occurence_kind::ORD, statement); + + const auto& opcode = statement.m_resemblence; + const auto& operands = statement.operands_ref().value; + if (opcode == context::id_storage::well_known::COPY && operands.size() == 1 && operands.front()->access_asm()) + { + auto sym_expr = dynamic_cast( + operands.front()->access_asm()->access_expr()->expression.get()); + + if (sym_expr) + add_copy_operand(sym_expr->value, sym_expr->get_range()); + } + + assign_statement_occurences(hlasm_ctx_.opencode_location()); } void lsp_analyzer::macrodef_started(const macrodef_start_data& data) @@ -137,17 +155,17 @@ void lsp_analyzer::opencode_finished() lsp::text_data_ref_t(file_text_)); } -void lsp_analyzer::assign_statement_occurences() +void lsp_analyzer::assign_statement_occurences(const utils::resource::resource_location& doc_location) { if (in_macro_) { - auto& file_occs = macro_occurences_[hlasm_ctx_.current_statement_location().resource_loc]; + auto& file_occs = macro_occurences_[doc_location]; file_occs.insert( file_occs.end(), std::move_iterator(stmt_occurences_.begin()), std::move_iterator(stmt_occurences_.end())); } else { - auto& file_occs = opencode_occurences_[hlasm_ctx_.current_statement_location().resource_loc]; + auto& file_occs = opencode_occurences_[doc_location]; file_occs.insert( file_occs.end(), std::move_iterator(stmt_occurences_.begin()), std::move_iterator(stmt_occurences_.end())); } @@ -172,6 +190,15 @@ void lsp_analyzer::collect_occurences(lsp::occurence_kind kind, const context::h } } +void lsp_analyzer::collect_occurences(lsp::occurence_kind kind, const semantics::preprocessor_statement_si& statement) +{ + occurence_collector collector(kind, hlasm_ctx_, stmt_occurences_); + + collect_occurence(statement.label_ref(), collector); + collect_occurence(statement.instruction_ref(), collector); + collect_occurence(statement.operands_ref(), collector); +} + void lsp_analyzer::collect_occurence(const semantics::label_si& label, occurence_collector& collector) { switch (label.type) @@ -207,6 +234,9 @@ void lsp_analyzer::collect_occurence(const semantics::instruction_si& instructio collector.occurences.emplace_back( opcode.opcode, macro_def ? std::move(*macro_def) : context::macro_def_ptr {}, instruction.field_range); } + else if (instruction.type == semantics::instruction_si_type::PREPROC) + collector.occurences.emplace_back( + std::get(instruction.value), context::macro_def_ptr {}, instruction.field_range); } void lsp_analyzer::collect_occurence(const semantics::operands_si& operands, occurence_collector& collector) diff --git a/parser_library/src/processing/statement_analyzers/lsp_analyzer.h b/parser_library/src/processing/statement_analyzers/lsp_analyzer.h index b074efd8b..b4955ef0a 100644 --- a/parser_library/src/processing/statement_analyzers/lsp_analyzer.h +++ b/parser_library/src/processing/statement_analyzers/lsp_analyzer.h @@ -48,6 +48,8 @@ class lsp_analyzer : public statement_analyzer statement_provider_kind prov_kind, processing_kind proc_kind) override; + void analyze(const semantics::preprocessor_statement_si& statement); + void macrodef_started(const macrodef_start_data& data); void macrodef_finished(context::macro_def_ptr macrodef, macrodef_processing_result&& result); @@ -56,9 +58,10 @@ class lsp_analyzer : public statement_analyzer void opencode_finished(); private: - void assign_statement_occurences(); + void assign_statement_occurences(const utils::resource::resource_location& doc_location); void collect_occurences(lsp::occurence_kind kind, const context::hlasm_statement& statement); + void collect_occurences(lsp::occurence_kind kind, const semantics::preprocessor_statement_si& statement); void collect_occurence(const semantics::label_si& label, occurence_collector& collector); void collect_occurence(const semantics::instruction_si& instruction, occurence_collector& collector); diff --git a/parser_library/src/processing/statement_processors/lookahead_processor.cpp b/parser_library/src/processing/statement_processors/lookahead_processor.cpp index 8d9c332c7..f3cdfd87b 100644 --- a/parser_library/src/processing/statement_processors/lookahead_processor.cpp +++ b/parser_library/src/processing/statement_processors/lookahead_processor.cpp @@ -117,10 +117,7 @@ void lookahead_processor::process_MACRO() { ++macro_nest_; } void lookahead_processor::process_MEND() { macro_nest_ -= macro_nest_ == 0 ? 0 : 1; } void lookahead_processor::process_COPY(const resolved_statement& statement) { - if (statement.operands_ref().value.size() == 1 && statement.operands_ref().value.front()->access_asm()) - { - asm_processor::process_copy(statement, ctx, lib_provider_, nullptr); - } + asm_processor::process_copy(statement, ctx, lib_provider_, nullptr); } lookahead_processor::process_table_t lookahead_processor::create_table() diff --git a/parser_library/src/processing/statement_processors/macrodef_processor.cpp b/parser_library/src/processing/statement_processors/macrodef_processor.cpp index 95ef8c936..024ca7da9 100644 --- a/parser_library/src/processing/statement_processors/macrodef_processor.cpp +++ b/parser_library/src/processing/statement_processors/macrodef_processor.cpp @@ -394,15 +394,10 @@ void macrodef_processor::process_COPY(const resolved_statement& statement) result_.definition.push_back(std::move(empty)); add_correct_copy_nest(); - if (statement.operands_ref().value.size() == 1 && statement.operands_ref().value.front()->access_asm()) + if (asm_processor::process_copy(statement, ctx, provider_, this)) { - if (asm_processor::process_copy(statement, ctx, provider_, this)) - { - result_.used_copy_members.insert(ctx.hlasm_ctx->current_copy_stack().back().copy_member_definition); - } + result_.used_copy_members.insert(ctx.hlasm_ctx->current_copy_stack().back().copy_member_definition); } - else - add_diagnostic(diagnostic_op::error_E058(statement.operands_ref().field_range)); omit_next_ = true; } diff --git a/parser_library/src/processing/statement_processors/ordinary_processor.h b/parser_library/src/processing/statement_processors/ordinary_processor.h index 8667dc6e2..acb289f73 100644 --- a/parser_library/src/processing/statement_processors/ordinary_processor.h +++ b/parser_library/src/processing/statement_processors/ordinary_processor.h @@ -26,7 +26,7 @@ namespace hlasm_plugin::parser_library::processing { -// statement processor that evaluates the writen code, processes instructions +// statement processor that evaluates the written code, processes instructions class ordinary_processor final : public statement_processor { static constexpr size_t NEST_LIMIT = 100; diff --git a/parser_library/src/processing/statement_processors/statement_processor.h b/parser_library/src/processing/statement_processors/statement_processor.h index 838c7551b..458214688 100644 --- a/parser_library/src/processing/statement_processors/statement_processor.h +++ b/parser_library/src/processing/statement_processors/statement_processor.h @@ -29,7 +29,7 @@ using processor_ptr = std::unique_ptr; // interface for statement processors // they are provided statements which they process in different fashion -// their processing can be interruped by changing the provider (with 'terminal_condition') and they can end their +// their processing can be interrupted by changing the provider (with 'terminal_condition') and they can end their // processing when their need for statements is satisfied class statement_processor : public diagnosable_ctx { diff --git a/parser_library/src/semantics/CMakeLists.txt b/parser_library/src/semantics/CMakeLists.txt index 44c673e15..d1664a464 100644 --- a/parser_library/src/semantics/CMakeLists.txt +++ b/parser_library/src/semantics/CMakeLists.txt @@ -24,6 +24,7 @@ target_sources(parser_library PRIVATE range_provider.h source_info_processor.cpp source_info_processor.h + statement.cpp statement.h statement_fields.cpp statement_fields.h diff --git a/parser_library/src/semantics/collector.cpp b/parser_library/src/semantics/collector.cpp index 22212c52d..903977376 100644 --- a/parser_library/src/semantics/collector.cpp +++ b/parser_library/src/semantics/collector.cpp @@ -188,8 +188,6 @@ void collector::append_operand_field(collector&& c) lit_.insert(lit_.end(), std::make_move_iterator(c.lit_.begin()), std::make_move_iterator(c.lit_.end())); } -const instruction_si& collector::peek_instruction() { return *instr_; } - context::shared_stmt_ptr collector::extract_statement(processing::processing_status status, range& statement_range) { if (!lbl_) diff --git a/parser_library/src/semantics/collector.h b/parser_library/src/semantics/collector.h index 441e5417c..d06ccdbac 100644 --- a/parser_library/src/semantics/collector.h +++ b/parser_library/src/semantics/collector.h @@ -64,7 +64,6 @@ class collector void append_operand_field(collector&& c); - const instruction_si& peek_instruction(); context::shared_stmt_ptr extract_statement(processing::processing_status status, range& statement_range); std::vector extract_hl_symbols(); void set_hl_symbols(std::vector); diff --git a/parser_library/src/semantics/source_info_processor.h b/parser_library/src/semantics/source_info_processor.h index 2e6ab309e..7866ddc24 100644 --- a/parser_library/src/semantics/source_info_processor.h +++ b/parser_library/src/semantics/source_info_processor.h @@ -22,7 +22,7 @@ namespace hlasm_plugin::parser_library::semantics { -// Stores info that are generated during parsing and do not require access to context (currently only highligting) +// Stores info that are generated during parsing and do not require access to context (currently only highlighting) class source_info_processor { public: diff --git a/parser_library/src/semantics/statement.cpp b/parser_library/src/semantics/statement.cpp new file mode 100644 index 000000000..72ec77b89 --- /dev/null +++ b/parser_library/src/semantics/statement.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 Broadcom. + * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Broadcom, Inc. - initial API and implementation + */ + +#include "statement.h" + +#include "expressions/mach_expr_term.h" +#include "semantics/operand_impls.h" + +namespace hlasm_plugin::parser_library::semantics { + +endevor_statement_si::endevor_statement_si(range stmt_range, + range instruction_range, + std::string_view copy_member, + range copy_member_range, + remarks_si remarks, + context::id_storage& ids) + : preprocessor_statement_si(std::move(stmt_range), + label_si(range()), + instruction_si(std::move(instruction_range), context::id_index("-INC"), true), + operands_si(copy_member_range, {}), + std::move(remarks), + context::id_storage::well_known::COPY) +{ + operands.value.emplace_back(std::make_unique( + std::make_unique(ids.add(copy_member), context::id_index(), copy_member_range), + std::string(copy_member), + std::move(copy_member_range))); +} + +} // namespace hlasm_plugin::parser_library::semantics \ No newline at end of file diff --git a/parser_library/src/semantics/statement.h b/parser_library/src/semantics/statement.h index 36aa67df9..7e1102a0d 100644 --- a/parser_library/src/semantics/statement.h +++ b/parser_library/src/semantics/statement.h @@ -158,6 +158,37 @@ struct statement_si_defer_done : public complete_statement const range& stmt_range_ref() const override { return deferred_stmt->stmt_range_ref(); } }; +struct preprocessor_statement_si + : public statement_si // todo think if the statement_si inheritance is too heavy for preprocessor statements +{ + context::id_index m_resemblence; + + preprocessor_statement_si(range stmt_range, + label_si label, + instruction_si instruction, + operands_si operands, + remarks_si remarks, + context::id_index resemblence) + : statement_si(std::move(stmt_range), + std::move(label), + std::move(instruction), + std::move(operands), + std::move(remarks), + {}) + , m_resemblence(resemblence) + {} +}; + +struct endevor_statement_si : public preprocessor_statement_si +{ + endevor_statement_si(range stmt_range, + range instruction_range, + std::string_view copy_member, + range copy_member_range, + remarks_si remarks, + context::id_storage& ids); +}; + } // namespace hlasm_plugin::parser_library::semantics #endif diff --git a/parser_library/src/semantics/statement_fields.h b/parser_library/src/semantics/statement_fields.h index 803bacbd5..b3a297edf 100644 --- a/parser_library/src/semantics/statement_fields.h +++ b/parser_library/src/semantics/statement_fields.h @@ -97,6 +97,7 @@ enum class instruction_si_type { ORD, CONC, + PREPROC, EMPTY }; @@ -105,8 +106,8 @@ using instruction_si_value_t = std::variant; // struct holding semantic information (si) about instruction field struct instruction_si { - instruction_si(range field_range, context::id_index value) - : type(instruction_si_type::ORD) + instruction_si(range field_range, context::id_index value, bool preproc = false) + : type(preproc ? instruction_si_type::PREPROC : instruction_si_type::ORD) , field_range(std::move(field_range)) , value(std::move(value)) {} diff --git a/parser_library/test/lsp/CMakeLists.txt b/parser_library/test/lsp/CMakeLists.txt index d90a67a55..1f6e8a15e 100644 --- a/parser_library/test/lsp/CMakeLists.txt +++ b/parser_library/test/lsp/CMakeLists.txt @@ -21,8 +21,9 @@ target_sources(library_test PRIVATE lsp_context_macro_in_opencode_test.cpp lsp_context_nested_macro_test.cpp lsp_context_ord_sym_test.cpp - lsp_context_test_helper.h + lsp_context_preprocessor_test.cpp lsp_context_seq_sym_test.cpp + lsp_context_test_helper.h lsp_context_var_sym_test.cpp lsp_features_test.cpp ) diff --git a/parser_library/test/lsp/lsp_context_preprocessor_test.cpp b/parser_library/test/lsp/lsp_context_preprocessor_test.cpp new file mode 100644 index 000000000..f14af0eb5 --- /dev/null +++ b/parser_library/test/lsp/lsp_context_preprocessor_test.cpp @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2022 Broadcom. + * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Broadcom, Inc. - initial API and implementation + */ + +#include "gtest/gtest.h" + +#include "../mock_parse_lib_provider.h" +#include "analyzer.h" +#include "context/instruction.h" +#include "utils/resource_location.h" +#include "workspaces/parse_lib_provider.h" + + +using namespace hlasm_plugin::parser_library; +using namespace hlasm_plugin::utils::resource; +using namespace hlasm_plugin::parser_library::lsp; + +namespace { +const resource_location mac_loc("MAC"); +const resource_location source_loc("OPEN"); +const resource_location member_loc("MEMBER"); +const resource_location member2_loc("MEMBER2"); +} // namespace + +class lsp_context_endevor_preprocessor_test : public testing::Test +{ +public: + lsp_context_endevor_preprocessor_test() + : lib_provider({ { "MEMBER", R"(R2 EQU 2 + LR R2,R2)" }, + { "MEMBER2", R"(R5 EQU 5 + LR R5,R5)" } }) + , a(contents, analyzer_options { source_loc, &lib_provider, endevor_preprocessor_options() }) {}; + + void SetUp() override { a.analyze(); } + void TearDown() override {} + +protected: + const std::string contents = + R"( +-INC MEMBER blabla +++INCLUDE MEMBER blabla +-INC MEMBER2 blabla +)"; + std::string content; + mock_parse_lib_provider lib_provider; + analyzer a; +}; + +TEST_F(lsp_context_endevor_preprocessor_test, go_to) +{ + // no jump, instr -INC + EXPECT_EQ(location(position(1, 1), source_loc), a.context().lsp_ctx->definition(source_loc, position(1, 1))); + // no jump, instr ++INCLUDE + EXPECT_EQ(location(position(2, 5), source_loc), a.context().lsp_ctx->definition(source_loc, position(2, 5))); + // no jump, instr -INC + EXPECT_EQ(location(position(3, 1), source_loc), a.context().lsp_ctx->definition(source_loc, position(3, 1))); + + // jump from source to included file + EXPECT_EQ(location(position(0, 3), member_loc), a.context().lsp_ctx->definition(source_loc, position(1, 8))); + // jump from source to included file + EXPECT_EQ(location(position(0, 3), member_loc), a.context().lsp_ctx->definition(source_loc, position(2, 14))); + // jump from source to included file + EXPECT_EQ(location(position(0, 3), member2_loc), a.context().lsp_ctx->definition(source_loc, position(3, 8))); + + // no jump + EXPECT_EQ(location(position(1, 15), source_loc), a.context().lsp_ctx->definition(source_loc, position(1, 15))); + // no jump + EXPECT_EQ(location(position(2, 21), source_loc), a.context().lsp_ctx->definition(source_loc, position(2, 21))); + // no jump + EXPECT_EQ(location(position(3, 15), source_loc), a.context().lsp_ctx->definition(source_loc, position(3, 15))); +} + +TEST_F(lsp_context_endevor_preprocessor_test, refs) +{ + location_list expected_inc_locations { + location(position(1, 0), source_loc), location(position(2, 0), source_loc), location(position(3, 0), source_loc) + }; + location_list expected_member_locations { location(position(1, 6), source_loc), + location(position(2, 11), source_loc) }; + location_list expected_member2_locations { location(position(3, 6), source_loc) }; + location_list expected_blabla_locations {}; + + // -INC/++INCLUDE reference + EXPECT_EQ(expected_inc_locations, a.context().lsp_ctx->references(source_loc, position(1, 1))); + // -INC/++INCLUDE reference + EXPECT_EQ(expected_inc_locations, a.context().lsp_ctx->references(source_loc, position(2, 5))); + // -INC/++INCLUDE reference + EXPECT_EQ(expected_inc_locations, a.context().lsp_ctx->references(source_loc, position(3, 2))); + + // MEMBER reference + EXPECT_EQ(expected_member_locations, a.context().lsp_ctx->references(source_loc, position(1, 8))); + // MEMBER reference + EXPECT_EQ(expected_member_locations, a.context().lsp_ctx->references(source_loc, position(2, 14))); + // MEMBER reference + EXPECT_EQ(expected_member2_locations, a.context().lsp_ctx->references(source_loc, position(3, 8))); + + // blabla reference + EXPECT_EQ(expected_blabla_locations, a.context().lsp_ctx->references(source_loc, position(1, 15))); + // blabla reference + EXPECT_EQ(expected_blabla_locations, a.context().lsp_ctx->references(source_loc, position(2, 21))); + // blabla reference + EXPECT_EQ(expected_blabla_locations, a.context().lsp_ctx->references(source_loc, position(3, 15))); +} diff --git a/parser_library/test/processing/cics_preprocessor_test.cpp b/parser_library/test/processing/cics_preprocessor_test.cpp index a08a674cb..1db6365e7 100644 --- a/parser_library/test/processing/cics_preprocessor_test.cpp +++ b/parser_library/test/processing/cics_preprocessor_test.cpp @@ -18,6 +18,7 @@ #include "../mock_parse_lib_provider.h" #include "preprocessor_options.h" #include "processing/preprocessor.h" +#include "semantics/source_info_processor.h" // test cics preprocessor emulator @@ -45,8 +46,11 @@ TEST(cics_preprocessor, asm_xopts_parsing) { "*ASM XOPTS(SP)", cics_preprocessor_options() }, }) { + semantics::source_info_processor src_info(false); + context::id_storage ids; + auto p = preprocessor::create( - cics_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, nullptr); + cics_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, nullptr, src_info, ids); auto result = p->generate_replacement(document(text_template)); EXPECT_GT(result.size(), 0); @@ -77,10 +81,13 @@ std::ostream& operator<<(std::ostream& os, const cics_preprocessor_tests_basics_ TEST_P(cics_preprocessor_tests, basics) { + semantics::source_info_processor src_info(false); + context::id_storage ids; const auto& [input, expected] = GetParam(); auto [text_template, config] = input; + auto p = preprocessor::create( - config, [](std::string_view) { return std::nullopt; }, nullptr); + config, [](std::string_view) { return std::nullopt; }, nullptr, src_info, ids); auto result = p->generate_replacement(document(text_template)); diff --git a/parser_library/test/processing/db2_preprocessor_test.cpp b/parser_library/test/processing/db2_preprocessor_test.cpp index e5ac9d635..77abec8ce 100644 --- a/parser_library/test/processing/db2_preprocessor_test.cpp +++ b/parser_library/test/processing/db2_preprocessor_test.cpp @@ -18,6 +18,7 @@ #include "../mock_parse_lib_provider.h" #include "preprocessor_options.h" #include "processing/preprocessor.h" +#include "semantics/source_info_processor.h" // test db2 preprocessor emulator using namespace hlasm_plugin::parser_library::processing; @@ -28,9 +29,30 @@ const auto copy1_loc = resource_location("COPY1"); const auto copy2_loc = resource_location("COPY2"); const auto member_loc = resource_location("MEMBER"); } // namespace -TEST(db2_preprocessor, first_line) + +class db2_preprocessor_test : public testing::Test { - auto p = preprocessor::create( +public: + db2_preprocessor_test() + : m_src_info(false) + {} + + std::unique_ptr create_preprocessor( + db2_preprocessor_options opts, library_fetcher libs, diagnostic_op_consumer_container* diags) + { + return preprocessor::create(opts, libs, diags, m_src_info, m_ids); + } + +protected: + semantics::source_info_processor m_src_info; + context::id_storage m_ids; + diagnostic_op_consumer_container m_diags; +}; + + +TEST_F(db2_preprocessor_test, first_line) +{ + auto p = create_preprocessor( db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, nullptr); std::string_view text = ""; @@ -43,9 +65,9 @@ TEST(db2_preprocessor, first_line) EXPECT_TRUE(std::all_of(result.begin(), result.end(), [](const auto& l) { return !l.is_original(); })); } -TEST(db2_preprocessor, last_line) +TEST_F(db2_preprocessor_test, last_line) { - auto p = preprocessor::create( + auto p = create_preprocessor( db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, nullptr); std::string_view text = "\n END "; @@ -58,9 +80,9 @@ TEST(db2_preprocessor, last_line) EXPECT_EQ(std::count_if(result.begin(), result.end(), [](const auto& l) { return l.text() == " END "; }), 1); } -TEST(db2_preprocessor, include) +TEST_F(db2_preprocessor_test, include) { - auto p = preprocessor::create( + auto p = create_preprocessor( db2_preprocessor_options {}, [](std::string_view s) { EXPECT_EQ(s, "MEMBER"); @@ -79,17 +101,17 @@ TEST(db2_preprocessor, include) 0); } -TEST(db2_preprocessor, include_sqlca) +TEST_F(db2_preprocessor_test, include_sqlca) { bool called = false; - auto p = preprocessor::create( + auto p = create_preprocessor( db2_preprocessor_options {}, [&called](std::string_view) { called = true; return std::nullopt; }, nullptr); - std::string_view text = "\n EXEC SQL INCLUDE SQLCA "; + std::string_view text = "\n EXEC SQL INCLUDE SqLcA "; auto result = p->generate_replacement(document(text)); @@ -99,17 +121,17 @@ TEST(db2_preprocessor, include_sqlca) 1); } -TEST(db2_preprocessor, include_sqlda) +TEST_F(db2_preprocessor_test, include_sqlda) { bool called = false; - auto p = preprocessor::create( + auto p = create_preprocessor( db2_preprocessor_options {}, [&called](std::string_view) { called = true; return std::nullopt; }, nullptr); - std::string_view text = "\n EXEC SQL INCLUDE SQLDA "; + std::string_view text = "\n EXEC SQL INCLUDE SqLdA "; auto result = p->generate_replacement(document(text)); @@ -119,10 +141,10 @@ TEST(db2_preprocessor, include_sqlda) 1); } -TEST(db2_preprocessor, sql_like) +TEST_F(db2_preprocessor_test, sql_like) { bool called = false; - auto p = preprocessor::create( + auto p = create_preprocessor( db2_preprocessor_options {}, [&called](std::string_view) { called = true; @@ -141,9 +163,9 @@ TEST(db2_preprocessor, sql_like) result.end()); } -TEST(db2_preprocessor, with_label) +TEST_F(db2_preprocessor_test, with_label) { - auto p = preprocessor::create( + auto p = create_preprocessor( db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, nullptr); std::string_view text = "\nABC EXEC SQL WHATEVER"; @@ -163,25 +185,23 @@ TEST(db2_preprocessor, with_label) result.end()); } -TEST(db2_preprocessor, missing_member) +TEST_F(db2_preprocessor_test, missing_member) { - diagnostic_op_consumer_container diags; - auto p = preprocessor::create( - db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, &diags); + auto p = create_preprocessor( + db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, &m_diags); std::string_view text = " EXEC SQL INCLUDE MISSING"; auto doc = p->generate_replacement(document(text)); EXPECT_NE(doc.size(), 0); - EXPECT_TRUE(matches_message_codes(diags.diags, { "DB002" })); + EXPECT_TRUE(matches_message_codes(m_diags.diags, { "DB002" })); } -TEST(db2_preprocessor, bad_continuation) +TEST_F(db2_preprocessor_test, bad_continuation) { - diagnostic_op_consumer_container diags; - auto p = preprocessor::create( - db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, &diags); + auto p = create_preprocessor( + db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, &m_diags); std::string_view text = R"( EXEC SQL PRETENT SQL STATEMENT X badcontinuation)"; @@ -189,25 +209,24 @@ badcontinuation)"; auto doc = p->generate_replacement(document(text)); EXPECT_NE(doc.size(), 0); - EXPECT_TRUE(matches_message_codes(diags.diags, { "DB001" })); + EXPECT_TRUE(matches_message_codes(m_diags.diags, { "DB001" })); } -TEST(db2_preprocessor, no_nested_include) +TEST_F(db2_preprocessor_test, no_nested_include) { - diagnostic_op_consumer_container diags; - auto p = preprocessor::create( + auto p = create_preprocessor( db2_preprocessor_options {}, [](std::string_view s) { EXPECT_EQ(s, "MEMBER"); return " EXEC SQL INCLUDE MEMBER"; }, - &diags); + &m_diags); std::string_view text = " EXEC SQL INCLUDE MEMBER "; auto doc = p->generate_replacement(document(text)); EXPECT_NE(doc.size(), 0); - EXPECT_TRUE(matches_message_codes(diags.diags, { "DB003" })); + EXPECT_TRUE(matches_message_codes(m_diags.diags, { "DB003" })); } TEST(db2_preprocessor, sqlsect_available) @@ -352,6 +371,22 @@ TEST(db2_preprocessor, include_empty) EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(member_loc)); } +TEST(db2_preprocessor, include_insensitive) +{ + mock_parse_lib_provider libs({ + { "MEMBER", "" }, + }); + std::string input = " EXEC SQL INCLUDE MeMbEr"; + + analyzer a(input, analyzer_options { &libs, db2_preprocessor_options {} }); + a.analyze(); + a.collect_diags(); + + EXPECT_EQ(a.diags().size(), (size_t)0); + + EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(member_loc)); +} + TEST(db2_preprocessor, include_nonexistent) { std::string input = " EXEC SQL INCLUDE MEMBER "; @@ -631,11 +666,10 @@ TEST(db2_preprocessor, end_sqldsect_injection) EXPECT_EQ(a.diags().size(), (size_t)0); } -TEST(db2_preprocessor, sql_types) +TEST_F(db2_preprocessor_test, sql_types) { - diagnostic_op_consumer_container diags; - auto p = preprocessor::create( - db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, &diags); + auto p = create_preprocessor( + db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, &m_diags); std::string_view text = R"( RE SQL TYPE IS RESULT_SET_LOCATOR VARYING RO SQL TYPE IS ROWID @@ -756,7 +790,7 @@ DFILE_NAME DS CL255 EXPECT_NE(doc.text().find(expected), std::string_view::npos); - EXPECT_TRUE(diags.diags.empty()); + EXPECT_TRUE(m_diags.diags.empty()); } TEST(db2_preprocessor, sql_types_with_space) @@ -824,9 +858,11 @@ TEST(db2_preprocessor, sql_type_fails) "A SQL TYPE IS TABLE LIKE AAA AS LOCATORR", }) { + semantics::source_info_processor src_info(false); + context::id_storage ids; diagnostic_op_consumer_container diags; auto p = preprocessor::create( - db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, &diags); + db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, &diags, src_info, ids); p->generate_replacement(document(text)); @@ -834,17 +870,16 @@ TEST(db2_preprocessor, sql_type_fails) } } -TEST(db2_preprocessor, sql_type_warn_on_continuation) +TEST_F(db2_preprocessor_test, sql_type_warn_on_continuation) { std::string_view text = "A SQL TYPE IS TABLE LIKE AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n" " AS LOCATOR"; - diagnostic_op_consumer_container diags; - auto p = preprocessor::create( - db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, &diags); + auto p = create_preprocessor( + db2_preprocessor_options {}, [](std::string_view) { return std::nullopt; }, &m_diags); p->generate_replacement(document(text)); - EXPECT_TRUE(matches_message_codes(diags.diags, { "DB005" })); + EXPECT_TRUE(matches_message_codes(m_diags.diags, { "DB005" })); } TEST(db2_preprocessor, no_codegen_for_unacceptable_sql_statement) @@ -929,9 +964,9 @@ RO_LEN EQU *-RO EXPECT_EQ(get_symbol_abs(a.hlasm_ctx(), "RO_LEN"), 42); } -TEST(db2_preprocessor, conditional) +TEST_F(db2_preprocessor_test, conditional) { - auto p = preprocessor::create( + auto p = create_preprocessor( db2_preprocessor_options("", true), [](std::string_view) { return std::nullopt; }, nullptr); std::string_view text = ""; diff --git a/parser_library/test/processing/endevor_preprocessor_test.cpp b/parser_library/test/processing/endevor_preprocessor_test.cpp index 237840c32..930b9ed60 100644 --- a/parser_library/test/processing/endevor_preprocessor_test.cpp +++ b/parser_library/test/processing/endevor_preprocessor_test.cpp @@ -22,119 +22,104 @@ using namespace hlasm_plugin::parser_library::processing; using namespace hlasm_plugin::utils::resource; -TEST(endevor_preprocessor, basic_inc) +class endevor_preprocessor_test : public testing::Test { - diagnostic_op_consumer_container diags; - - int callback_count = 0; - - auto p = preprocessor::create( - endevor_preprocessor_options(), - [&callback_count](std::string_view s) { - EXPECT_EQ(s, "AAA"); - ++callback_count; - return std::string("TEST"); - }, - &diags); +public: + endevor_preprocessor_test() + : m_src_info(false) + {} + + std::unique_ptr create_preprocessor(library_fetcher libs) + { + return preprocessor::create(endevor_preprocessor_options(), libs, &m_diags, m_src_info, m_ids); + } + +protected: + semantics::source_info_processor m_src_info; + context::id_storage m_ids; + diagnostic_op_consumer_container m_diags; + int m_callback_count = 0; +}; + +TEST_F(endevor_preprocessor_test, basic_inc) +{ + auto p = create_preprocessor([&callback_count = m_callback_count](std::string_view s) { + EXPECT_EQ(s, "AAA"); + ++callback_count; + return std::string("TEST"); + }); auto result = p->generate_replacement(document("-INC AAA")); - EXPECT_EQ(callback_count, 1); - EXPECT_TRUE(diags.diags.empty()); + EXPECT_EQ(m_callback_count, 1); + EXPECT_TRUE(m_diags.diags.empty()); ASSERT_EQ(result.size(), 1); EXPECT_EQ(result.at(0).text(), "TEST"); } -TEST(endevor_preprocessor, basic_include) +TEST_F(endevor_preprocessor_test, basic_include) { - diagnostic_op_consumer_container diags; - - int callback_count = 0; - - auto p = preprocessor::create( - endevor_preprocessor_options(), - [&callback_count](std::string_view s) { - EXPECT_EQ(s, "AAA"); - ++callback_count; - return std::string("TEST"); - }, - &diags); + auto p = create_preprocessor([&callback_count = m_callback_count](std::string_view s) { + EXPECT_EQ(s, "AAA"); + ++callback_count; + return std::string("TEST"); + }); auto result = p->generate_replacement(document("++INCLUDE AAA")); - EXPECT_EQ(callback_count, 1); - EXPECT_TRUE(diags.diags.empty()); + EXPECT_EQ(m_callback_count, 1); + EXPECT_TRUE(m_diags.diags.empty()); ASSERT_EQ(result.size(), 1); EXPECT_EQ(result.at(0).text(), "TEST"); } -TEST(endevor_preprocessor, missing_member) +TEST_F(endevor_preprocessor_test, missing_member) { - diagnostic_op_consumer_container diags; - - int callback_count = 0; - - auto p = preprocessor::create( - endevor_preprocessor_options(), - [&callback_count](std::string_view s) { - EXPECT_EQ(s, "AAA"); - ++callback_count; - return std::nullopt; - }, - &diags); + auto p = create_preprocessor([&callback_count = m_callback_count](std::string_view s) { + EXPECT_EQ(s, "AAA"); + ++callback_count; + return std::nullopt; + }); auto result = p->generate_replacement(document("++INCLUDE AAA\nBBB")); - EXPECT_EQ(callback_count, 1); + EXPECT_EQ(m_callback_count, 1); ASSERT_EQ(result.size(), 1); EXPECT_EQ(result.at(0).text(), "BBB"); - EXPECT_TRUE(matches_message_codes(diags.diags, { "END001" })); + EXPECT_TRUE(matches_message_codes(m_diags.diags, { "END001" })); } -TEST(endevor_preprocessor, cycle) +TEST_F(endevor_preprocessor_test, cycle) { - diagnostic_op_consumer_container diags; - - int callback_count = 0; - - auto p = preprocessor::create( - endevor_preprocessor_options(), - [&callback_count](std::string_view s) { - EXPECT_EQ(s, "AAA"); - ++callback_count; - return std::string("-INC AAA"); - }, - &diags); + auto p = create_preprocessor([&callback_count = m_callback_count](std::string_view s) { + EXPECT_EQ(s, "AAA"); + ++callback_count; + return std::string("-INC AaA"); + }); auto result = p->generate_replacement(document("++INCLUDE AAA")); - EXPECT_EQ(callback_count, 1); + EXPECT_EQ(m_callback_count, 1); EXPECT_EQ(result.size(), 0); - EXPECT_TRUE(matches_message_codes(diags.diags, { "END002" })); + EXPECT_TRUE(matches_message_codes(m_diags.diags, { "END002" })); } -TEST(endevor_preprocessor, nested) +TEST_F(endevor_preprocessor_test, nested) { - diagnostic_op_consumer_container diags; - - int callback_count = 0; - - auto p = preprocessor::create( - endevor_preprocessor_options(), - [&callback_count](std::string_view s) -> std::optional { + auto p = + create_preprocessor([&callback_count = m_callback_count](std::string_view s) -> std::optional { ++callback_count; if (s == "MEMBER") return "BBB\n-INC NESTED\nDDD"; if (s == "NESTED") return "CCC"; return std::nullopt; - }, - &diags); + }); auto result = p->generate_replacement(document("AAA\n-INC MEMBER\nEEE")); - EXPECT_EQ(callback_count, 2); - EXPECT_TRUE(diags.diags.empty()); + EXPECT_EQ(m_callback_count, 2); + EXPECT_TRUE(m_diags.diags.empty()); EXPECT_EQ(result.size(), 5); EXPECT_EQ(result.text(), "AAA\nBBB\nCCC\nDDD\nEEE\n"); } @@ -147,7 +132,7 @@ TESTVAL EQU 42 )" }, }); std::string input = R"( --INC MEMBER +-INC MeMbEr )"; analyzer a(input, analyzer_options { &libs, endevor_preprocessor_options {} }); diff --git a/parser_library/test/semantics/highlighting_test.cpp b/parser_library/test/semantics/highlighting_test.cpp index d91df5b59..34b48ad21 100644 --- a/parser_library/test/semantics/highlighting_test.cpp +++ b/parser_library/test/semantics/highlighting_test.cpp @@ -384,3 +384,22 @@ TEST(highlighting, multiline_macro_param) EXPECT_EQ(tokens, expected); } + +TEST(highlighting, endevor_preprocessor_statement) +{ + const std::string contents = R"( +-INC MEMBER bla bla)"; + + analyzer a( + contents, analyzer_options { source_file_loc, endevor_preprocessor_options(), collect_highlighting_info::yes }); + a.analyze(); + + const auto& tokens = a.source_processor().semantic_tokens(); + semantics::lines_info expected = { + token_info({ { 1, 0 }, { 1, 4 } }, hl_scopes::instruction), + token_info({ { 1, 6 }, { 1, 12 } }, hl_scopes::operand), + token_info({ { 1, 13 }, { 1, 20 } }, hl_scopes::remark), + }; + + EXPECT_EQ(tokens, expected); +}