feat: Endevor preprocessor statements highlighting and parsing

clients/vscode-hlasmplugin/CHANGELOG.md

@@ -6,6 +6,7 @@
 - Command for downloading copybooks allows selections of data sets which should be downloaded
 - Code actions for an unknown operation code
 - Quick fixes for typos in instruction and macro names added to the code actions
+- Endevor preprocessor statements highlighting and parsing
 
 ## [1.5.0](https://github.com/eclipse/che-che4z-lsp-for-hlasm/compare/1.4.0...1.5.0) (2022-11-02)
 

language_server/src/lsp/lsp_server.h

@@ -54,7 +54,7 @@ class server final : public hlasm_plugin::language_server::server,
     void respond(const json& id, const std::string& requested_method, const json& args) override;
     // Sends notification to LSP client using send_message_provider.
     void notify(const std::string& method, const json& args) override;
-    // Sends errorous respond to LSP client using send_message_provider.
+    // Sends erroneous respond to LSP client using send_message_provider.
     void respond_error(const json& id,
         const std::string& requested_method,
         int err_code,

parser_library/src/analyzer.cpp

@@ -50,13 +50,15 @@ workspaces::parse_lib_provider& analyzer_options::get_lib_provider() const
         return workspaces::empty_parse_lib_provider::instance;
 }
 
-std::unique_ptr<processing::preprocessor> analyzer_options::get_preprocessor(
-    processing::library_fetcher asm_lf, diagnostic_op_consumer& diag_consumer) const
+std::unique_ptr<processing::preprocessor> analyzer_options::get_preprocessor(processing::library_fetcher asm_lf,
+    diagnostic_op_consumer& diag_consumer,
+    semantics::source_info_processor& src_proc,
+    context::id_storage& ids) const
 {
-    const auto transform_preprocessor = [&asm_lf, &diag_consumer](const preprocessor_options& po) {
+    const auto transform_preprocessor = [&asm_lf, &diag_consumer, &src_proc, &ids](const preprocessor_options& po) {
         return std::visit(
-            [&asm_lf, &diag_consumer](const auto& p) -> std::unique_ptr<processing::preprocessor> {
-                return processing::preprocessor::create(p, std::move(asm_lf), &diag_consumer);
+            [&asm_lf, &diag_consumer, &src_proc, &ids](const auto& p) -> std::unique_ptr<processing::preprocessor> {
+                return processing::preprocessor::create(p, std::move(asm_lf), &diag_consumer, src_proc, ids);
             },
             po);
     };
@@ -65,16 +67,29 @@ std::unique_ptr<processing::preprocessor> analyzer_options::get_preprocessor(
     else if (preprocessor_args.size() == 1)
         return transform_preprocessor(preprocessor_args.front());
 
-    struct combined_preprocessor : processing::preprocessor
+    struct combined_preprocessor final : processing::preprocessor
     {
         std::vector<std::unique_ptr<processing::preprocessor>> pp;
+
         document generate_replacement(document doc) override
         {
+            clear_statements();
+
             for (const auto& p : pp)
                 doc = p->generate_replacement(std::move(doc));
+
             return doc;
         }
+
+        std::vector<std::shared_ptr<semantics::preprocessor_statement_si>> take_statements() override
+        {
+            for (const auto& p : pp)
+                set_statements(p->take_statements());
+
+            return preprocessor::take_statements();
+        }
     } tmp;
+
     std::transform(
         preprocessor_args.begin(), preprocessor_args.end(), std::back_inserter(tmp.pp), transform_preprocessor);
 
@@ -103,7 +118,9 @@ analyzer::analyzer(const std::string& text, analyzer_options opts)
 
                         return result;
                     },
-                    *this),
+                    *this,
+                    src_proc_,
+                    ctx_.hlasm_ctx->ids()),
                 opts.parsing_opencode == file_is_opencode::yes ? processing::opencode_provider_options { true, 10 }
                                                                : processing::opencode_provider_options {},
                 opts.vf_monitor),

parser_library/src/analyzer.h

@@ -74,8 +74,10 @@ class analyzer_options
     context::hlasm_context& get_hlasm_context();
     analyzing_context& get_context();
     workspaces::parse_lib_provider& get_lib_provider() const;
-    std::unique_ptr<processing::preprocessor> get_preprocessor(
-        processing::library_fetcher, diagnostic_op_consumer&) const;
+    std::unique_ptr<processing::preprocessor> get_preprocessor(processing::library_fetcher,
+        diagnostic_op_consumer&,
+        semantics::source_info_processor&,
+        context::id_storage& ids) const;
 
     friend class analyzer;
 

parser_library/src/context/copy_member.h

@@ -26,7 +26,7 @@ struct copy_member;
 // structure represents COPY member in HLASM macro library
 struct copy_member
 {
-    // member idenifier
+    // member identifier
     const id_index name;
     // block of statements defining the member
     cached_block cached_definition;

parser_library/src/context/statement_cache.h

@@ -26,7 +26,7 @@ struct complete_statement;
 namespace hlasm_plugin::parser_library::context {
 
 // storage used to store one deferred statement in many parsed formats
-// used by macro and copy definition to avoid multiple re-parsing of a deferrend stataments
+// used by macro and copy definition to avoid multiple re-parsing of a deferred statements
 class statement_cache
 {
 public:

parser_library/src/diagnostic.cpp

@@ -2708,10 +2708,22 @@ diagnostic_s diagnostic_s::warning_L0006(const utils::resource::resource_locatio
         diagnostic_tag::none);
 }
 
+diagnostic_s diagnostic_s::fade(const utils::resource::resource_location& loc, const range& range)
+{
+    return diagnostic_s(loc.get_uri(),
+        range,
+        diagnostic_severity::hint,
+        "PREPROC",
+        "Statement overwritten by preprocessor",
+        {},
+        diagnostic_tag::unnecessary); // todo move and make more verbose
+}
+
 diagnostic_op diagnostic_op::error_S100(std::string_view message, const range& range)
 {
     return diagnostic_op(diagnostic_severity::error, "S100", concat("Long ordinary symbol name - ", message), range);
 }
+
 std::string diagnostic_decorate_message(std::string_view field, std::string_view message)
 {
     static const std::string_view prefix = "While evaluating the result of substitution '";

parser_library/src/diagnostic.h

@@ -896,6 +896,8 @@ class diagnostic_s
 
     static diagnostic_s info_SUP(const utils::resource::resource_location& file_name);
 
+    static diagnostic_s fade(const utils::resource::resource_location& loc, const range& range);
+
     /*
     E01x - wrong format
     - E010 - unknown name

parser_library/src/processing/instruction_sets/asm_processor.cpp

@@ -350,7 +350,7 @@ void asm_processor::process_data_instruction(rebuilt_statement stmt)
     // Why is this so complicated?
     // 1. We cannot represent the individual operands because of bitfields.
     // 2. We cannot represent the whole area as a single dependency when the alignment requirements are growing.
-    // Therefore, we split the operands into chunks depending on the alignent.
+    // Therefore, we split the operands into chunks depending on the alignment.
     // Whenever the alignment requirement increases between consecutive operands, we start a new chunk.
     for (auto it = operands.begin(); it != operands.end();)
     {
@@ -723,51 +723,79 @@ void asm_processor::process(std::shared_ptr<const processing::resolved_statement
     }
 }
 
-bool asm_processor::process_copy(const semantics::complete_statement& stmt,
-    analyzing_context ctx,
+bool asm_processor::parse_copy(analyzing_context ctx,
     workspaces::parse_lib_provider& lib_provider,
+    context::id_index copy_member_id,
+    const range& operand_range,
+    const range& stmt_range,
     diagnosable_ctx* diagnoser)
 {
-    auto& expr = stmt.operands_ref().value.front()->access_asm()->access_expr()->expression;
-    auto sym_expr = dynamic_cast<expressions::mach_expr_symbol*>(expr.get());
-
-    if (!sym_expr)
-    {
-        if (diagnoser)
-            diagnoser->add_diagnostic(diagnostic_op::error_E058(stmt.operands_ref().value.front()->operand_range));
-        return false;
-    }
-
-    auto tmp = ctx.hlasm_ctx->copy_members().find(sym_expr->value);
+    auto tmp = ctx.hlasm_ctx->copy_members().find(copy_member_id);
 
     if (tmp == ctx.hlasm_ctx->copy_members().end())
     {
         bool result = lib_provider.parse_library(
-            sym_expr->value.to_string(), ctx, workspaces::library_data { processing_kind::COPY, sym_expr->value });
+            copy_member_id.to_string(), ctx, workspaces::library_data { processing_kind::COPY, copy_member_id });
 
         if (!result)
         {
             if (diagnoser)
-                diagnoser->add_diagnostic(diagnostic_op::error_E058(stmt.operands_ref().value.front()->operand_range));
+                diagnoser->add_diagnostic(diagnostic_op::error_E058(operand_range));
             return false;
         }
     }
+
     auto whole_copy_stack = ctx.hlasm_ctx->whole_copy_stack();
 
-    auto cycle_tmp = std::find(whole_copy_stack.begin(), whole_copy_stack.end(), sym_expr->value);
+    auto cycle_tmp = std::find(whole_copy_stack.begin(), whole_copy_stack.end(), copy_member_id);
 
     if (cycle_tmp != whole_copy_stack.end())
     {
         if (diagnoser)
-            diagnoser->add_diagnostic(diagnostic_op::error_E062(stmt.stmt_range_ref()));
+            diagnoser->add_diagnostic(diagnostic_op::error_E062(stmt_range));
         return false;
     }
 
-    ctx.hlasm_ctx->enter_copy_member(sym_expr->value);
-
     return true;
 }
 
+bool asm_processor::process_copy(const semantics::complete_statement& stmt,
+    analyzing_context ctx,
+    workspaces::parse_lib_provider& lib_provider,
+    diagnosable_ctx* diagnoser,
+    bool enter_copy)
+{
+    if (stmt.operands_ref().value.size() != 1 || !stmt.operands_ref().value.front()->access_asm()
+        || !stmt.operands_ref().value.front()->access_asm()->access_expr())
+    {
+        if (diagnoser)
+            diagnoser->add_diagnostic(diagnostic_op::error_E058(stmt.operands_ref().field_range));
+        return false;
+    }
+
+    auto& expr = stmt.operands_ref().value.front()->access_asm()->access_expr()->expression;
+    auto sym_expr = dynamic_cast<const expressions::mach_expr_symbol*>(expr.get());
+
+    if (!sym_expr)
+    {
+        if (diagnoser)
+            diagnoser->add_diagnostic(diagnostic_op::error_E058(stmt.operands_ref().value.front()->operand_range));
+        return false;
+    }
+
+    auto result = parse_copy(ctx,
+        lib_provider,
+        sym_expr->value,
+        stmt.operands_ref().value.front()->operand_range,
+        stmt.stmt_range_ref(),
+        diagnoser);
+
+    if (result && enter_copy)
+        ctx.hlasm_ctx->enter_copy_member(sym_expr->value);
+
+    return result;
+}
+
 asm_processor::process_table_t asm_processor::create_table()
 {
     process_table_t table;

parser_library/src/processing/instruction_sets/asm_processor.h

@@ -39,10 +39,18 @@ class asm_processor : public low_language_processor
 
     void process(std::shared_ptr<const processing::resolved_statement> stmt) override;
 
+    static bool parse_copy(analyzing_context ctx,
+        workspaces::parse_lib_provider& lib_provider,
+        context::id_index copy_member_id,
+        const range& operand_range,
+        const range& stmt_range,
+        diagnosable_ctx* diagnoser);
+
     static bool process_copy(const semantics::complete_statement& stmt,
         analyzing_context ctx,
         workspaces::parse_lib_provider& lib_provider,
-        diagnosable_ctx* diagnoser);
+        diagnosable_ctx* diagnoser,
+        bool enter_copy = true);
 
 private:
     opencode_provider* open_code_;

parser_library/src/processing/opencode_provider.cpp

@@ -212,7 +212,7 @@ std::shared_ptr<const context::hlasm_statement> opencode_provider::process_looka
     const range& op_range)
 {
     m_ctx->hlasm_ctx->set_source_position(collector.current_instruction().field_range.start);
-    auto proc_status = proc.get_processing_status(collector.peek_instruction());
+    auto proc_status = proc.get_processing_status(collector.current_instruction());
 
     if (op_text
         && proc_status.first.form != processing_form::IGNORED
@@ -265,7 +265,7 @@ std::shared_ptr<const context::hlasm_statement> opencode_provider::process_ordin
         return nullptr;
 
     m_ctx->hlasm_ctx->set_source_position(collector.current_instruction().field_range.start);
-    auto proc_status = proc.get_processing_status(collector.peek_instruction());
+    auto proc_status = proc.get_processing_status(collector.current_instruction());
 
     if (op_text)
     {
@@ -597,6 +597,13 @@ bool opencode_provider::finished() const
     return std::none_of(o.begin(), o.end(), [](const auto& c) { return c.suspended(); });
 }
 
+std::vector<std::shared_ptr<semantics::preprocessor_statement_si>>
+opencode_provider::get_preprocessor_statements() const
+{
+    return m_preprocessor ? m_preprocessor->take_statements()
+                          : std::vector<std::shared_ptr<semantics::preprocessor_statement_si>>();
+}
+
 parsing::hlasmparser_multiline& opencode_provider::parser()
 {
     if (!m_line_fed)

parser_library/src/processing/opencode_provider.h

@@ -139,6 +139,8 @@ class opencode_provider final : public statement_provider
 
     bool finished() const override;
 
+    std::vector<std::shared_ptr<semantics::preprocessor_statement_si>> get_preprocessor_statements() const;
+
 private:
     void feed_line(const parsing::parser_holder& p, bool is_process);
     bool is_comment();

parser_library/src/processing/preprocessor.cpp

@@ -15,6 +15,8 @@
 #include "preprocessor.h"
 
 #include "lexing/logical_line.h"
+#include "semantics/source_info_processor.h"
+#include "semantics/statement.h"
 #include "utils/unicode_text.h"
 
 namespace hlasm_plugin::parser_library::processing {
@@ -42,4 +44,40 @@ bool preprocessor::is_continued(std::string_view s)
     return !cont.empty() && cont != " ";
 }
 
+void preprocessor::clear_statements() { m_statements.clear(); }
+
+void preprocessor::set_statement(std::shared_ptr<semantics::preprocessor_statement_si> stmt)
+{
+    m_statements.emplace_back(std::move(stmt));
+}
+
+void preprocessor::set_statements(std::vector<std::shared_ptr<semantics::preprocessor_statement_si>> stmts)
+{
+    m_statements.insert(
+        m_statements.end(), std::make_move_iterator(stmts.begin()), std::make_move_iterator(stmts.end()));
+}
+
+std::vector<std::shared_ptr<semantics::preprocessor_statement_si>> preprocessor::take_statements()
+{
+    return std::move(m_statements);
+}
+
+void preprocessor::do_highlighting(
+    const semantics::preprocessor_statement_si& stmt, semantics::source_info_processor& src_proc)
+{
+    if (stmt.label_ref().type != semantics::label_si_type::EMPTY)
+        src_proc.add_hl_symbol(token_info(stmt.label_ref().field_range, semantics::hl_scopes::label));
+
+    src_proc.add_hl_symbol(token_info(stmt.instruction_ref().field_range, semantics::hl_scopes::instruction));
+
+    for (const auto& op : stmt.operands_ref().value)
+    {
+        if (op)
+            src_proc.add_hl_symbol(token_info(op->operand_range, semantics::hl_scopes::operand));
+    }
+
+    if (stmt.remarks_ref().value.size())
+        src_proc.add_hl_symbol(token_info(stmt.remarks_ref().field_range, semantics::hl_scopes::remark));
+}
+
 } // namespace hlasm_plugin::parser_library::processing