From 8b576fab7da69e83b03aab1713d89715052f711d Mon Sep 17 00:00:00 2001 From: slavek-kucera <53339291+slavek-kucera@users.noreply.github.com> Date: Thu, 23 Mar 2023 15:27:53 +0100 Subject: [PATCH] fix: Closing a dependency without saving does not trigger reparsing --- benchmark/benchmark.cpp | 7 +- clients/vscode-hlasmplugin/CHANGELOG.md | 1 + .../src/parsing_metadata_serialization.cpp | 12 +- language_server/test/telemetry_test.cpp | 2 +- parser_library/fuzzer/fuzzer.cpp | 4 +- parser_library/include/protocol.h | 14 +- parser_library/src/analyzer.cpp | 6 +- parser_library/src/analyzer.h | 4 +- parser_library/src/context/hlasm_context.cpp | 19 +- parser_library/src/context/hlasm_context.h | 8 - .../src/debugging/debug_lib_provider.cpp | 4 +- .../src/debugging/debug_lib_provider.h | 4 +- .../src/processing/opencode_provider.cpp | 16 +- .../src/processing/opencode_provider.h | 5 +- .../src/processing/processing_manager.cpp | 2 - .../src/workspaces/parse_lib_provider.cpp | 7 +- .../src/workspaces/parse_lib_provider.h | 9 +- parser_library/src/workspaces/processor.h | 2 - .../src/workspaces/processor_file_impl.cpp | 19 -- .../src/workspaces/processor_file_impl.h | 10 +- parser_library/src/workspaces/workspace.cpp | 322 +++++++++++------- parser_library/src/workspaces/workspace.h | 28 +- parser_library/test/async_macro_parsing.cpp | 4 +- parser_library/test/gtest_stringers.cpp | 4 +- .../lsp/lsp_context_preprocessor_test.cpp | 46 ++- parser_library/test/metrics_test.cpp | 7 - parser_library/test/mock_parse_lib_provider.h | 44 ++- .../test/processing/db2_preprocessor_test.cpp | 27 +- .../test/workspace/processor_file_test.cpp | 4 +- .../test/workspace/workspace_test.cpp | 54 ++- utils/include/utils/CMakeLists.txt | 1 + utils/include/utils/transform_inserter.h | 55 +++ 32 files changed, 461 insertions(+), 290 deletions(-) create mode 100644 utils/include/utils/transform_inserter.h diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 0efc8ffde..22143db78 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -138,7 +138,7 @@ json parse_one_file(const std::string& source_file, + metrics.lookahead_statements + metrics.reparsed_statements; s.average_stmt_ms += (exec_statements / (double)time); s.average_line_ms += metrics.lines / (double)time; - s.all_files += metrics.files; + s.all_files += collector.data.ws_info.files_processed; s.whole_time += time; auto top_messages = benchmark::get_top_messages(diag_counter.message_counts); @@ -165,9 +165,10 @@ json parse_one_file(const std::string& source_file, { "Continued Statements", metrics.continued_statements }, { "Non-continued Statements", metrics.non_continued_statements }, { "Lines", metrics.lines }, - { "Files", metrics.files }, + { "Files", collector.data.ws_info.files_processed }, }); + auto first_ws_info = collector.data.ws_info; auto first_parse_metrics = metrics; auto first_diag_counter = diag_counter; long long reparse_time = 0; @@ -226,7 +227,7 @@ json parse_one_file(const std::string& source_file, << "Lines: " << first_parse_metrics.lines << '\n' << "Executed Statement/ms: " << exec_statements / (double)time << '\n' << "Line/ms: " << first_parse_metrics.lines / (double)time << '\n' - << "Files: " << first_parse_metrics.files << '\n' + << "Files: " << first_ws_info.files_processed << '\n' << "Top messages: " << top_messages.dump() << '\n' << '\n' << std::endl; diff --git a/clients/vscode-hlasmplugin/CHANGELOG.md b/clients/vscode-hlasmplugin/CHANGELOG.md index 82501da6c..142348712 100644 --- a/clients/vscode-hlasmplugin/CHANGELOG.md +++ b/clients/vscode-hlasmplugin/CHANGELOG.md @@ -9,6 +9,7 @@ - New or changed files are parsed only when they are opened in the editor - High CPU usage while going to the symbol definition - Inconsistent identification of inactive statements +- Closing a dependency without saving does not trigger reparsing ## [1.7.0](https://github.com/eclipse/che-che4z-lsp-for-hlasm/compare/1.6.0...1.7.0) (2023-03-08) diff --git a/language_server/src/parsing_metadata_serialization.cpp b/language_server/src/parsing_metadata_serialization.cpp index c54596962..8d62ebc62 100644 --- a/language_server/src/parsing_metadata_serialization.cpp +++ b/language_server/src/parsing_metadata_serialization.cpp @@ -20,14 +20,18 @@ namespace hlasm_plugin::parser_library { void to_json(nlohmann::json& j, const parser_library::workspace_file_info& info) { - j = nlohmann::json { { "config_parsing", info.config_parsing }, + j = nlohmann::json { + { "config_parsing", info.config_parsing }, { "diagnostics_suppressed", info.diagnostics_suppressed }, - { "processor_group_found", info.processor_group_found } }; + { "processor_group_found", info.processor_group_found }, + { "files_processed", info.files_processed }, + }; } void to_json(nlohmann::json& j, const parser_library::performance_metrics& metrics) { - j = nlohmann::json { { "Open Code Statements", metrics.open_code_statements }, + j = nlohmann::json { + { "Open Code Statements", metrics.open_code_statements }, { "Copy Statements", metrics.copy_statements }, { "Macro Statements", metrics.macro_statements }, { "Copy Def Statements", metrics.copy_def_statements }, @@ -37,7 +41,7 @@ void to_json(nlohmann::json& j, const parser_library::performance_metrics& metri { "Continued Statements", metrics.continued_statements }, { "Non-continued Statements", metrics.non_continued_statements }, { "Lines", metrics.lines }, - { "Files", metrics.files } }; + }; } void to_json(nlohmann::json& j, const parser_library::parsing_metadata& metadata) diff --git a/language_server/test/telemetry_test.cpp b/language_server/test/telemetry_test.cpp index 4592f3600..ec6174aab 100644 --- a/language_server/test/telemetry_test.cpp +++ b/language_server/test/telemetry_test.cpp @@ -78,12 +78,12 @@ TEST(telemetry, lsp_server_did_open) nlohmann::json& metrics = telemetry_reply["params"]["measurements"]; - EXPECT_EQ(metrics["Files"], 1); EXPECT_GT(metrics["duration"], 0U); EXPECT_EQ(metrics["error_count"], 1); nlohmann::json& ws_info = telemetry_reply["params"]["properties"]; + EXPECT_EQ(ws_info["files_processed"], 1); EXPECT_EQ(ws_info["diagnostics_suppressed"], false); } diff --git a/parser_library/fuzzer/fuzzer.cpp b/parser_library/fuzzer/fuzzer.cpp index 61d6ac009..f30d07183 100644 --- a/parser_library/fuzzer/fuzzer.cpp +++ b/parser_library/fuzzer/fuzzer.cpp @@ -70,7 +70,7 @@ class fuzzer_lib_provider : public parse_lib_provider callback(true); } - bool has_library(std::string_view library, resource_location* url) const override + bool has_library(std::string_view library, resource_location* url) override { auto lib = read_library_name(library); if (!lib.has_value()) @@ -81,7 +81,7 @@ class fuzzer_lib_provider : public parse_lib_provider } void get_library(std::string_view library, - std::function>)> callback) const override + std::function>)> callback) override { assert(callback); auto lib = read_library_name(library); diff --git a/parser_library/include/protocol.h b/parser_library/include/protocol.h index 89a87e598..13a31a040 100644 --- a/parser_library/include/protocol.h +++ b/parser_library/include/protocol.h @@ -278,11 +278,13 @@ struct PARSER_LIBRARY_EXPORT performance_metrics size_t lookahead_statements = 0; size_t continued_statements = 0; size_t non_continued_statements = 0; - size_t files = 0; + + bool operator==(const performance_metrics&) const noexcept = default; }; struct PARSER_LIBRARY_EXPORT workspace_file_info { + size_t files_processed = 0; bool config_parsing = false; bool diagnostics_suppressed = false; bool processor_group_found = false; @@ -294,16 +296,6 @@ struct PARSER_LIBRARY_EXPORT parsing_metadata workspace_file_info ws_info; }; -bool inline operator==(const performance_metrics& lhs, const performance_metrics& rhs) -{ - return lhs.lines == rhs.lines && lhs.macro_def_statements == rhs.macro_def_statements - && lhs.macro_statements == rhs.macro_statements && lhs.open_code_statements == rhs.open_code_statements - && lhs.copy_def_statements == rhs.copy_def_statements && lhs.copy_statements == rhs.copy_statements - && lhs.reparsed_statements == rhs.reparsed_statements && lhs.lookahead_statements == rhs.lookahead_statements - && lhs.continued_statements == rhs.continued_statements - && lhs.non_continued_statements == rhs.non_continued_statements && lhs.files == rhs.files; -} - struct PARSER_LIBRARY_EXPORT diagnostic_list { diagnostic_list(); diff --git a/parser_library/src/analyzer.cpp b/parser_library/src/analyzer.cpp index ec6ac216f..6d783d28c 100644 --- a/parser_library/src/analyzer.cpp +++ b/parser_library/src/analyzer.cpp @@ -170,11 +170,7 @@ void analyzer::collect_diags() const collect_diags_from_child(field_parser_); } -const performance_metrics& analyzer::get_metrics() const -{ - ctx_.hlasm_ctx->fill_metrics_files(); - return ctx_.hlasm_ctx->metrics; -} +const performance_metrics& analyzer::get_metrics() const { return ctx_.hlasm_ctx->metrics; } void analyzer::register_stmt_analyzer(statement_analyzer* stmt_analyzer) { diff --git a/parser_library/src/analyzer.h b/parser_library/src/analyzer.h index 99a83f45a..056821df4 100644 --- a/parser_library/src/analyzer.h +++ b/parser_library/src/analyzer.h @@ -148,13 +148,13 @@ class analyzer : public diagnosable_ctx processing::statement_fields_parser field_parser_; - std::vector vf_handles_; + std::vector> vf_handles_; processing::processing_manager mngr_; public: analyzer(std::string_view text, analyzer_options opts = {}); - std::vector take_vf_handles() { return std::move(vf_handles_); } + auto take_vf_handles() { return std::move(vf_handles_); } analyzing_context context() const; context::hlasm_context& hlasm_ctx(); diff --git a/parser_library/src/context/hlasm_context.cpp b/parser_library/src/context/hlasm_context.cpp index 654956969..0105a5acc 100644 --- a/parser_library/src/context/hlasm_context.cpp +++ b/parser_library/src/context/hlasm_context.cpp @@ -372,7 +372,6 @@ hlasm_context::hlasm_context( init_instruction_map(opcode_mnemo_, *ids_, asm_options_.instr_set); add_global_system_vars(scope_stack_.emplace_back()); - visited_files_.insert(file_loc); push_statement_processing(processing::processing_kind::ORDINARY, std::move(file_loc)); } @@ -642,8 +641,6 @@ std::vector hlasm_context::whole_copy_stack() const return ret; } -void hlasm_context::fill_metrics_files() { metrics.files = visited_files_.size(); } - const hlasm_context::global_variable_storage& hlasm_context::globals() const { return globals_; } var_sym_ptr hlasm_context::get_var_sym(id_index name) const @@ -894,8 +891,6 @@ macro_invo_ptr hlasm_context::enter_macro(id_index name, macro_data_ptr label_pa add_system_vars_to_scope(new_scope); add_global_system_vars(new_scope); - visited_files_.insert(macro_def->definition_location.resource_loc); - ++SYSNDX_; mnote_last_max = 0; @@ -927,24 +922,17 @@ const location* hlasm_context::current_macro_definition_location() const const utils::resource::resource_location& hlasm_context::opencode_location() const { return opencode_file_location_; } -const std::set& hlasm_context::get_visited_files() const { return visited_files_; } - copy_member_ptr hlasm_context::add_copy_member( id_index member, statement_block definition, location definition_location) { auto& copydef = copy_members_[member]; if (!copydef) copydef = std::make_shared(member, std::move(definition), definition_location); - visited_files_.insert(std::move(definition_location.resource_loc)); return copydef; } -void hlasm_context::add_copy_member(copy_member_ptr member) -{ - visited_files_.insert(member->definition_location.resource_loc); - copy_members_[member->name] = std::move(member); -} +void hlasm_context::add_copy_member(copy_member_ptr member) { copy_members_[member->name] = std::move(member); } copy_member_ptr hlasm_context::get_copy_member(id_index member) const @@ -968,11 +956,6 @@ const hlasm_context::copy_member_storage& hlasm_context::copy_members() { return void hlasm_context::leave_copy_member() { source_stack_.back().copy_stack.pop_back(); } -void hlasm_context::add_preprocessor_dependency(const utils::resource::resource_location& file_loc) -{ - visited_files_.emplace(file_loc); -} - void hlasm_context::apply_source_snapshot(source_snapshot snapshot) { assert(std::transform_reduce(source_stack_.begin(), diff --git a/parser_library/src/context/hlasm_context.h b/parser_library/src/context/hlasm_context.h index 09fa9ef6f..ab8e1c1ee 100644 --- a/parser_library/src/context/hlasm_context.h +++ b/parser_library/src/context/hlasm_context.h @@ -81,8 +81,6 @@ class hlasm_context // opencode file location utils::resource::resource_location opencode_file_location_; - // all files processes via macro or copy member invocation - std::set visited_files_; // Compiler options asm_option asm_options_; @@ -131,8 +129,6 @@ class hlasm_context // gets opencode file location const utils::resource::resource_location& opencode_location() const; - // accesses visited files - const std::set& get_visited_files() const; // gets current source const source_context& current_source() const; @@ -187,7 +183,6 @@ class hlasm_context // performance metrics performance_metrics metrics; - void fill_metrics_files(); // return map of global set vars const global_variable_storage& globals() const; @@ -259,9 +254,6 @@ class hlasm_context // leaves current copy member void leave_copy_member(); - // register preprocessor dependency - void add_preprocessor_dependency(const utils::resource::resource_location& file_loc); - // creates specified global set symbol template set_sym_ptr create_global_variable(id_index id, bool is_scalar) diff --git a/parser_library/src/debugging/debug_lib_provider.cpp b/parser_library/src/debugging/debug_lib_provider.cpp index 08072d8f6..3382bd93b 100644 --- a/parser_library/src/debugging/debug_lib_provider.cpp +++ b/parser_library/src/debugging/debug_lib_provider.cpp @@ -66,7 +66,7 @@ void debug_lib_provider::parse_library( callback(false); } -bool debug_lib_provider::has_library(std::string_view library, utils::resource::resource_location* loc) const +bool debug_lib_provider::has_library(std::string_view library, utils::resource::resource_location* loc) { for (const auto& lib : m_libraries) if (lib->has_file(library, loc)) @@ -75,7 +75,7 @@ bool debug_lib_provider::has_library(std::string_view library, utils::resource:: } void debug_lib_provider::get_library(std::string_view library, - std::function>)> callback) const + std::function>)> callback) { assert(callback); utils::resource::resource_location url; diff --git a/parser_library/src/debugging/debug_lib_provider.h b/parser_library/src/debugging/debug_lib_provider.h index 7b9d80f0f..86589bac3 100644 --- a/parser_library/src/debugging/debug_lib_provider.h +++ b/parser_library/src/debugging/debug_lib_provider.h @@ -63,11 +63,11 @@ class debug_lib_provider final : public workspaces::parse_lib_provider workspaces::library_data data, std::function callback) override; - bool has_library(std::string_view library, utils::resource::resource_location* loc) const override; + bool has_library(std::string_view library, utils::resource::resource_location* loc) override; void get_library(std::string_view library, std::function>)> callback) - const override; + override; }; } // namespace hlasm_plugin::parser_library::debugging diff --git a/parser_library/src/processing/opencode_provider.cpp b/parser_library/src/processing/opencode_provider.cpp index 8d36bb8ed..90e576fec 100644 --- a/parser_library/src/processing/opencode_provider.cpp +++ b/parser_library/src/processing/opencode_provider.cpp @@ -51,7 +51,7 @@ opencode_provider::opencode_provider(std::string_view text, std::unique_ptr prep, opencode_provider_options opts, virtual_file_monitor* virtual_file_monitor, - std::vector& vf_handles) + std::vector>& vf_handles) : statement_provider(statement_provider_kind::OPEN) , m_input_document(text) , m_singleline { parsing::parser_holder::create(&src_proc, ctx.hlasm_ctx.get(), &diag_consumer, false), @@ -522,11 +522,12 @@ utils::task opencode_provider::run_preprocessor() } else { + auto file_handle = m_virtual_file_monitor->file_generated(new_file->second); + auto file_location = generate_virtual_file_name(file_handle.file_id(), virtual_file_name.to_string_view()); + m_vf_handles.emplace_back(std::move(file_handle), file_location); return start_nested_parser(new_file->second, analyzer_options { - generate_virtual_file_name( - m_vf_handles.emplace_back(m_virtual_file_monitor->file_generated(new_file->second)).file_id(), - virtual_file_name.to_string_view()), + std::move(file_location), m_lib_provider, *m_ctx, workspaces::library_data { processing_kind::COPY, virtual_file_name }, @@ -597,11 +598,12 @@ utils::task opencode_provider::convert_ainsert_buffer_to_copybook() auto new_file = m_virtual_files.try_emplace(virtual_copy_name, std::move(result)).first; + auto file_handle = m_virtual_file_monitor->file_generated(new_file->second); + auto file_location = generate_virtual_file_name(file_handle.file_id(), virtual_copy_name.to_string_view()); + m_vf_handles.emplace_back(std::move(file_handle), file_location); co_await start_nested_parser(new_file->second, analyzer_options { - generate_virtual_file_name( - m_vf_handles.emplace_back(m_virtual_file_monitor->file_generated(new_file->second)).file_id(), - virtual_copy_name.to_string_view()), + std::move(file_location), m_lib_provider, *m_ctx, workspaces::library_data { processing_kind::COPY, virtual_copy_name }, diff --git a/parser_library/src/processing/opencode_provider.h b/parser_library/src/processing/opencode_provider.h index 143e32e60..a1f39a1f3 100644 --- a/parser_library/src/processing/opencode_provider.h +++ b/parser_library/src/processing/opencode_provider.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -134,7 +135,7 @@ class opencode_provider final : public statement_provider std::unique_ptr m_preprocessor; virtual_file_monitor* m_virtual_file_monitor; - std::vector& m_vf_handles; + std::vector>& m_vf_handles; public: // rewinds position in file @@ -152,7 +153,7 @@ class opencode_provider final : public statement_provider std::unique_ptr preprocessor, opencode_provider_options opts, virtual_file_monitor* virtual_file_monitor, - std::vector& vf_handles); + std::vector>& vf_handles); parsing::hlasmparser_multiline& parser(); // for testing only diff --git a/parser_library/src/processing/processing_manager.cpp b/parser_library/src/processing/processing_manager.cpp index d7db5105e..89e06dcca 100644 --- a/parser_library/src/processing/processing_manager.cpp +++ b/parser_library/src/processing/processing_manager.cpp @@ -254,8 +254,6 @@ void processing_manager::finish_preprocessor() static const context::statement_block stmt_block; - ctx_.hlasm_ctx->add_preprocessor_dependency(inc_member_details->loc); - ctx_.lsp_ctx->add_copy(std::make_shared(hlasm_ctx_.ids().add(inc_member_details->name), stmt_block, location(position(0, 0), inc_member_details->loc)), diff --git a/parser_library/src/workspaces/parse_lib_provider.cpp b/parser_library/src/workspaces/parse_lib_provider.cpp index 7fc0d4195..60aff50c0 100644 --- a/parser_library/src/workspaces/parse_lib_provider.cpp +++ b/parser_library/src/workspaces/parse_lib_provider.cpp @@ -25,12 +25,9 @@ void empty_parse_lib_provider::parse_library( assert(callback); callback(false); }; -bool empty_parse_lib_provider::has_library(std::string_view, utils::resource::resource_location*) const -{ - return false; -}; +bool empty_parse_lib_provider::has_library(std::string_view, utils::resource::resource_location*) { return false; }; void empty_parse_lib_provider::get_library(std::string_view, - std::function>)> callback) const + std::function>)> callback) { assert(callback); callback(std::nullopt); diff --git a/parser_library/src/workspaces/parse_lib_provider.h b/parser_library/src/workspaces/parse_lib_provider.h index 95e7767bd..82c1edd0c 100644 --- a/parser_library/src/workspaces/parse_lib_provider.h +++ b/parser_library/src/workspaces/parse_lib_provider.h @@ -46,11 +46,10 @@ class parse_lib_provider virtual void parse_library( std::string_view library, analyzing_context ctx, library_data data, std::function callback) = 0; - virtual bool has_library(std::string_view library, utils::resource::resource_location* url) const = 0; + virtual bool has_library(std::string_view library, utils::resource::resource_location* url) = 0; virtual void get_library(std::string_view library, - std::function>)> callback) - const = 0; + std::function>)> callback) = 0; protected: ~parse_lib_provider() = default; @@ -61,10 +60,10 @@ class empty_parse_lib_provider final : public parse_lib_provider { public: void parse_library(std::string_view, analyzing_context, library_data, std::function callback) override; - bool has_library(std::string_view, utils::resource::resource_location*) const override; + bool has_library(std::string_view, utils::resource::resource_location*) override; void get_library(std::string_view, std::function>)> callback) - const override; + override; static empty_parse_lib_provider instance; }; diff --git a/parser_library/src/workspaces/processor.h b/parser_library/src/workspaces/processor.h index 337d25a02..23f621e81 100644 --- a/parser_library/src/workspaces/processor.h +++ b/parser_library/src/workspaces/processor.h @@ -65,10 +65,8 @@ class processor : public virtual diagnosable class processor_file : public processor { public: - virtual const std::set& dependencies() = 0; virtual const semantics::lines_info& get_hl_info() = 0; virtual const lsp::lsp_context* get_lsp_context() const = 0; - virtual const std::set& files_to_close() = 0; virtual const performance_metrics& get_metrics() = 0; virtual bool has_opencode_lsp_info() const = 0; virtual bool has_macro_lsp_info() const = 0; diff --git a/parser_library/src/workspaces/processor_file_impl.cpp b/parser_library/src/workspaces/processor_file_impl.cpp index a580ab7ab..3cb30092e 100644 --- a/parser_library/src/workspaces/processor_file_impl.cpp +++ b/parser_library/src/workspaces/processor_file_impl.cpp @@ -57,8 +57,6 @@ bool processor_file_impl::parse(parse_lib_provider& lib_provider, fms, }); - auto old_dep = m_dependencies; - processing::hit_count_analyzer hc_analyzer(new_analyzer.hlasm_ctx()); new_analyzer.register_stmt_analyzer(&hc_analyzer); @@ -79,30 +77,13 @@ bool processor_file_impl::parse(parse_lib_provider& lib_provider, m_last_results.vf_handles = new_analyzer.take_vf_handles(); m_last_results.hc_opencode_map = hc_analyzer.take_hit_count_map(); - m_dependencies.clear(); - for (auto& file : new_analyzer.hlasm_ctx().get_visited_files()) - if (file != m_file->get_location()) - m_dependencies.insert(file); - - m_files_to_close.clear(); - // files that used to be dependencies but are not anymore should be closed internally - for (const auto& file : old_dep) - { - if (!m_dependencies.contains(file)) - m_files_to_close.insert(file); - } - return true; } -const std::set& processor_file_impl::dependencies() { return m_dependencies; } - const semantics::lines_info& processor_file_impl::get_hl_info() { return m_last_results.hl_info; } const lsp::lsp_context* processor_file_impl::get_lsp_context() const { return m_last_results.lsp_context.get(); } -const std::set& processor_file_impl::files_to_close() { return m_files_to_close; } - const performance_metrics& processor_file_impl::get_metrics() { return m_last_results.metrics; } bool processor_file_impl::should_collect_hl(context::hlasm_context* ctx) const diff --git a/parser_library/src/workspaces/processor_file_impl.h b/parser_library/src/workspaces/processor_file_impl.h index 13625a93d..a4c4ef1b1 100644 --- a/parser_library/src/workspaces/processor_file_impl.h +++ b/parser_library/src/workspaces/processor_file_impl.h @@ -48,11 +48,8 @@ class processor_file_impl final : public processor_file, public diagnosable_impl // Starts parser with new (empty) context bool parse(parse_lib_provider&, asm_option, std::vector, virtual_file_monitor*) override; - const std::set& dependencies() override; - const semantics::lines_info& get_hl_info() override; const lsp::lsp_context* get_lsp_context() const override; - const std::set& files_to_close() override; const performance_metrics& get_metrics() override; bool has_opencode_lsp_info() const override; @@ -71,6 +68,8 @@ class processor_file_impl final : public processor_file, public diagnosable_impl bool should_collect_hl(context::hlasm_context* ctx = nullptr) const; + auto take_vf_handles() noexcept { return std::move(m_last_results.vf_handles); } + private: file_manager& m_file_mngr; std::shared_ptr m_file; @@ -85,15 +84,12 @@ class processor_file_impl final : public processor_file, public diagnosable_impl std::shared_ptr> fade_messages = std::make_shared>(); performance_metrics metrics; - std::vector vf_handles; + std::vector> vf_handles; processing::hit_count_map hc_opencode_map; processing::hit_count_map hc_macro_map; } m_last_results; std::atomic* m_cancel; - - std::set m_dependencies; - std::set m_files_to_close; }; } // namespace hlasm_plugin::parser_library::workspaces diff --git a/parser_library/src/workspaces/workspace.cpp b/parser_library/src/workspaces/workspace.cpp index 7311e4a20..a5c523896 100644 --- a/parser_library/src/workspaces/workspace.cpp +++ b/parser_library/src/workspaces/workspace.cpp @@ -31,6 +31,7 @@ #include "utils/factory.h" #include "utils/levenshtein_distance.h" #include "utils/path.h" +#include "utils/transform_inserter.h" using hlasm_plugin::utils::resource::resource_location; using hlasm_plugin::utils::resource::resource_location_hasher; @@ -42,8 +43,14 @@ struct workspace_parse_lib_provider final : public parse_lib_provider workspace& ws; std::vector> libraries; workspace::processor_file_compoments& pfc; - std::unordered_map>, resource_location_hasher> - next_macro_cache; + + std::map, virtual_file_handle>, + std::less<>> + next_dependencies; + std::map> next_member_map; + std::unordered_map, resource_location_hasher, std::equal_to<>> + current_file_map; workspace_parse_lib_provider(workspace& ws, workspace::processor_file_compoments& pfc) : ws(ws) @@ -51,105 +58,140 @@ struct workspace_parse_lib_provider final : public parse_lib_provider , pfc(pfc) {} + void append_files_to_close(std::set& files_to_close) + { + std::set_difference(pfc.m_dependencies.begin(), + pfc.m_dependencies.end(), + next_dependencies.begin(), + next_dependencies.end(), + utils::transform_inserter( + files_to_close, [](const auto& v) -> const auto& { return v.first; }), + [](const auto& l, const auto& r) { return l.first < r.first; }); + } + + resource_location get_url(std::string_view library) + { + if (auto it = next_member_map.find(library); it != next_member_map.end()) + { + return it->second; + } + else if (resource_location url; std::none_of(libraries.begin(), + libraries.end(), + [&url, &library](const auto& lib) { return lib->has_file(library, &url); })) + { + next_member_map.emplace(library, resource_location()); + return resource_location(); + } + else + { + next_member_map.emplace(library, url); + return url; + } + } + + std::shared_ptr get_file(const resource_location& url) + { + return current_file_map + .try_emplace(url, utils::factory([this, &url]() { return ws.file_manager_.add_file(url); })) + .first->second; + } + + auto& get_cache(const resource_location& url, const std::shared_ptr& file) + { + return std::get>( + next_dependencies + .try_emplace(url, utils::factory([&url, &file, this]() { + auto version = file->get_version(); + if (auto it = pfc.m_dependencies.find(url); it != pfc.m_dependencies.end() + && std::get>(it->second)->version == version) + return std::get>(it->second); + + return std::make_shared(version, ws.get_file_manager(), file); + })) + .first->second) + ->cache; + } + // Inherited via parse_lib_provider void parse_library( std::string_view library, analyzing_context ctx, library_data data, std::function callback) override { assert(callback); - resource_location url; - for (const auto& lib : libraries) + resource_location url = get_url(library); + if (url.empty()) { - if (!lib->has_file(library, &url)) - continue; - - auto& macro_pfc = ws.add_processor_file_impl(url); - auto found = macro_pfc.m_processor_file; - assert(found); - - auto file = found->current_source(); + callback(false); + return; + } + std::shared_ptr file = get_file(url); + // TODO: if file is in error do something? - auto cache_key = macro_cache_key::create_from_context(*ctx.hlasm_ctx, data); + auto& macro_pfc = ws.add_processor_file_impl(file); + auto found = macro_pfc.m_processor_file; + assert(found); - auto& mc = next_macro_cache - .try_emplace(url, utils::factory([&url, &file, this]() { - auto version = file->get_version(); - if (auto it = pfc.m_macro_cache.find(url); - it != pfc.m_macro_cache.end() && it->second->first == version) - return it->second; + auto cache_key = macro_cache_key::create_from_context(*ctx.hlasm_ctx, data); - return std::make_shared>( - std::piecewise_construct, std::tie(version), std::tie(ws.get_file_manager(), file)); - })) - .first->second->second; + auto& mc = get_cache(url, file); - if (mc.load_from_cache(cache_key, ctx)) - { - callback(true); - return; - } + if (mc.load_from_cache(cache_key, ctx)) + { + callback(true); + return; + } - const bool collect_hl = found->should_collect_hl(ctx.hlasm_ctx.get()); - analyzer a(file->get_text(), - analyzer_options { - std::move(url), - this, - std::move(ctx), - data, - collect_hl ? collect_highlighting_info::yes : collect_highlighting_info::no, - }); + const bool collect_hl = found->should_collect_hl(ctx.hlasm_ctx.get()); + analyzer a(file->get_text(), + analyzer_options { + std::move(url), + this, + std::move(ctx), + data, + collect_hl ? collect_highlighting_info::yes : collect_highlighting_info::no, + }); - processing::hit_count_analyzer hc_analyzer(a.hlasm_ctx()); - a.register_stmt_analyzer(&hc_analyzer); + processing::hit_count_analyzer hc_analyzer(a.hlasm_ctx()); + a.register_stmt_analyzer(&hc_analyzer); - for (auto co_a = a.co_analyze(); !co_a.done(); co_a.resume()) + for (auto co_a = a.co_analyze(); !co_a.done(); co_a.resume()) + { + if (ws.cancel_ && ws.cancel_->load(std::memory_order_relaxed)) { - if (ws.cancel_ && ws.cancel_->load(std::memory_order_relaxed)) - { - callback(false); - return; - } + callback(false); + return; } + } - found->diags().clear(); - found->collect_diags_from_child(a); + found->diags().clear(); + found->collect_diags_from_child(a); - mc.save_macro(cache_key, a); - found->m_last_macro_analyzer_with_lsp = collect_hl; - if (collect_hl) - found->m_last_results.hl_info = a.take_semantic_tokens(); + mc.save_macro(cache_key, a); + found->m_last_macro_analyzer_with_lsp = collect_hl; + if (collect_hl) + found->m_last_results.hl_info = a.take_semantic_tokens(); - found->m_last_results.hc_macro_map = hc_analyzer.take_hit_count_map(); + found->m_last_results.hc_macro_map = hc_analyzer.take_hit_count_map(); - callback(true); - return; - } - - callback(false); + callback(true); } - bool has_library(std::string_view library, resource_location* loc) const override + + bool has_library(std::string_view library, resource_location* loc) override { - return std::any_of(libraries.begin(), libraries.end(), [&library, loc](const auto& lib) { - return lib->has_file(library, loc); - }); + auto url = get_url(library); + bool result = !url.empty(); + if (loc) + *loc = std::move(url); + return result; } + void get_library(std::string_view library, - std::function>)> callback) const override + std::function>)> callback) override { assert(callback); - resource_location url; - for (const auto& lib : libraries) - { - if (!lib->has_file(library, &url)) - continue; - - auto content = ws.file_manager_.get_file_content(url); - if (!content.has_value()) - break; - - callback(std::make_pair(std::move(content).value(), std::move(url))); - return; - } - callback(std::nullopt); + if (auto url = get_url(library); url.empty()) + callback(std::nullopt); + else + callback(std::make_pair(get_file(url)->get_text(), std::move(url))); } }; @@ -376,7 +418,7 @@ void workspace::retrieve_fade_messages(std::vector& fms) const bool take_also_opencode_hc = true; if (const auto& pf_rl = pf.get_location(); - &get_proc_grp_by_program(pf_rl) == &implicit_proc_grp && is_dependency_(pf_rl)) + &get_proc_grp_by_program(pf_rl) == &implicit_proc_grp && is_dependency(pf_rl)) take_also_opencode_hc = false; for (const auto& [__, opened_file_rl] : opened_files_uris) @@ -405,25 +447,28 @@ std::vector> workspace::find_related_opencodes( for (const auto& [_, component] : m_processor_files) { - if (component.m_processor_file->dependencies().contains(document_loc)) + if (component.m_dependencies.contains(document_loc)) opencodes.push_back(component.m_processor_file); } return opencodes; } -void workspace::delete_diags(std::shared_ptr file) +void workspace::delete_diags(processor_file_compoments& pfc) { - file->diags().clear(); + // TODO: + // this function just looks wrong, we delete diagnostics for dependencies + // regardless of in what files they are used + pfc.m_processor_file->diags().clear(); - for (const auto& dep : file->dependencies()) + for (const auto& [dep, _] : pfc.m_dependencies) { auto dep_file = find_processor_file(dep); if (dep_file) dep_file->diags().clear(); } - file->diags().push_back(diagnostic_s::info_SUP(file->get_location())); + pfc.m_processor_file->diags().push_back(diagnostic_s::info_SUP(pfc.m_processor_file->get_location())); } void workspace::show_message(const std::string& message) @@ -438,6 +483,7 @@ const ws_uri& workspace::uri() const { return location_.get_uri(); } void workspace::reparse_after_config_refresh() { + std::set files_to_close; // Reparse every opened file when configuration is changed for (auto& [fname, comp] : m_processor_files) { @@ -449,13 +495,12 @@ void workspace::reparse_after_config_refresh() if (!comp.m_processor_file->parse(ws_lib, get_asm_options(fname), get_preprocessor_options(fname), &fm_vfm_)) continue; + ws_lib.append_files_to_close(files_to_close); + (void)parse_successful(comp, std::move(ws_lib)); } - for (const auto& [_, component] : m_processor_files) - { - filter_and_close_dependencies_(component.m_processor_file->files_to_close(), component.m_processor_file); - } + filter_and_close_dependencies(std::move(files_to_close)); } namespace { @@ -477,7 +522,7 @@ std::vector workspace::populate_files_to_ { if (!component.m_opened) continue; - if (component.m_processor_file->dependencies().contains(file_location)) + if (component.m_dependencies.contains(file_location)) files_to_parse.push_back(&component); } } @@ -505,9 +550,9 @@ workspace_file_info workspace::parse_file(const resource_location& file_location } // TODO: what about removing files??? what if depentands_ points to not existing file? - auto files_to_parse = populate_files_to_parse(file_location, file_content_status); - for (auto* component : files_to_parse) + std::set files_to_close; + for (auto* component : populate_files_to_parse(file_location, file_content_status)) { assert(component); const auto& f = component->m_processor_file; @@ -519,12 +564,14 @@ workspace_file_info workspace::parse_file(const resource_location& file_location if (!f->parse(ws_lib, get_asm_options(f_loc), get_preprocessor_options(f_loc), &fm_vfm_)) continue; + ws_lib.append_files_to_close(files_to_close); + ws_file_info = parse_successful(*component, std::move(ws_lib)); } // second check after all dependants are there to close all files that used to be dependencies - for (const auto* component : files_to_parse) - filter_and_close_dependencies_(component->m_processor_file->files_to_close(), component->m_processor_file); + + filter_and_close_dependencies(std::move(files_to_close)); return ws_file_info; } @@ -541,10 +588,16 @@ workspace_file_info workspace::parse_successful(processor_file_compoments& comp, if (&grp == &implicit_proc_grp && (int64_t)f->diags().size() > get_config().diag_supress_limit) { ws_file_info.diagnostics_suppressed = true; - delete_diags(f); + delete_diags(comp); } - comp.m_macro_cache = std::move(libs.next_macro_cache); + for (auto&& [vfh, url] : f->take_vf_handles()) + libs.next_dependencies.try_emplace(std::move(url), std::move(vfh)); + + ws_file_info.files_processed = libs.next_dependencies.size() + 1; // TODO: identify error states? + + comp.m_dependencies = std::move(libs.next_dependencies); + comp.m_member_map = std::move(libs.next_member_map); return ws_file_info; } @@ -558,7 +611,7 @@ workspace_file_info workspace::did_open_file( const resource_location& file_location, open_file_result file_content_status) { if (!m_configuration.is_configuration_file(file_location)) - add_processor_file_impl(file_location).m_opened = true; + add_processor_file_impl(file_manager_.add_file(file_location)).m_opened = true; return parse_file(file_location, file_content_status); } @@ -571,21 +624,38 @@ void workspace::did_close_file(const resource_location& file_location) fcomp->second.m_opened = false; + bool found_dependency = false; // first check whether the file is a dependency - // if so, simply close it, no other action is needed - if (is_dependency_(file_location)) - return; + for (std::shared_ptr file; const auto& [_, component] : m_processor_files) + { + auto it = component.m_dependencies.find(file_location); + if (it == component.m_dependencies.end()) + continue; + if (!std::holds_alternative>(it->second)) + continue; + + found_dependency = true; - std::vector deps_to_cleanup; + if (!file) + file = file_manager_.add_file(file_location); + + if (file->get_version() == std::get>(it->second)->version) + continue; + + parse_file(file_location, open_file_result::changed_content); + break; + } + if (found_dependency) + return; // find if the file is a dependant const auto& file = fcomp->second.m_processor_file; - const auto& deps = file->dependencies(); + std::set files_to_close; + for (const auto& [dep, _] : fcomp->second.m_dependencies) + files_to_close.insert(dep); // filter the dependencies that should not be closed - filter_and_close_dependencies_(deps, file); - deps_to_cleanup.reserve(deps.size()); - deps_to_cleanup.assign(deps.begin(), deps.end()); + filter_and_close_dependencies(std::move(files_to_close), file.get()); // close the file itself m_processor_files.erase(fcomp); @@ -835,53 +905,58 @@ std::vector> workspace::make_opcode_suggestion( return result; } -template -void erase_unique_ordered(T& from, const T& what) +template +void erase_ordered(T& from, const U& what, Projector p = Projector()) { - for (auto f = from.begin(), w = what.begin(); f != from.end() && w != what.end();) + auto f = from.begin(); + auto w = what.begin(); + while (f != from.end() && w != what.end()) { - if (auto c = (*f) <=> (*w); c == 0) + if (auto c = *f <=> std::invoke(p, *w); c == 0) { f = from.erase(f); - ++w; + if constexpr (!Multi) + ++w; } else if (c < 0) - f = from.lower_bound(*w); + f = from.lower_bound(std::invoke(p, *w)); else w = what.lower_bound(*f); } } -void workspace::filter_and_close_dependencies_( - std::set dependencies, std::shared_ptr file) +void workspace::filter_and_close_dependencies( + std::set files_to_close_candidates, const processor_file_impl* file_to_ignore) { // filters the files that are dependencies of other dependants and externally open files for (const auto& [_, component] : m_processor_files) { - if (dependencies.empty()) + if (files_to_close_candidates.empty()) return; - if (component.m_opened) - dependencies.erase(component.m_processor_file->get_location()); - - if (component.m_processor_file->get_location() == file->get_location()) + if (component.m_processor_file.get() == file_to_ignore) continue; - erase_unique_ordered(dependencies, component.m_processor_file->dependencies()); + if (component.m_opened) + files_to_close_candidates.erase(component.m_processor_file->get_location()); + + erase_ordered(files_to_close_candidates, + component.m_dependencies, + &decltype(component.m_dependencies)::value_type::first); } // close all exclusive dependencies of file - for (const auto& dep : dependencies) + for (const auto& dep : files_to_close_candidates) { m_processor_files.erase(dep); } } -bool workspace::is_dependency_(const resource_location& file_location) const +bool workspace::is_dependency(const resource_location& file_location) const { for (const auto& [_, component] : m_processor_files) { - if (component.m_processor_file->dependencies().contains(file_location)) + if (component.m_dependencies.contains(file_location)) return true; } return false; @@ -923,16 +998,17 @@ std::vector workspace::get_preprocessor_options(const reso return get_proc_grp_by_program(file_location).preprocessors(); } -workspace::processor_file_compoments& workspace::add_processor_file_impl(const resource_location& file_location) +workspace::processor_file_compoments& workspace::add_processor_file_impl(std::shared_ptr f) { - if (auto p = find_processor_file_impl(file_location)) + const auto& loc = f->get_location(); + if (auto p = find_processor_file_impl(loc)) return *p; processor_file_compoments pfc { - std::make_shared(file_manager_.add_file(file_location), file_manager_, cancel_), + std::make_shared(std::move(f), file_manager_, cancel_), }; - return m_processor_files.insert_or_assign(file_location, std::move(pfc)).first->second; + return m_processor_files.insert_or_assign(loc, std::move(pfc)).first->second; } std::shared_ptr workspace::find_processor_file(const resource_location& file_location) const diff --git a/parser_library/src/workspaces/workspace.h b/parser_library/src/workspaces/workspace.h index 4e3d906a5..05f9c6209 100644 --- a/parser_library/src/workspaces/workspace.h +++ b/parser_library/src/workspaces/workspace.h @@ -16,6 +16,7 @@ #define HLASMPLUGIN_PARSERLIBRARY_WORKSPACE_H #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include "diagnosable_impl.h" @@ -133,11 +135,11 @@ class workspace : public diagnosable_impl void reparse_after_config_refresh(); - void filter_and_close_dependencies_(std::set dependencies, std::shared_ptr file); - bool is_dependency_(const resource_location& file_location) const; + void filter_and_close_dependencies( + std::set files_to_close_candidates, const processor_file_impl* file_to_ignore = nullptr); + bool is_dependency(const resource_location& file_location) const; std::vector> find_related_opencodes(const resource_location& document_loc) const; - void delete_diags(std::shared_ptr file); void show_message(const std::string& message); @@ -149,13 +151,22 @@ class workspace : public diagnosable_impl workspace_configuration m_configuration; + struct dependency_cache + { + dependency_cache(version_t version, const file_manager& fm, std::shared_ptr file) + : version(version) + , cache(fm, std::move(file)) + {} + version_t version; + macro_cache cache; + }; + struct processor_file_compoments { std::shared_ptr m_processor_file; - std::unordered_map>, - resource_location_hasher> - m_macro_cache; + std::map, virtual_file_handle>, std::less<>> + m_dependencies; + std::map> m_member_map; resource_location m_alternative_config = resource_location(); @@ -169,11 +180,12 @@ class workspace : public diagnosable_impl std::vector populate_files_to_parse( const utils::resource::resource_location& file_location, open_file_result file_content_status); - processor_file_compoments& add_processor_file_impl(const resource_location& file); + processor_file_compoments& add_processor_file_impl(std::shared_ptr f); processor_file_compoments* find_processor_file_impl(const resource_location& file); const processor_file_compoments* find_processor_file_impl(const resource_location& file) const; friend struct workspace_parse_lib_provider; workspace_file_info parse_successful(processor_file_compoments& comp, workspace_parse_lib_provider libs); + void delete_diags(processor_file_compoments& pfc); }; } // namespace hlasm_plugin::parser_library::workspaces diff --git a/parser_library/test/async_macro_parsing.cpp b/parser_library/test/async_macro_parsing.cpp index a4a9c50cb..5e926546c 100644 --- a/parser_library/test/async_macro_parsing.cpp +++ b/parser_library/test/async_macro_parsing.cpp @@ -52,7 +52,7 @@ struct async_macro_parsing_fixture : ::testing::Test, parse_lib_provider m_nested_analyzers.emplace_back(std::move(a_ptr), std::move(a_task)); } } - bool has_library(std::string_view library, resource::resource_location* url) const override + bool has_library(std::string_view library, resource::resource_location* url) override { auto it = m_files.find(library); if (it == m_files.end()) @@ -62,7 +62,7 @@ struct async_macro_parsing_fixture : ::testing::Test, parse_lib_provider return true; } void get_library(std::string_view library, - std::function>)> callback) const override + std::function>)> callback) override { if (auto it = m_files.find(library); it != m_files.end()) callback(std::make_pair(it->second, resource::resource_location(it->first))); diff --git a/parser_library/test/gtest_stringers.cpp b/parser_library/test/gtest_stringers.cpp index 2811afaf8..4f0209390 100644 --- a/parser_library/test/gtest_stringers.cpp +++ b/parser_library/test/gtest_stringers.cpp @@ -42,8 +42,8 @@ std::ostream& operator<<(std::ostream& stream, const performance_metrics& item) { return stream << "continued statements: " << item.continued_statements << "\n copy def statements: " << item.copy_def_statements - << "\n copy statements: " << item.copy_statements << "\n files: " << item.files - << "\n lines: " << item.lines << "\n lookahead statements: " << item.lookahead_statements + << "\n copy statements: " << item.copy_statements << "\n lines: " << item.lines + << "\n lookahead statements: " << item.lookahead_statements << "\n macro def statements: " << item.macro_def_statements << "\n macro statements: " << item.macro_statements << "\n non continued statements: " << item.non_continued_statements diff --git a/parser_library/test/lsp/lsp_context_preprocessor_test.cpp b/parser_library/test/lsp/lsp_context_preprocessor_test.cpp index 8144bb064..18e5b6005 100644 --- a/parser_library/test/lsp/lsp_context_preprocessor_test.cpp +++ b/parser_library/test/lsp/lsp_context_preprocessor_test.cpp @@ -38,10 +38,16 @@ const resource_location member_loc("MEMBER"); const resource_location member2_loc("MEMBER2"); const std::vector> member_list { - { "MEMBER", R"(R2 EQU 2 - LR R2,R2)" }, - { "MEMBER2", R"(R5 EQU 5 - LR R5,R5)" }, + { + "MEMBER", + R"(R2 EQU 2 + LR R2,R2)", + }, + { + "MEMBER2", + R"(R5 EQU 5 + LR R5,R5)", + }, }; class lsp_context_preprocessor_test : public testing::Test @@ -51,23 +57,38 @@ class lsp_context_preprocessor_test : public testing::Test std::shared_ptr lib_provider, preprocessor_options preproc_options) : lib_provider(lib_provider) - , a(contents, analyzer_options { source_loc, lib_provider.get(), preproc_options }) + , a(contents, analyzer_options { source_loc, lib_provider.get(), preproc_options, &vf_monitor }) {} - void SetUp() override { a.analyze(); } + void SetUp() override + { + a.analyze(); + vf_files = a.take_vf_handles(); + } protected: std::shared_ptr lib_provider; + struct : virtual_file_monitor + { + unsigned long long next_id = 12345; + std::vector generated_handles; + virtual_file_handle file_generated(std::string_view) final + { + generated_handles.emplace_back(next_id++); + + return virtual_file_handle(std::make_shared(generated_handles.back())); + } + } vf_monitor; analyzer a; + std::vector> vf_files; std::optional find_preproc_file(std::string_view name) { - const auto& files = a.hlasm_ctx().get_visited_files(); - - auto it = std::find_if( - files.begin(), files.end(), [name](const resource_location& f) { return f.get_uri().ends_with(name); }); + for (const auto& [_, url] : vf_files) + if (url.get_uri().ends_with(name)) + return url; - return it != files.end() ? std::make_optional(*it) : std::nullopt; + return std::nullopt; } inline bool reloc_symbol_checker(std::string_view source) @@ -340,7 +361,8 @@ C EXEC SQL INCLUDE SqLdA)"; void SetUp() override { - a.analyze(); + lsp_context_preprocessor_test::SetUp(); + preproc1_loc = find_preproc_file("PREPROCESSOR_1.hlasm"); ASSERT_TRUE(preproc1_loc.has_value()); diff --git a/parser_library/test/metrics_test.cpp b/parser_library/test/metrics_test.cpp index 9276d140e..7a11c386b 100644 --- a/parser_library/test/metrics_test.cpp +++ b/parser_library/test/metrics_test.cpp @@ -141,13 +141,6 @@ TEST_F(benchmark_test, continued_statements) EXPECT_EQ(a->get_metrics().non_continued_statements, (size_t)0); } -TEST_F(benchmark_test, files) -{ - setUpAnalyzer(" MAC\n COPY COPYFILE\n MAC"); - // only 3 files visited -> macro, copy and open code, each once - EXPECT_EQ(a->get_metrics().files, (size_t)3); -} - TEST_F(benchmark_test, reparsed_statements) { setUpAnalyzer(" MAC\n"); diff --git a/parser_library/test/mock_parse_lib_provider.h b/parser_library/test/mock_parse_lib_provider.h index 5aeaadc0a..f23ad906c 100644 --- a/parser_library/test/mock_parse_lib_provider.h +++ b/parser_library/test/mock_parse_lib_provider.h @@ -21,9 +21,27 @@ namespace hlasm_plugin::parser_library { +struct mock_file_stats_t +{ + size_t parse_requests = 0; + size_t existence_requests = 0; + size_t content_requests = 0; +}; + +struct mock_file_t +{ + mock_file_t(std::string c) + : content(std::move(c)) + {} + + std::string content; + + mutable mock_file_stats_t stats; +}; + class mock_parse_lib_provider : public workspaces::parse_lib_provider { - std::unordered_map> m_files; + std::unordered_map> m_files; public: std::unordered_map, utils::hashers::string_hasher, std::equal_to<>> @@ -50,7 +68,8 @@ class mock_parse_lib_provider : public workspaces::parse_lib_provider return; } - auto a = std::make_unique(it->second, + ++it->second.stats.parse_requests; + auto a = std::make_unique(it->second.content, analyzer_options { hlasm_plugin::utils::resource::resource_location(library), this, std::move(ctx), data }); a->analyze(); a->collect_diags(); @@ -59,10 +78,13 @@ class mock_parse_lib_provider : public workspaces::parse_lib_provider callback(true); } - bool has_library(std::string_view library, utils::resource::resource_location* loc) const override + bool has_library(std::string_view library, utils::resource::resource_location* loc) override { - if (!m_files.count(library)) + auto it = m_files.find(library); + if (it == m_files.end()) return false; + + ++it->second.stats.existence_requests; if (loc) *loc = utils::resource::resource_location(library); return true; @@ -71,7 +93,7 @@ class mock_parse_lib_provider : public workspaces::parse_lib_provider void get_library(std::string_view library, std::function>)> callback) - const override + override { auto it = m_files.find(library); if (it == m_files.end()) @@ -80,8 +102,18 @@ class mock_parse_lib_provider : public workspaces::parse_lib_provider return; } + ++it->second.stats.content_requests; callback(std::pair( - it->second, utils::resource::resource_location(library))); + it->second.content, utils::resource::resource_location(library))); + } + + std::optional get_stats(std::string_view library) const + { + auto it = m_files.find(library); + if (it == m_files.end()) + return std::nullopt; + else + return it->second.stats; } }; diff --git a/parser_library/test/processing/db2_preprocessor_test.cpp b/parser_library/test/processing/db2_preprocessor_test.cpp index fd1b6c5e7..ed04cec32 100644 --- a/parser_library/test/processing/db2_preprocessor_test.cpp +++ b/parser_library/test/processing/db2_preprocessor_test.cpp @@ -36,6 +36,7 @@ namespace { const auto copy1_loc = resource_location("COPY1"); const auto copy2_loc = resource_location("COPY2"); const auto member_loc = resource_location("MEMBER"); +const mock_file_stats_t invalid_stats { (size_t)-1, (size_t)-1, (size_t)-1 }; constexpr auto empty_library_fetcher = [](std::string) -> hlasm_plugin::utils::value_task< @@ -380,7 +381,7 @@ TEST(db2_preprocessor, continuation_in_buffer) EXPECT_EQ(a.diags().size(), (size_t)0); EXPECT_EQ(get_var_value(a.hlasm_ctx(), "A"), 1); - EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(member_loc)); + EXPECT_NE(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, -1); } TEST(db2_preprocessor, include_valid) @@ -412,7 +413,7 @@ TEST(db2_preprocessor, include_valid) EXPECT_EQ(a.diags().size(), (size_t)0); - EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(member_loc)); + EXPECT_NE(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, -1); } } @@ -439,7 +440,7 @@ TEST(db2_preprocessor, include_double) a.collect_diags(); EXPECT_TRUE(matches_message_codes(a.diags(), { "DB002" })); - EXPECT_EQ(a.hlasm_ctx().get_visited_files().count(member_loc), 0); + EXPECT_EQ(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, 0); } } @@ -459,7 +460,7 @@ TEST(db2_preprocessor, include_member_not_present) a.collect_diags(); EXPECT_TRUE(matches_message_codes(a.diags(), { "DB007" })); - EXPECT_EQ(a.hlasm_ctx().get_visited_files().count(member_loc), 0); + EXPECT_EQ(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, 0); } } @@ -475,7 +476,7 @@ TEST(db2_preprocessor, include_insensitive) a.collect_diags(); EXPECT_EQ(a.diags().size(), (size_t)0); - EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(member_loc)); + EXPECT_NE(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, -1); } TEST(db2_preprocessor, include_nonexistent) @@ -500,7 +501,7 @@ TEST(db2_preprocessor, include_invalid) a.analyze(); a.collect_diags(); - EXPECT_EQ(a.hlasm_ctx().get_visited_files().count(member_loc), 0); + EXPECT_EQ(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, 0); } TEST(db2_preprocessor, ago_in_include) @@ -520,7 +521,7 @@ TEST(db2_preprocessor, ago_in_include) EXPECT_EQ(a.diags().size(), (size_t)0); - EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(member_loc)); + EXPECT_NE(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, -1); EXPECT_EQ(get_var_value(a.hlasm_ctx(), "A"), 1); } @@ -544,7 +545,7 @@ TEST(db2_preprocessor, ago_into_include) EXPECT_EQ(a.diags().size(), (size_t)0); - EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(member_loc)); + EXPECT_NE(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, -1); EXPECT_EQ(get_var_value(a.hlasm_ctx(), "A"), 1); EXPECT_EQ(get_var_value(a.hlasm_ctx(), "B"), 2); } @@ -569,7 +570,7 @@ TEST(db2_preprocessor, ago_from_include) EXPECT_EQ(a.diags().size(), (size_t)0); - EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(member_loc)); + EXPECT_NE(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, -1); EXPECT_EQ(get_var_value(a.hlasm_ctx(), "A"), 1); EXPECT_EQ(get_var_value(a.hlasm_ctx(), "B"), 2); } @@ -598,7 +599,7 @@ TEST(db2_preprocessor, ago_around_include) EXPECT_EQ(a.diags().size(), (size_t)0); - EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(member_loc)); + EXPECT_NE(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, -1); EXPECT_EQ(get_var_value(a.hlasm_ctx(), "A"), 2); } @@ -625,9 +626,9 @@ TEST(db2_preprocessor, copy_in_include) EXPECT_EQ(a.diags().size(), (size_t)0); - EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(copy1_loc)); - EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(copy2_loc)); - EXPECT_TRUE(a.hlasm_ctx().get_visited_files().count(member_loc)); + EXPECT_NE(libs.get_stats("COPY1").value_or(invalid_stats).content_requests, -1); + EXPECT_NE(libs.get_stats("COPY2").value_or(invalid_stats).content_requests, -1); + EXPECT_NE(libs.get_stats("MEMBER").value_or(invalid_stats).content_requests, -1); EXPECT_EQ(get_var_value(a.hlasm_ctx(), "A1"), 1); EXPECT_EQ(get_var_value(a.hlasm_ctx(), "A2"), 2); diff --git a/parser_library/test/workspace/processor_file_test.cpp b/parser_library/test/workspace/processor_file_test.cpp index 5a4a35231..0a23c0b5b 100644 --- a/parser_library/test/workspace/processor_file_test.cpp +++ b/parser_library/test/workspace/processor_file_test.cpp @@ -64,7 +64,7 @@ TEST(processor_file, parse_macro) EXPECT_CALL(*library, has_file(std::string_view("MAC"), _)) .WillRepeatedly(DoAll(SetArgPointee<1>(macro_loc), Return(true))); - ws.did_open_file(opencode_loc, open_file_result::changed_content); + auto wf_info = ws.did_open_file(opencode_loc, open_file_result::changed_content); // Opencode file tests @@ -85,13 +85,13 @@ TEST(processor_file, parse_macro) EXPECT_EQ(ws.semantic_tokens(opencode_loc), open_expected_hl); performance_metrics expected_metrics; - expected_metrics.files = 2; expected_metrics.lines = 6; expected_metrics.macro_def_statements = 4; expected_metrics.macro_statements = 2; expected_metrics.non_continued_statements = 6; expected_metrics.open_code_statements = 2; EXPECT_EQ(ws.find_processor_file(opencode_loc)->get_metrics(), expected_metrics); + EXPECT_EQ(wf_info.files_processed, 2); // Macro file tests semantics::lines_info macro_expected_hl { diff --git a/parser_library/test/workspace/workspace_test.cpp b/parser_library/test/workspace/workspace_test.cpp index fc6f3dd8d..95090d058 100644 --- a/parser_library/test/workspace/workspace_test.cpp +++ b/parser_library/test/workspace/workspace_test.cpp @@ -21,6 +21,7 @@ #include "utils/path.h" #include "utils/platform.h" #include "utils/resource_location.h" +#include "workspaces/file_manager.h" #include "workspaces/file_manager_impl.h" #include "workspaces/workspace.h" @@ -240,18 +241,24 @@ const resource_location faulty_macro_loc("lib/ERROR"); const resource_location correct_macro_loc("lib/CORRECT"); } // namespace -class file_manager_extended : public file_manager_impl +class file_manager_extended : public file_manager_impl, public external_file_reader { + std::map file_contents = { + { proc_grps_loc, pgroups_file }, + { pgm_conf_loc, pgmconf_file }, + { source1_loc, source_using_macro_file }, + { source2_loc, source_using_macro_file }, + { source3_loc, source_using_macro_file_no_error }, + { faulty_macro_loc, faulty_macro_file }, + { correct_macro_loc, correct_macro_file }, + }; + public: file_manager_extended() + : file_manager_impl(static_cast(*this)) { - did_open_file(proc_grps_loc, 1, pgroups_file); - did_open_file(pgm_conf_loc, 1, pgmconf_file); - did_open_file(source1_loc, 1, source_using_macro_file); - did_open_file(source2_loc, 1, source_using_macro_file); - did_open_file(source3_loc, 1, source_using_macro_file_no_error); - did_open_file(faulty_macro_loc, 1, faulty_macro_file); - did_open_file(correct_macro_loc, 1, correct_macro_file); + for (const auto& [loc, content] : file_contents) + did_open_file(loc, 1, content); } list_directory_result list_directory_files(const hlasm_plugin::utils::resource::resource_location&) const override @@ -262,6 +269,14 @@ class file_manager_extended : public file_manager_impl return { { { "ERROR", faulty_macro_loc } }, hlasm_plugin::utils::path::list_directory_rc::done }; } + + std::optional load_text(const resource_location& document_loc) const override + { + if (auto it = file_contents.find(document_loc); it != file_contents.end()) + return it->second; + return std::nullopt; + } + bool insert_correct_macro = true; }; @@ -371,6 +386,29 @@ TEST_F(workspace_test, did_close_file) ASSERT_EQ(collect_and_get_diags_size(ws), (size_t)0); } +TEST_F(workspace_test, did_close_file_without_save) +{ + file_manager_extended file_manager; + workspace ws(empty_loc, "workspace_name", file_manager, config, global_settings); + + ws.open(); + + ws.did_open_file(source3_loc); + ws.did_open_file(correct_macro_loc); + EXPECT_EQ(collect_and_get_diags_size(ws), 0); + + document_change c(range(), "ERR", 3); + file_manager.did_change_file(correct_macro_loc, 2, &c, 1); + ws.did_change_file(correct_macro_loc, &c, 1); + + EXPECT_EQ(collect_and_get_diags_size(ws), (size_t)1); + EXPECT_TRUE(match_strings({ correct_macro_loc })); + + file_manager.did_close_file(correct_macro_loc); + ws.did_close_file(correct_macro_loc); + EXPECT_EQ(collect_and_get_diags_size(ws), 0); +} + TEST_F(workspace_test, did_change_watched_files) { file_manager_extended file_manager; diff --git a/utils/include/utils/CMakeLists.txt b/utils/include/utils/CMakeLists.txt index d40d155a5..d3424e4be 100644 --- a/utils/include/utils/CMakeLists.txt +++ b/utils/include/utils/CMakeLists.txt @@ -28,6 +28,7 @@ target_sources(hlasm_utils PUBLIC task.h text_matchers.h time.h + transform_inserter.h truth_table.h unicode_text.h ) diff --git a/utils/include/utils/transform_inserter.h b/utils/include/utils/transform_inserter.h new file mode 100644 index 000000000..55c94eff5 --- /dev/null +++ b/utils/include/utils/transform_inserter.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 Broadcom. + * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Broadcom, Inc. - initial API and implementation + */ + +#ifndef HLASMPLUGIN_UTILS_TRANSFORM_INSERTER_H +#define HLASMPLUGIN_UTILS_TRANSFORM_INSERTER_H + +#include +#include +#include + +namespace hlasm_plugin::utils { + +template +class transform_inserter +{ + C* container; + [[no_unique_address]] Transform transform; + +public: + using iterator_category = std::output_iterator_tag; + using value_type = void; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = void; + + transform_inserter(C& c, Transform t) + : container(&c) + , transform(std::move(t)) + {} + + template + transform_inserter& operator=(V&& value) + { + container->insert(std::invoke(transform, std::forward(value))); + return *this; + } + transform_inserter& operator*() { return *this; } + transform_inserter& operator++() { return *this; } + transform_inserter& operator++(int) { return *this; } +}; + +} // namespace hlasm_plugin::utils + +#endif