feat: Telemetry reporting (#187)

.github/workflows/CI.yml

@@ -200,6 +200,11 @@ jobs:
         run: cd clients/vscode-hlasmplugin && npm ci && npx semantic-release --dry-run
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Inject telemetry key
+        run: |
+          cd clients/vscode-hlasmplugin/src
+          export TEL_KEY=`node -e "console.log(Buffer.from('${{ secrets.TELEMETRY_KEY }}').toString('base64'))"`
+          sed -i "s/const TELEMETRY_KEY_ENCODED = TELEMETRY_DEFAULT_KEY/const TELEMETRY_KEY_ENCODED = '$TEL_KEY'/" telemetry.ts
       - name: Package VSIX
         run: cd clients/vscode-hlasmplugin/ && npm ci && npm run package
       - name: Get VSIX filename
@@ -231,6 +236,11 @@ jobs:
         run: cd clients/vscode-hlasmplugin && npm ci && npx semantic-release --dry-run
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Inject telemetry key
+        run: |
+          cd clients/vscode-hlasmplugin/src
+          export TEL_KEY=`node -e "console.log(Buffer.from('${{ secrets.TELEMETRY_KEY }}').toString('base64'))"`
+          sed -i "s/const TELEMETRY_KEY_ENCODED = TELEMETRY_DEFAULT_KEY/const TELEMETRY_KEY_ENCODED = '$TEL_KEY'/" telemetry.ts
       - name: Configure
         run: mkdir build && cd build && cmake -G Ninja ../
       - name: Build

benchmark/CMakeLists.txt

@@ -12,7 +12,9 @@
 
 project(benchmark)
 
-add_executable(benchmark ${PROJECT_SOURCE_DIR}/benchmark.cpp)
+add_executable(benchmark
+    benchmark.cpp
+    diagnostic_counter.h)
 
 target_link_libraries(benchmark nlohmann_json::nlohmann_json)
 

benchmark/benchmark.cpp

@@ -22,7 +22,10 @@
 #include <sstream>
 #include <unordered_map>
 
+#include "../language_server/src/parsing_metadata_collector.h"
+#include "../language_server/src/parsing_metadata_serialization.h"
 #include "config/pgm_conf.h"
+#include "diagnostic_counter.h"
 #include "nlohmann/json.hpp"
 #include "workspace_manager.h"
 #include "workspaces/file_impl.h"
@@ -59,47 +62,11 @@
  * - Files                    - total number of parsed files
  */
 
-using json = nlohmann::json;
-
-class diagnostic_counter : public hlasm_plugin::parser_library::diagnostics_consumer
-{
-public:
-    void consume_diagnostics(hlasm_plugin::parser_library::diagnostic_list diagnostics) override
-    {
-        for (size_t i = 0; i < diagnostics.diagnostics_size(); i++)
-        {
-            auto diag_sev = diagnostics.diagnostics(i).severity();
-            if (diag_sev == hlasm_plugin::parser_library::diagnostic_severity::error)
-                error_count++;
-            else if (diag_sev == hlasm_plugin::parser_library::diagnostic_severity::warning)
-                warning_count++;
-            message_counts[diagnostics.diagnostics(i).code()]++;
-        }
-    }
+using namespace hlasm_plugin;
 
-    void clear_counters()
-    {
-        error_count = 0;
-        warning_count = 0;
-        message_counts.clear();
-    }
-
-    size_t error_count = 0;
-    size_t warning_count = 0;
-
-    std::unordered_map<std::string, unsigned> message_counts;
-};
-
-class metrics_collector : public hlasm_plugin::parser_library::performance_metrics_consumer
-{
-public:
-    void consume_performance_metrics(const hlasm_plugin::parser_library::performance_metrics& metrics) override
-    {
-        metrics_ = metrics;
-    }
+using json = nlohmann::json;
 
-    hlasm_plugin::parser_library::performance_metrics metrics_;
-};
+namespace {
 
 struct all_file_stats
 {
@@ -113,21 +80,6 @@ struct all_file_stats
     size_t failed_file_opens = 0;
 };
 
-json get_top_messages(const std::unordered_map<std::string, unsigned>& msgs, size_t limit = 3)
-{
-    std::vector<std::pair<std::string, unsigned>> top_msgs(limit);
-
-    constexpr const auto cmp_msg = [](const auto& a, const auto& b) { return a.second > b.second; };
-
-    const auto last = std::partial_sort_copy(msgs.begin(), msgs.end(), top_msgs.begin(), top_msgs.end(), cmp_msg);
-    top_msgs.erase(last, top_msgs.end());
-
-    json result = json::object();
-    for (auto&& [key, value] : top_msgs)
-        result[std::move(key)] = value;
-    return result;
-}
-
 json parse_one_file(const std::string& source_file,
     const std::string& ws_folder,
     all_file_stats& s,
@@ -150,10 +102,10 @@ json parse_one_file(const std::string& source_file,
 
     // new workspace manager
     hlasm_plugin::parser_library::workspace_manager ws;
-    diagnostic_counter diag_counter;
+    hlasm_plugin::benchmark::diagnostic_counter diag_counter;
     ws.register_diagnostics_consumer(&diag_counter);
-    metrics_collector collector;
-    ws.register_performance_metrics_consumer(&collector);
+    hlasm_plugin::language_server::parsing_metadata_collector collector;
+    ws.register_parsing_metadata_consumer(&collector);
     // input folder as new workspace
     ws.add_workspace(ws_folder.c_str(), ws_folder.c_str());
 
@@ -179,42 +131,36 @@ json parse_one_file(const std::string& source_file,
         return json({ { "File", source_file }, { "Success", false }, { "Reason", "Crash" } });
     }
 
+    const auto& metrics = collector.data.metrics;
+
     auto c_end = std::clock();
     auto end = std::chrono::high_resolution_clock::now();
     auto time = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
-    auto exec_statements = collector.metrics_.open_code_statements + collector.metrics_.copy_statements
-        + collector.metrics_.macro_statements + collector.metrics_.lookahead_statements
-        + collector.metrics_.reparsed_statements;
+    auto exec_statements = metrics.open_code_statements + metrics.copy_statements + metrics.macro_statements
+        + metrics.lookahead_statements + metrics.reparsed_statements;
     s.average_stmt_ms += (exec_statements / (double)time);
-    s.average_line_ms += collector.metrics_.lines / (double)time;
-    s.all_files += collector.metrics_.files;
+    s.average_line_ms += metrics.lines / (double)time;
+    s.all_files += metrics.files;
     s.whole_time += time;
 
-    auto top_messages = get_top_messages(diag_counter.message_counts);
+    auto top_messages = hlasm_plugin::benchmark::get_top_messages(diag_counter.message_counts);
 
     json result({ { "File", source_file },
         { "Success", true },
         { "Errors", diag_counter.error_count },
         { "Warnings", diag_counter.warning_count },
         { "Wall Time (ms)", time },
         { "CPU Time (ms/n)", 1000.0 * (c_end - c_start) / CLOCKS_PER_SEC },
-        { "Open Code Statements", collector.metrics_.open_code_statements },
-        { "Copy Statements", collector.metrics_.copy_statements },
-        { "Macro Statements", collector.metrics_.macro_statements },
-        { "Copy Def Statements", collector.metrics_.copy_def_statements },
-        { "Macro Def Statements", collector.metrics_.macro_def_statements },
-        { "Lookahead Statements", collector.metrics_.lookahead_statements },
-        { "Reparsed Statements", collector.metrics_.reparsed_statements },
-        { "Continued Statements", collector.metrics_.continued_statements },
-        { "Non-continued Statements", collector.metrics_.non_continued_statements },
         { "Executed Statements", exec_statements },
-        { "Lines", collector.metrics_.lines },
         { "ExecStatement/ms", exec_statements / (double)time },
-        { "Line/ms", collector.metrics_.lines / (double)time },
-        { "Files", collector.metrics_.files },
+        { "Line/ms", metrics.lines / (double)time },
         { "Top messages", std::move(top_messages) } });
 
-    auto first_parse_metrics = collector.metrics_;
+    json metrics_json(metrics);
+
+    result.insert(metrics_json.begin(), metrics_json.end());
+
+    auto first_parse_metrics = metrics;
     auto first_diag_counter = diag_counter;
     long long reparse_time = 0;
     // Reparse to benchmark macro caching
@@ -286,6 +232,7 @@ std::string get_file_message(size_t iter, size_t begin, size_t end, const std::s
     s << "[" << base_message << " " << iter << "/(" << begin << "-" << end << ")] ";
     return s.str();
 }
+} // namespace
 
 int main(int argc, char** argv)
 {
@@ -449,4 +396,4 @@ int main(int argc, char** argv)
         std::clog << "Parse finished\n\n" << std::endl;
     }
     return 0;
-}
+}

benchmark/diagnostic_counter.h

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2021 Broadcom.
+ * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Broadcom, Inc. - initial API and implementation
+ */
+
+#ifndef HLASMPLUGIN_LANGUAGESERVER_DIAGNOSTIC_COUNTER_H
+#define HLASMPLUGIN_LANGUAGESERVER_DIAGNOSTIC_COUNTER_H
+
+#include "nlohmann/json.hpp"
+#include "workspace_manager.h"
+
+namespace hlasm_plugin::benchmark {
+
+class diagnostic_counter : public hlasm_plugin::parser_library::diagnostics_consumer
+{
+public:
+    void consume_diagnostics(hlasm_plugin::parser_library::diagnostic_list diagnostics) override
+    {
+        for (size_t i = 0; i < diagnostics.diagnostics_size(); i++)
+        {
+            if (auto diag_sev = diagnostics.diagnostics(i).severity();
+                diag_sev == hlasm_plugin::parser_library::diagnostic_severity::error)
+                error_count++;
+            else if (diag_sev == hlasm_plugin::parser_library::diagnostic_severity::warning)
+                warning_count++;
+            message_counts[diagnostics.diagnostics(i).code()]++;
+        }
+    }
+
+    void clear_counters()
+    {
+        error_count = 0;
+        warning_count = 0;
+        message_counts.clear();
+    }
+
+    size_t error_count = 0;
+    size_t warning_count = 0;
+
+    std::unordered_map<std::string, unsigned> message_counts;
+};
+
+inline nlohmann::json get_top_messages(const std::unordered_map<std::string, unsigned>& msgs, size_t limit = 3)
+{
+    std::vector<std::pair<std::string, unsigned>> top_msgs(limit);
+
+    constexpr const auto cmp_msg = [](const auto& a, const auto& b) { return a.second > b.second; };
+
+    const auto last = std::partial_sort_copy(msgs.begin(), msgs.end(), top_msgs.begin(), top_msgs.end(), cmp_msg);
+    top_msgs.erase(last, top_msgs.end());
+
+    nlohmann::json result = nlohmann::json::object();
+    for (auto&& [key, value] : top_msgs)
+        result[std::move(key)] = value;
+    return result;
+}
+
+
+} // namespace hlasm_plugin::benchmark
+
+
+
+#endif

clients/vscode-hlasmplugin/README.md

@@ -244,3 +244,30 @@ It can by configured using the `preprocessor` key in processor group:
 - If you have a question about how to accomplish something with the extension, or come across a problem, file an issue on [GitHub](https://github.com/eclipse/che-che4z-lsp-for-hlasm)
 - Contributions are always welcome! Please see our [GitHub](https://github.com/eclipse/che-che4z-lsp-for-hlasm) repository for more information.
 - Any and all feedback is appreciated and welcome!
+
+## Privacy Notice
+The extensions for Visual Studio Code developed by Broadcom Inc., including its corporate affiliates and subsidiaries, ("Broadcom") are provided free of charge, but in order to better understand and meet its users’ needs, Broadcom may collect, use, analyze and retain anonymous users’ metadata and interaction data, (collectively, “Usage Data”) and aggregate such Usage Data with similar Usage Data of other Broadcom customers. Please find more detailed information in [License and Service Terms & Repository](https://www.broadcom.com/company/legal/licensing).
+
+This data collection uses built-in Microsoft VS Code Telemetry, which can be disabled, at your sole discretion, if you do not want to send Usage Data. See the `telemetry.telemetryLevel` and `telemetry.enableTelemetry` (deprecated) settings of VS Code. 
+
+The current release of HLASM Language Support collects anonymous data for the following events:
+* Activation of this VS Code extension
+* Open and close of files
+* Invocation of the following features: Autocomplete, Go to definition, Find all references and Hover
+* Launch of Macro tracer and subsequent user interactions with the debugging interface (step into, step over, continue, set breakpoints)
+* Malfunctions of the extension
+
+Each such event is logged with the following information:
+* Event time
+* Operating system and version
+* Country or region
+* Anonymous user and session ID
+* Version numbers of Microsoft VS Code and HLASM Language Support
+* Extension response time
+
+Additionally, when a file is opened, the following information are logged:
+* Number of diagnostics (errors and warnings)
+* Number of files that were parsed including macros and COPY files
+* Number of parsed lines
+* Whether the diagnostics were suppressed
+* Whether matching processor group was found in the configuration

clients/vscode-hlasmplugin/package.json

@@ -59,6 +59,7 @@
     "vscode-test": "^1.3.0"
   },
   "dependencies": {
+    "vscode-extension-telemetry": "^0.4.2",
     "vscode-languageclient": "^5.2.1"
   },
   "scripts": {