diff --git a/analyzer/codechecker_analyzer/analysis_manager.py b/analyzer/codechecker_analyzer/analysis_manager.py index c087262fa2..680685f1e1 100644 --- a/analyzer/codechecker_analyzer/analysis_manager.py +++ b/analyzer/codechecker_analyzer/analysis_manager.py @@ -17,11 +17,14 @@ import traceback import zipfile +from pathlib import Path from threading import Timer import multiprocess import psutil +import plistlib + from codechecker_common.logger import get_logger from codechecker_common.review_status_handler import ReviewStatusHandler @@ -57,8 +60,40 @@ def print_analyzer_statistic_summary(metadata_analyzers, status, msg=None): LOG.info(" %s: %s", analyzer_type, res) -def worker_result_handler(results, metadata_tool, output_path): - """ Print the analysis summary. """ +def merge_plists(results, output_path, plist_file_name): + """ + Merge the plist files generated by the analyzers into a single plist file. + Deletes the original plist files after merging. + """ + LOG.info("Merging plist files into %s", plist_file_name) + plist_data = [] + single_plist = Path(output_path, plist_file_name) + for _, _, _, _, original_plist, _ in results: + original_plist = Path(original_plist) + if os.path.exists(original_plist): + with open(original_plist, 'rb') as plist: + LOG.debug(f"Merging original plist {original_plist}") + plist_data.append(plistlib.load(plist)) + + with open(single_plist, 'wb') as plist: + LOG.debug(f"Dumping merged plist file into {single_plist}") + plistlib.dump(plist_data, plist) + + LOG.debug(f"Removing original plist file {original_plist}") + original_plist.unlink() + + +def worker_result_handler(results, + metadata_tool, + output_path, + plist_file_name): + """ + Handle analysis results after all the analyzer threads returned. It may + merge all the plist output files into one, and print the analysis summary. + """ + if plist_file_name: + merge_plists(results, output_path, plist_file_name) + skipped_num = 0 reanalyzed_num = 0 metadata_analyzers = metadata_tool['analyzers'] @@ -719,8 +754,8 @@ def skip_cpp(compile_actions, skip_handlers): return analyze, skip -def start_workers(actions_map, actions, analyzer_config_map, - jobs, output_path, skip_handlers, filter_handlers, +def start_workers(actions_map, actions, analyzer_config_map, jobs, + output_path, plist_file_name, skip_handlers, filter_handlers, rs_handler: ReviewStatusHandler, metadata_tool, quiet_analyze, capture_analysis_output, generate_reproducer, timeout, ctu_reanalyze_on_failure, statistics_data, manager, @@ -815,7 +850,10 @@ def signal_handler(signum, _): analyzed_actions, 1, callback=lambda results: worker_result_handler( - results, metadata_tool, output_path) + results, + metadata_tool, + output_path, + plist_file_name) ).get(timeout) pool.close() diff --git a/analyzer/codechecker_analyzer/analyzer.py b/analyzer/codechecker_analyzer/analyzer.py index b788461e9f..a07f1c863f 100644 --- a/analyzer/codechecker_analyzer/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzer.py @@ -333,6 +333,7 @@ def perform_analysis(args, skip_handlers, filter_handlers, analysis_manager.start_workers(actions_map, actions, config_map, args.jobs, args.output_path, + args.plist_file_name, skip_handlers, filter_handlers, rs_handler, diff --git a/analyzer/codechecker_analyzer/buildlog/build_action.py b/analyzer/codechecker_analyzer/buildlog/build_action.py index 692e164a6a..1a43fcd5a8 100644 --- a/analyzer/codechecker_analyzer/buildlog/build_action.py +++ b/analyzer/codechecker_analyzer/buildlog/build_action.py @@ -76,3 +76,6 @@ def with_attr(self, attr, value): details = {key: getattr(self, key) for key in BuildAction.__slots__} details[attr] = value return BuildAction(**details) + + def __repr__(self): + return str(self) diff --git a/analyzer/codechecker_analyzer/cmd/analyze.py b/analyzer/codechecker_analyzer/cmd/analyze.py index 440018e243..13dc3a57c2 100644 --- a/analyzer/codechecker_analyzer/cmd/analyze.py +++ b/analyzer/codechecker_analyzer/cmd/analyze.py @@ -220,6 +220,16 @@ def add_arguments_to_parser(parser): default=argparse.SUPPRESS, help="Store the analysis output in the given folder.") + parser.add_argument('--plist-file-name', + type=str, + dest="plist_file_name", + default='', + required=False, + help="If given, all the `.plist` files containing " + "the analyzer result files will be merged " + "into a single `.plist` file in the report " + "output folder given by `-o/--output`.") + parser.add_argument('--compiler-info-file', dest="compiler_info_file", required=False, diff --git a/analyzer/codechecker_analyzer/cmd/check.py b/analyzer/codechecker_analyzer/cmd/check.py index fa2ef4e505..7fde58677c 100644 --- a/analyzer/codechecker_analyzer/cmd/check.py +++ b/analyzer/codechecker_analyzer/cmd/check.py @@ -112,6 +112,16 @@ def add_arguments_to_parser(parser): "temporary directory which will be removed after " "the analysis.") + parser.add_argument('--plist-file-name', + type=str, + dest="plist_file_name", + required=False, + default='', + help="If given, all the `.plist` files containing " + "the analyzer result files will be merged " + "into a single `.plist` file in the report " + "output folder given by `-o/--output`.") + parser.add_argument('-t', '--type', '--output-format', dest="output_format", required=False, @@ -915,6 +925,7 @@ def __update_if_key_exists(source, target, key): 'skipfile', 'drop_skipped_reports', 'files', + 'plist_file_name', 'analyzers', 'add_compiler_defaults', 'cppcheck_args_cfg_file', diff --git a/analyzer/tests/functional/analyze/test_analyze.py b/analyzer/tests/functional/analyze/test_analyze.py index ecabdaec4f..61cfe8ac44 100644 --- a/analyzer/tests/functional/analyze/test_analyze.py +++ b/analyzer/tests/functional/analyze/test_analyze.py @@ -21,6 +21,8 @@ import unittest import zipfile +from pathlib import Path + from libtest import env from codechecker_report_converter.report import report_file @@ -1200,6 +1202,50 @@ def test_disable_all_checkers(self): # Checkers of all 3 analyzers are disabled. self.assertEqual(out.count("No checkers enabled for"), 5) + def test_single_plist(self): + """ + Test if specified with the `--plist-file-name` flag. + Analyze output should contain the indication of merging. + Merged plist should be created at the end of the analysis. + Only one `.plist` should remain at the end of the analysis. + """ + build_json = os.path.join( + self.test_workspace, "build_success.json") + source_file = os.path.join( + self.test_dir, "success.c") + build_log = [{"directory": self.test_workspace, + "command": "gcc -c " + source_file, + "file": source_file}] + + with open(build_json, 'w', + encoding="utf-8", errors="ignore") as outfile: + json.dump(build_log, outfile) + + merged_plist_name = "merged.plist" + + analyze_cmd = [self._codechecker_cmd, "analyze", + "--plist-file-name", merged_plist_name, + "-o", self.report_dir, build_json] + + print(analyze_cmd) + process = subprocess.Popen( + analyze_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.test_dir, + encoding="utf-8", + errors="ignore") + out, _ = process.communicate() + + # Output show merging + self.assertIn("Merging plist files into " + merged_plist_name, out) + + # Only the merged plist should remain + for file in Path(self.report_dir).glob("*.plist"): + self.assertEqual(file.name, merged_plist_name) + + print(out) + def test_analyzer_and_checker_config(self): """Test analyzer configuration through command line flags.""" build_json = os.path.join(self.test_workspace, "build_success.json") diff --git a/tools/report-converter/codechecker_report_converter/report/parser/plist.py b/tools/report-converter/codechecker_report_converter/report/parser/plist.py index 3f79878f06..e6e7811d74 100644 --- a/tools/report-converter/codechecker_report_converter/report/parser/plist.py +++ b/tools/report-converter/codechecker_report_converter/report/parser/plist.py @@ -206,20 +206,25 @@ def get_reports( if not plist: return reports - metadata = plist.get('metadata') + if not isinstance(plist, list): + plist = [plist] - files = get_file_index_map( - plist, source_dir_path, self._file_cache) + for sub_plist in plist: - for diag in plist.get('diagnostics', []): - report = self.__create_report( - analyzer_result_file_path, diag, files, metadata) + metadata = sub_plist.get('metadata') - if report.report_hash is None: - report.report_hash = get_report_hash( - report, HashType.PATH_SENSITIVE) + files = get_file_index_map( + sub_plist, source_dir_path, self._file_cache) + + for diag in sub_plist.get('diagnostics', []): + report = self.__create_report( + analyzer_result_file_path, diag, files, metadata) + + if report.report_hash is None: + report.report_hash = get_report_hash( + report, HashType.PATH_SENSITIVE) - reports.append(report) + reports.append(report) except KeyError as ex: LOG.warning("Failed to get file path id! Found files: %s. " "KeyError: %s", files, ex)