From a6781c8358bb88de9c889bc63944ec8ad16bd110 Mon Sep 17 00:00:00 2001 From: Henk van der Laan Date: Tue, 2 Jun 2020 19:44:06 +0200 Subject: [PATCH] Fix GCC 9 Code coverage by adding its JSON format GCC 9 changed the intermediate format to a JSON based format and with it changed the meaning of the `-i` flag. Because of these changes it was not possible to generate code coverage with GCC 9. This patch addresses that by adding its format next to the existing GCov parser. Addresses: #9406 --- .../devtools/coverageoutputgenerator/BUILD | 17 +++ .../coverageoutputgenerator/Constants.java | 1 + .../GcovJsonParser.java | 120 ++++++++++++++++++ .../coverageoutputgenerator/Main.java | 21 ++- tools/test/collect_cc_coverage.sh | 18 ++- 5 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD index db22b5ce5ba1ef..84031601852e82 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD @@ -49,6 +49,7 @@ java_library( srcs = glob(["*.java"]), deps = [ "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", "//third_party/java/jcommander", @@ -117,6 +118,21 @@ java_library( ], ) +java_library( + name = "GcovJsonParser", + srcs = [ + "GcovJsonParser.java", + "Parser.java", + ], + deps = [ + ":BranchCoverage", + ":Constants", + ":LineCoverage", + ":SourceFileCoverage", + "//third_party:gson", + ], +) + java_library( name = "LcovParser", srcs = [ @@ -159,6 +175,7 @@ java_library( ":Constants", ":Coverage", ":GcovParser", + ":GcovJsonParser", ":LcovMergerFlags", ":LcovParser", ":LcovPrinter", diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java index a4b1b5398dd5da..808e257d8f521f 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java @@ -39,6 +39,7 @@ class Constants { static final String TAKEN = "-"; static final String TRACEFILE_EXTENSION = ".dat"; static final String GCOV_EXTENSION = ".gcov"; + static final String GCOV_JSON_EXTENSION = ".gcov.json.gz"; static final String PROFDATA_EXTENSION = ".profdata"; static final String DELIMITER = ","; static final String GCOV_VERSION_MARKER = "version:"; diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java new file mode 100644 index 00000000000000..4d019b7076b3b1 --- /dev/null +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java @@ -0,0 +1,120 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.coverageoutputgenerator; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; +import com.google.gson.Gson; +import com.google.gson.annotations.SerializedName; + +/** + * A {@link Parser} for gcov intermediate json format introduced in GCC 9.1. See the flag {@code --intermediate-format} in gcov documentation. + */ +public class GcovJsonParser { + private static final Logger logger = Logger.getLogger(GcovParser.class.getName()); + private final InputStream inputStream; + + private GcovJsonParser(InputStream inputStream) { + this.inputStream = inputStream; + } + + public static List parse(InputStream inputStream) throws IOException { + return new GcovJsonParser(inputStream).parse(); + } + + private List parse() throws IOException { + ArrayList allSourceFiles = new ArrayList<>(); + try (InputStream gzipStream = new GZIPInputStream(inputStream)) { + ByteArrayOutputStream contents = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = gzipStream.read(buffer)) != -1) { + contents.write(buffer, 0, length); + } + Gson gson = new Gson(); + GcovJsonFormat document = gson.fromJson(contents.toString(), GcovJsonFormat.class); + if (!document.format_version.equals("1")) { + logger.log(Level.WARNING, "Expect GCov JSON format version 1, got format version " + document.format_version); + } + for (GcovJsonFile file : document.files) { + SourceFileCoverage currentFileCoverage = new SourceFileCoverage(file.file); + for (GcovJsonFunction function : file.functions) { + currentFileCoverage.addLineNumber(function.name, function.start_line); + currentFileCoverage.addFunctionExecution(function.name, function.execution_count); + } + for (GcovJsonLine line : file.lines) { + currentFileCoverage.addLine(line.line_number, LineCoverage.create(line.line_number, line.count, null)); + for (GcovJsonBranch branch : line.branches) { + currentFileCoverage.addBranch(line.line_number, BranchCoverage.create(line.line_number, branch.count)); + } + } + allSourceFiles.add(currentFileCoverage); + } + } + + return allSourceFiles; + } +} + +// Classes for the Gson data mapper representing the structure of the GCov JSON format + +class GcovJsonFormat { + String gcc_version; + GcovJsonFile[] files; + String format_version; + String current_working_directory; + String data_file; +} + +class GcovJsonFile { + String file; + GcovJsonFunction[] functions; + GcovJsonLine[] lines; +} + +class GcovJsonFunction { + int blocks; + int end_column; + int start_line; + String name; + int blocks_executed; + int execution_count; + String demangled_name; + int start_column; + int end_line; +} + +class GcovJsonLine { + GcovJsonBranch[] branches; + int count; + int line_number; + boolean unexecuted_block; + String function_name; +} + +class GcovJsonBranch { + boolean fallthrough; + int count; + @SerializedName("throw") + boolean _throw; +} diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java index 21f3565359f9c2..d252c35e131d69 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java @@ -15,6 +15,7 @@ package com.google.devtools.coverageoutputgenerator; import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_EXTENSION; +import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_JSON_EXTENSION; import static com.google.devtools.coverageoutputgenerator.Constants.PROFDATA_EXTENSION; import static com.google.devtools.coverageoutputgenerator.Constants.TRACEFILE_EXTENSION; import static java.nio.charset.StandardCharsets.UTF_8; @@ -78,8 +79,13 @@ static int runWithArgs(String... args) throws ExecutionException, InterruptedExc getTracefiles(flags, filesInCoverageDir), LcovParser::parse, flags.parseParallelism()), - parseFiles( - getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism())); + Coverage.merge( + parseFiles( + getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism()), + parseFiles( + getGcovJsonInfoFiles(filesInCoverageDir), + GcovJsonParser::parse, + flags.parseParallelism()))); if (flags.sourcesToReplaceFile() != null) { coverage.maybeReplaceSourceFileNames(getMapFromFile(flags.sourcesToReplaceFile())); @@ -221,6 +227,16 @@ private static List getGcovInfoFiles(List filesInCoverageDir) { return gcovFiles; } + private static List getGcovJsonInfoFiles(List filesInCoverageDir) { + List gcovJsonFiles = getFilesWithExtension(filesInCoverageDir, GCOV_JSON_EXTENSION); + if (gcovJsonFiles.isEmpty()) { + logger.log(Level.INFO, "No gcov json file found."); + } else { + logger.log(Level.INFO, "Found " + gcovJsonFiles.size() + " gcov json files."); + } + return gcovJsonFiles; + } + /** * Returns a .profdata file from the given files or null if none or more profdata files were * found. @@ -352,6 +368,7 @@ static List getCoverageFilesInDir(String dir) { p -> p.toString().endsWith(TRACEFILE_EXTENSION) || p.toString().endsWith(GCOV_EXTENSION) + || p.toString().endsWith(GCOV_JSON_EXTENSION) || p.toString().endsWith(PROFDATA_EXTENSION)) .map(path -> path.toFile()) .collect(Collectors.toList()); diff --git a/tools/test/collect_cc_coverage.sh b/tools/test/collect_cc_coverage.sh index 181433bf33ff1f..81987aab127aad 100755 --- a/tools/test/collect_cc_coverage.sh +++ b/tools/test/collect_cc_coverage.sh @@ -125,10 +125,20 @@ function gcov_coverage() { # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84879). "${GCOV}" -i $COVERAGE_GCOV_OPTIONS -o "$(dirname ${gcda})" "${gcda}" - # Append all .gcov files in the current directory to the output file. - cat *.gcov >> "$output_file" - # Delete the .gcov files. - rm *.gcov + gcov_version=$("${GCOV}" --version | sed -n -r -e 's/^.*\s([0-9]\.[0-9]\.[0-9])\s?.*$/\1/p') + + # Check the gcov version so we can process the data correctly + if [ "$(printf '%s\n%s' "9.1.0" "$gcov_version" | sort -V | head -n 1)" != "$gcov_version" ]; then + # GCOV 9.1.0 or higher generate a JSON based coverage format + # The output is generated into multiple files: "$(basename ${gcda}).gcov.json.gz" + # Concatenating JSON documents does not yield a valid document, so they are moved individually + mv -- *.gcov.json.gz "$(dirname "$output_file")" + else + # Append all .gcov files in the current directory to the output file. + cat -- *.gcov >> "$output_file" + # Delete the .gcov files. + rm -- *.gcov + fi fi fi done < "${COVERAGE_MANIFEST}"