From 4b385b6119d55e14521271659659825fbf36df16 Mon Sep 17 00:00:00 2001 From: Charles Mita Date: Tue, 10 Dec 2024 17:46:22 +0100 Subject: [PATCH 1/2] Make the llvm coverage tests a little more robust. Check that the actual coverage data simply contains the expected coverage data, rather than checking for an exact match. This matters for when we start to process baseline coverage, which may result in header files getting included in the report. This is in-keeping with the other coverage tests. Also adds the branch data into the reports - this is never disabled when using LLVM so we should check it. --- .../bazel/bazel_coverage_cc_test_llvm.sh | 55 +++++++++++++++---- src/test/shell/bazel/coverage_helpers.sh | 20 +++++++ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh b/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh index 630c5610666047..b1a6373befb121 100755 --- a/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh +++ b/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh @@ -148,6 +148,10 @@ FN:3,_Z1ab FNDA:1,_Z1ab FNF:1 FNH:1 +BRDA:4,0,0,1 +BRDA:4,0,1,0 +BRF:2 +BRH:1 DA:3,1 DA:4,1 DA:5,1 @@ -159,7 +163,7 @@ LH:5 LF:7 end_of_record" - assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')" + assert_llvm_cc_coverage_result "$expected_result" "$(get_coverage_file_path_from_test_log)" } function test_cc_test_llvm_coverage_produces_lcov_report_with_split_postprocessing() { @@ -178,6 +182,10 @@ FN:3,_Z1ab FNDA:1,_Z1ab FNF:1 FNH:1 +BRDA:4,0,0,1 +BRDA:4,0,1,0 +BRF:2 +BRH:1 DA:3,1 DA:4,1 DA:5,1 @@ -189,7 +197,7 @@ LH:5 LF:7 end_of_record" - assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')" + assert_llvm_cc_coverage_result "$expected_result" "$(get_coverage_file_path_from_test_log)" } function test_cc_test_with_runtime_objects_not_in_runfiles() { @@ -234,6 +242,10 @@ FN:4,main FNDA:1,main FNF:1 FNH:1 +BRDA:5,0,0,1 +BRDA:5,0,1,0 +BRF:2 +BRH:1 DA:4,1 DA:5,1 DA:6,1 @@ -243,7 +255,7 @@ LH:5 LF:5 end_of_record" - assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')" + assert_llvm_cc_coverage_result "$expected_result" "$(get_coverage_file_path_from_test_log)" } function setup_external_cc_target() { @@ -331,11 +343,15 @@ function test_external_cc_target_can_collect_coverage() { bazel coverage --combined_report=lcov --test_output=all \ @other_repo//:t --instrumentation_filter=// &>$TEST_log || fail "Coverage for @other_repo//:t failed" - local expected_result='SF:b.cc + local expected_b_cc='SF:b.cc FN:1,_Z1bb FNDA:1,_Z1bb FNF:1 FNH:1 +BRDA:2,0,0,1 +BRDA:2,0,1,0 +BRF:2 +BRH:1 DA:1,1 DA:2,1 DA:3,1 @@ -345,12 +361,17 @@ DA:6,0 DA:7,1 LH:5 LF:7 -end_of_record -SF:external/+_repo_rules+other_repo/a.cc +end_of_record' + + local expected_a_cc='SF:external/+_repo_rules+other_repo/a.cc FN:4,_Z1ab FNDA:1,_Z1ab FNF:1 FNH:1 +BRDA:5,0,0,1 +BRDA:5,0,1,0 +BRF:2 +BRH:1 DA:4,1 DA:5,1 DA:6,1 @@ -362,8 +383,10 @@ LH:5 LF:7 end_of_record' - assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')" - assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat | grep -v '^BR')" + assert_llvm_cc_coverage_result "$expected_b_cc" "$(get_coverage_file_path_from_test_log)" + assert_llvm_cc_coverage_result "$expected_a_cc" "$(get_coverage_file_path_from_test_log)" + assert_llvm_cc_coverage_result "$expected_b_cc" "bazel-out/_coverage/_coverage_report.dat" + assert_llvm_cc_coverage_result "$expected_a_cc" "bazel-out/_coverage/_coverage_report.dat" } function test_external_cc_target_coverage_not_collected_by_default() { @@ -378,6 +401,10 @@ FN:1,_Z1bb FNDA:1,_Z1bb FNF:1 FNH:1 +BRDA:2,0,0,1 +BRDA:2,0,1,0 +BRF:2 +BRH:1 DA:1,1 DA:2,1 DA:3,1 @@ -389,8 +416,8 @@ LH:5 LF:7 end_of_record' - assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')" - assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat | grep -v '^BR')" + assert_llvm_cc_coverage_result "$expected_result" "$(get_coverage_file_path_from_test_log)" + assert_llvm_cc_coverage_result "$expected_result" "bazel-out/_coverage/_coverage_report.dat" } function test_coverage_with_tmp_in_path() { @@ -445,6 +472,10 @@ FN:3,_Z1ab FNDA:1,_Z1ab FNF:1 FNH:1 +BRDA:4,0,0,1 +BRDA:4,0,1,0 +BRF:2 +BRH:1 DA:3,1 DA:4,1 DA:5,1 @@ -456,8 +487,8 @@ LH:5 LF:7 end_of_record' - assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')" - assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat | grep -v '^BR')" + assert_llvm_cc_coverage_result "$expected_result" "$(get_coverage_file_path_from_test_log)" + assert_llvm_cc_coverage_result "$expected_result" "bazel-out/_coverage/_coverage_report.dat" } run_suite "test tests" diff --git a/src/test/shell/bazel/coverage_helpers.sh b/src/test/shell/bazel/coverage_helpers.sh index d2b5756c42fdf2..f0da9d3d4040f2 100644 --- a/src/test/shell/bazel/coverage_helpers.sh +++ b/src/test/shell/bazel/coverage_helpers.sh @@ -97,6 +97,26 @@ function assert_cc_coverage_result() { assert_coverage_result "$expected_coverage" "$output_file" } +# Asserts if the given expected coverage result is included in the given output +# file, accounting for the fact that branch coverage is not output prior to LLVM +# version 12 +# +# - expected coverage The expected result that must be included in the output. +# - output_file The location of the coverage output file. +function assert_llvm_cc_coverage_result() { + local expected_coverage="${1}"; shift + local output_file="${1}"; shift + + # LLVM does not output branch coverage before version 12 + local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3) + if [ "$clang_version" -lt 12 ]; then + expected_coverage=$(echo "$expected_coverage" | grep -v "^BR") + fi + + assert_coverage_result "$expected_coverage" "$output_file" + +} + # Returns the path of the code coverage report that was generated by Bazel by # looking at the current $TEST_log. The method fails if TEST_log does not # contain any coverage report for a passed test. From 1ecd5ba0ce40f0115231f8ee4556d21e94a103bd Mon Sep 17 00:00:00 2001 From: Charles Mita Date: Wed, 11 Dec 2024 16:08:35 +0100 Subject: [PATCH 2/2] Add a test for header coverage in the C++ LLVM coverage tests. --- .../bazel/bazel_coverage_cc_test_llvm.sh | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh b/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh index b1a6373befb121..1199afe530ff3d 100755 --- a/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh +++ b/src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh @@ -491,4 +491,90 @@ end_of_record' assert_llvm_cc_coverage_result "$expected_result" "bazel-out/_coverage/_coverage_report.dat" } +function test_coverage_for_header() { + setup_llvm_coverage_tools_for_lcov || return 0 + + cat << EOF > BUILD +cc_library( + name = "foo", + srcs = ["foo.cc"], + hdrs = ["foo.h"], +) + +cc_test( + name = "foo_test", + srcs = ["foo_test.cc"], + deps = [":foo"], +) +EOF + +cat << EOF > foo.h +template +T fooify(T x) { + if (x < 0) { + return -1 * x; + } + return x + x*x; +} + +int calc_foo(int x); +EOF + +cat << EOF > foo.cc +#include "foo.h" + +int calc_foo(int x) { + return fooify(x); +} +EOF + +cat << EOF > foo_test.cc + +#include "foo.h" + +int main() { + int f = calc_foo(4); + return f == 20 ? 0 : 1; +} +EOF + + local expected_foo_h="SF:foo.h +FN:2,_Z6fooifyIiET_S0_ +FNDA:1,_Z6fooifyIiET_S0_ +FNF:1 +FNH:1 +BRDA:3,0,0,0 +BRDA:3,0,1,1 +BRF:2 +BRH:1 +DA:2,1 +DA:3,1 +DA:4,0 +DA:5,0 +DA:6,1 +DA:7,1 +LH:4 +LF:6 +end_of_record" + +local expected_foo_cc="SF:foo.cc +FN:3,_Z8calc_fooi +FNDA:1,_Z8calc_fooi +FNF:1 +FNH:1 +DA:3,1 +DA:4,1 +DA:5,1 +LH:3 +LF:3 +end_of_record" + + bazel coverage --nobuild_runfile_links --test_output=all //:foo_test \ + &>$TEST_log || fail "Coverage for //:foo_test failed" + + cov_file="$(get_coverage_file_path_from_test_log)" + assert_llvm_cc_coverage_result "$expected_foo_h" "$cov_file" + assert_llvm_cc_coverage_result "$expected_foo_cc" "$cov_file" +} + run_suite "test tests"