Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make the llvm coverage tests a little more robust. #24624

Closed
wants to merge 2 commits into from
Closed

Make the llvm coverage tests a little more robust. #24624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4b385b6
Make the llvm coverage tests a little more robust.
c-mita Dec 10, 2024
1ecd5ba
Add a test for header coverage in the C++ LLVM coverage tests.
c-mita Dec 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 129 additions & 12 deletions src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,10 @@ FN:3,_Z1ab
FNDA:1,_Z1ab
FNF:1
FNH:1
BRDA:4,0,0,1
BRDA:4,0,1,0
BRF:2
BRH:1
DA:3,1
DA:4,1
DA:5,1
Expand All @@ -159,7 +163,7 @@ LH:5
LF:7
end_of_record"

assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
assert_llvm_cc_coverage_result "$expected_result" "$(get_coverage_file_path_from_test_log)"
}

function test_cc_test_llvm_coverage_produces_lcov_report_with_split_postprocessing() {
Expand All @@ -178,6 +182,10 @@ FN:3,_Z1ab
FNDA:1,_Z1ab
FNF:1
FNH:1
BRDA:4,0,0,1
BRDA:4,0,1,0
BRF:2
BRH:1
DA:3,1
DA:4,1
DA:5,1
Expand All @@ -189,7 +197,7 @@ LH:5
LF:7
end_of_record"

assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
assert_llvm_cc_coverage_result "$expected_result" "$(get_coverage_file_path_from_test_log)"
}

function test_cc_test_with_runtime_objects_not_in_runfiles() {
Expand Down Expand Up @@ -234,6 +242,10 @@ FN:4,main
FNDA:1,main
FNF:1
FNH:1
BRDA:5,0,0,1
BRDA:5,0,1,0
BRF:2
BRH:1
DA:4,1
DA:5,1
DA:6,1
Expand All @@ -243,7 +255,7 @@ LH:5
LF:5
end_of_record"

assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
assert_llvm_cc_coverage_result "$expected_result" "$(get_coverage_file_path_from_test_log)"
}

function setup_external_cc_target() {
Expand Down Expand Up @@ -331,11 +343,15 @@ function test_external_cc_target_can_collect_coverage() {
bazel coverage --combined_report=lcov --test_output=all \
@other_repo//:t --instrumentation_filter=// &>$TEST_log || fail "Coverage for @other_repo//:t failed"

local expected_result='SF:b.cc
local expected_b_cc='SF:b.cc
FN:1,_Z1bb
FNDA:1,_Z1bb
FNF:1
FNH:1
BRDA:2,0,0,1
BRDA:2,0,1,0
BRF:2
BRH:1
DA:1,1
DA:2,1
DA:3,1
Expand All @@ -345,12 +361,17 @@ DA:6,0
DA:7,1
LH:5
LF:7
end_of_record
SF:external/+_repo_rules+other_repo/a.cc
end_of_record'

local expected_a_cc='SF:external/+_repo_rules+other_repo/a.cc
FN:4,_Z1ab
FNDA:1,_Z1ab
FNF:1
FNH:1
BRDA:5,0,0,1
BRDA:5,0,1,0
BRF:2
BRH:1
DA:4,1
DA:5,1
DA:6,1
Expand All @@ -362,8 +383,10 @@ LH:5
LF:7
end_of_record'

assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat | grep -v '^BR')"
assert_llvm_cc_coverage_result "$expected_b_cc" "$(get_coverage_file_path_from_test_log)"
assert_llvm_cc_coverage_result "$expected_a_cc" "$(get_coverage_file_path_from_test_log)"
assert_llvm_cc_coverage_result "$expected_b_cc" "bazel-out/_coverage/_coverage_report.dat"
assert_llvm_cc_coverage_result "$expected_a_cc" "bazel-out/_coverage/_coverage_report.dat"
}

function test_external_cc_target_coverage_not_collected_by_default() {
Expand All @@ -378,6 +401,10 @@ FN:1,_Z1bb
FNDA:1,_Z1bb
FNF:1
FNH:1
BRDA:2,0,0,1
BRDA:2,0,1,0
BRF:2
BRH:1
DA:1,1
DA:2,1
DA:3,1
Expand All @@ -389,8 +416,8 @@ LH:5
LF:7
end_of_record'

assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat | grep -v '^BR')"
assert_llvm_cc_coverage_result "$expected_result" "$(get_coverage_file_path_from_test_log)"
assert_llvm_cc_coverage_result "$expected_result" "bazel-out/_coverage/_coverage_report.dat"
}

function test_coverage_with_tmp_in_path() {
Expand Down Expand Up @@ -445,6 +472,10 @@ FN:3,_Z1ab
FNDA:1,_Z1ab
FNF:1
FNH:1
BRDA:4,0,0,1
BRDA:4,0,1,0
BRF:2
BRH:1
DA:3,1
DA:4,1
DA:5,1
Expand All @@ -456,8 +487,94 @@ LH:5
LF:7
end_of_record'

assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat | grep -v '^BR')"
assert_llvm_cc_coverage_result "$expected_result" "$(get_coverage_file_path_from_test_log)"
assert_llvm_cc_coverage_result "$expected_result" "bazel-out/_coverage/_coverage_report.dat"
}

function test_coverage_for_header() {
setup_llvm_coverage_tools_for_lcov || return 0

cat << EOF > BUILD
cc_library(
name = "foo",
srcs = ["foo.cc"],
hdrs = ["foo.h"],
)

cc_test(
name = "foo_test",
srcs = ["foo_test.cc"],
deps = [":foo"],
)
EOF

cat << EOF > foo.h
template<typename T>
T fooify(T x) {
if (x < 0) {
return -1 * x;
}
return x + x*x;
}

int calc_foo(int x);
EOF

cat << EOF > foo.cc
#include "foo.h"

int calc_foo(int x) {
return fooify<int>(x);
}
EOF

cat << EOF > foo_test.cc

#include "foo.h"

int main() {
int f = calc_foo(4);
return f == 20 ? 0 : 1;
}
EOF

local expected_foo_h="SF:foo.h
FN:2,_Z6fooifyIiET_S0_
FNDA:1,_Z6fooifyIiET_S0_
FNF:1
FNH:1
BRDA:3,0,0,0
BRDA:3,0,1,1
BRF:2
BRH:1
DA:2,1
DA:3,1
DA:4,0
DA:5,0
DA:6,1
DA:7,1
LH:4
LF:6
end_of_record"

local expected_foo_cc="SF:foo.cc
FN:3,_Z8calc_fooi
FNDA:1,_Z8calc_fooi
FNF:1
FNH:1
DA:3,1
DA:4,1
DA:5,1
LH:3
LF:3
end_of_record"

bazel coverage --nobuild_runfile_links --test_output=all //:foo_test \
&>$TEST_log || fail "Coverage for //:foo_test failed"

cov_file="$(get_coverage_file_path_from_test_log)"
assert_llvm_cc_coverage_result "$expected_foo_h" "$cov_file"
assert_llvm_cc_coverage_result "$expected_foo_cc" "$cov_file"
}

run_suite "test tests"
20 changes: 20 additions & 0 deletions src/test/shell/bazel/coverage_helpers.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,26 @@ function assert_cc_coverage_result() {
assert_coverage_result "$expected_coverage" "$output_file"
}

# Asserts if the given expected coverage result is included in the given output
# file, accounting for the fact that branch coverage is not output prior to LLVM
# version 12
#
# - expected coverage The expected result that must be included in the output.
# - output_file The location of the coverage output file.
function assert_llvm_cc_coverage_result() {
local expected_coverage="${1}"; shift
local output_file="${1}"; shift

# LLVM does not output branch coverage before version 12
local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3)
if [ "$clang_version" -lt 12 ]; then
expected_coverage=$(echo "$expected_coverage" | grep -v "^BR")
fi

assert_coverage_result "$expected_coverage" "$output_file"

}

# Returns the path of the code coverage report that was generated by Bazel by
# looking at the current $TEST_log. The method fails if TEST_log does not
# contain any coverage report for a passed test.
Expand Down
Loading