From f2c31df1466f267cef214fc338f03a5e4e05b631 Mon Sep 17 00:00:00 2001
From: Jonathan Woollett-Light <jcawl@amazon.co.uk>
Date: Wed, 19 Oct 2022 14:27:09 +0100
Subject: [PATCH] Update code coverage
Updated code coverage to use grcov.
Signed-off-by: Jonathan Woollett-Light <jcawl@amazon.co.uk>
---
.gitignore | 1 +
.../integration_tests/build/test_coverage.py | 179 ++++++++----------
.../integration_tests/build/test_unittests.py | 12 +-
3 files changed, 86 insertions(+), 106 deletions(-)
diff --git a/.gitignore b/.gitignore
index 4d5a5f7934b..76713f74350 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,4 @@ __pycache__
.vscode
test_results/*
*.core
+*.profraw
diff --git a/tests/integration_tests/build/test_coverage.py b/tests/integration_tests/build/test_coverage.py
index 2922130a772..235fa50ee38 100644
--- a/tests/integration_tests/build/test_coverage.py
+++ b/tests/integration_tests/build/test_coverage.py
@@ -8,15 +8,10 @@
target should be put in `s3://spec.firecracker` and automatically updated.
"""
-
import os
-import platform
-import re
-import shutil
import pytest
from framework import utils
-import host_tools.cargo_build as host # pylint: disable=import-error
from host_tools import proc
# We have different coverages based on the host kernel version. This is
@@ -29,114 +24,100 @@
# Checkout the cpuid crate. In the future other
# differences may appear.
if utils.is_io_uring_supported():
- COVERAGE_DICT = {"Intel": 82.99, "AMD": 82.31, "ARM": 82.41}
+ COVERAGE_DICT = {"Intel": 90.84, "AMD": 90.11, "ARM": 90.51}
else:
- COVERAGE_DICT = {"Intel": 80.15, "AMD": 79.48, "ARM": 79.59}
+ COVERAGE_DICT = {"Intel": 88.39, "AMD": 87.67, "ARM": 87.95}
PROC_MODEL = proc.proc_type()
-COVERAGE_MAX_DELTA = 0.05
-
-CARGO_KCOV_REL_PATH = os.path.join(host.CARGO_BUILD_REL_PATH, "kcov")
-
-KCOV_COVERAGE_FILE = "index.js"
-"""kcov will aggregate coverage data in this file."""
+# Toolchain target architecture.
+if ("Intel" in PROC_MODEL) or ("AMD" in PROC_MODEL):
+ ARCH = "x86_64"
+elif "ARM" in PROC_MODEL:
+ ARCH = "aarch64"
+else:
+ raise Exception(f"Unsupported processor model ({PROC_MODEL})")
-KCOV_COVERED_LINES_REGEX = r'"covered_lines":"(\d+)"'
-"""Regex for extracting number of total covered lines found by kcov."""
+# Toolchain target.
+# Currently profiling with `aarch64-unknown-linux-musl` is unsupported (see
+# https://github.com/rust-lang/rustup/issues/3095#issuecomment-1280705619) therefore we profile and
+# run coverage with the `gnu` toolchains and run unit tests with the `musl` toolchains.
+TARGET = f"{ARCH}-unknown-linux-gnu"
-KCOV_TOTAL_LINES_REGEX = r'"total_lines" : "(\d+)"'
-"""Regex for extracting number of total executable lines found by kcov."""
+# We allow coverage to have a max difference of `COVERAGE_MAX_DELTA` as percentage before failing
+# the test.
+COVERAGE_MAX_DELTA = 0.05
-SECCOMPILER_BUILD_DIR = "../build/seccompiler"
+# grcov 0.8.* requires GLIBC >2.27, this is not present in ubuntu 18.04, when we update the docker
+# container with a newer version of ubuntu we can also update this.
+GRCOV_VERSION = "0.7.1"
@pytest.mark.timeout(400)
-def test_coverage(test_fc_session_root_path, test_session_tmp_path):
- """Test line coverage for rust tests is within bounds.
-
- The result is extracted from the $KCOV_COVERAGE_FILE file created by kcov
- after a coverage run.
+def test_coverage():
+ """Test code coverage
@type: build
"""
- proc_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
- assert len(proc_model) == 1, "Could not get processor model!"
- coverage_target_pct = COVERAGE_DICT[proc_model[0]]
- exclude_pattern = (
- "${CARGO_HOME:-$HOME/.cargo/},"
- "build/,"
- "tests/,"
- "usr/lib/gcc,"
- "lib/x86_64-linux-gnu/,"
- "test_utils.rs,"
- # The following files/directories are auto-generated
- "bootparam.rs,"
- "elf.rs,"
- "mpspec.rs,"
- "msr_index.rs,"
- "bindings.rs,"
- "_gen"
+ # Get coverage target.
+ processor_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
+ assert len(processor_model) == 1, "Could not get processor model!"
+ coverage_target = COVERAGE_DICT[processor_model[0]]
+
+ # Re-direct to repository root.
+ os.chdir("..")
+
+ # Add `llvm-tools-preview` component
+ utils.run_cmd("rustup component add llvm-tools-preview")
+
+ # Generate test profiles.
+ utils.run_cmd(
+ f'\
+ env RUSTFLAGS="-Cinstrument-coverage" \
+ LLVM_PROFILE_FILE="coverage-%p-%m.profraw" \
+ cargo test --all --target={TARGET} -- --test-threads=1 \
+ '
)
- exclude_region = "'mod tests {'"
- target = "{}-unknown-linux-musl".format(platform.machine())
-
- cmd = (
- 'CARGO_WRAPPER="kcov" RUSTFLAGS="{}" CARGO_TARGET_DIR={} '
- "cargo kcov --all "
- "--target {} --output {} -- "
- "--exclude-pattern={} "
- "--exclude-region={} --verify"
- ).format(
- host.get_rustflags(),
- os.path.join(test_fc_session_root_path, CARGO_KCOV_REL_PATH),
- target,
- test_session_tmp_path,
- exclude_pattern,
- exclude_region,
- )
- # We remove the seccompiler custom build directory, created by the
- # vmm-level `build.rs`.
- # If we don't delete it before and after running the kcov command, we will
- # run into linker errors.
- shutil.rmtree(SECCOMPILER_BUILD_DIR, ignore_errors=True)
- # By default, `cargo kcov` passes `--exclude-pattern=$CARGO_HOME --verify`
- # to kcov. To pass others arguments, we need to include the defaults.
- utils.run_cmd(cmd)
-
- shutil.rmtree(SECCOMPILER_BUILD_DIR)
-
- coverage_file = os.path.join(test_session_tmp_path, KCOV_COVERAGE_FILE)
- with open(coverage_file, encoding="utf-8") as cov_output:
- contents = cov_output.read()
- covered_lines = int(re.findall(KCOV_COVERED_LINES_REGEX, contents)[0])
- total_lines = int(re.findall(KCOV_TOTAL_LINES_REGEX, contents)[0])
- coverage = covered_lines / total_lines * 100
- print("Number of executable lines: {}".format(total_lines))
- print("Number of covered lines: {}".format(covered_lines))
- print("Thus, coverage is: {:.2f}%".format(coverage))
-
- coverage_low_msg = (
- "Current code coverage ({:.2f}%) is >{:.2f}% below the target ({}%).".format(
- coverage, COVERAGE_MAX_DELTA, coverage_target_pct
- )
- )
-
- assert coverage >= coverage_target_pct - COVERAGE_MAX_DELTA, coverage_low_msg
- # Get the name of the variable that needs updating.
- namespace = globals()
- cov_target_name = [name for name in namespace if namespace[name] is COVERAGE_DICT][
- 0
- ]
-
- coverage_high_msg = (
- "Current code coverage ({:.2f}%) is >{:.2f}% above the target ({}%).\n"
- "Please update the value of {}.".format(
- coverage, COVERAGE_MAX_DELTA, coverage_target_pct, cov_target_name
- )
+ # Generate coverage report.
+ utils.run_cmd(
+ f'\
+ cargo install --version {GRCOV_VERSION} grcov \
+ && grcov . \
+ -s . \
+ --binary-path ./build/cargo_target/{TARGET}/debug/ \
+ --ignore "build/*" \
+ -t html \
+ --branch \
+ --ignore-not-existing \
+ -o ./build/cargo_target/{TARGET}/debug/coverage \
+ '
)
- assert coverage <= coverage_target_pct + COVERAGE_MAX_DELTA, coverage_high_msg
-
- return (f"{coverage}%", f"{coverage_target_pct}% +/- {COVERAGE_MAX_DELTA * 100}%")
+ # Extract coverage from html report.
+ #
+ # The line looks like `<abbr title="44724 / 49237">90.83 %</abbr></p>` and is the first
+ # occurrence of the `<abbr>` element in the file.
+ #
+ # When we update grcov to 0.8.* we can update this to pull the coverage from a generated .json
+ # file.
+ index = open(
+ f"./build/cargo_target/{TARGET}/debug/coverage/index.html", encoding="utf-8"
+ )
+ index_contents = index.read()
+ end = index_contents.find(" %</abbr></p>")
+ start = index_contents[:end].rfind(">")
+ coverage_str = index_contents[start + 1 : end]
+ coverage = float(coverage_str)
+
+ # Compare coverage.
+ high = coverage_target * (1.0 + COVERAGE_MAX_DELTA)
+ low = coverage_target * (1.0 - COVERAGE_MAX_DELTA)
+ assert (
+ coverage >= low
+ ), f"Current code coverage ({coverage:.2f}%) is more than {COVERAGE_MAX_DELTA:.2f}% below \
+ the target ({coverage_target:.2f}%)"
+ assert (
+ coverage <= high
+ ), f"Current code coverage ({coverage:.2f}%) is more than {COVERAGE_MAX_DELTA:.2f}% above \
+ the target ({coverage_target:.2f}%)"
diff --git a/tests/integration_tests/build/test_unittests.py b/tests/integration_tests/build/test_unittests.py
index 5ba0be585c3..5477e728689 100644
--- a/tests/integration_tests/build/test_unittests.py
+++ b/tests/integration_tests/build/test_unittests.py
@@ -7,9 +7,10 @@
import host_tools.cargo_build as host # pylint:disable=import-error
MACHINE = platform.machine()
-# No need to run unittests for musl since
-# we run coverage with musl for all platforms.
-TARGET = "{}-unknown-linux-gnu".format(MACHINE)
+# Currently profiling with `aarch64-unknown-linux-musl` is unsupported (see
+# https://github.com/rust-lang/rustup/issues/3095#issuecomment-1280705619) therefore we profile and
+# run coverage with the `gnu` toolchains and run unit tests with the `musl` toolchains.
+TARGET = "{}-unknown-linux-musl".format(MACHINE)
def test_unittests(test_fc_session_root_path):
@@ -20,7 +21,4 @@ def test_unittests(test_fc_session_root_path):
"""
extra_args = "--release --target {} ".format(TARGET)
- host.cargo_test(
- test_fc_session_root_path,
- extra_args=extra_args
- )
+ host.cargo_test(test_fc_session_root_path, extra_args=extra_args)