Add targets that generate test coverage information. (#20578) (#20676)

* add support for coverage enabled apps in build_examples.py * Switch to gcc builds * Coverage generation support * Fix typo * Update ordering for builds * Restyle * Update scripts/build/builders/gn.py Co-authored-by: Terence Hampson <[email protected]> Co-authored-by: Terence Hampson <[email protected]> Co-authored-by: Andrei Litvin <[email protected]> Co-authored-by: Terence Hampson <[email protected]>
project-chip · Jul 13, 2022 · 22032ff · 22032ff
1 parent eb2fb5b
commit 22032ff
Show file tree

Hide file tree

Showing 4 changed files with 91 additions and 3 deletions.
diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py
@@ -294,6 +294,10 @@ def HostTargets():
     for target in builder.AllVariants():
         yield target
 
+    # limited subset for coverage
+    yield target_native.Extend('all-clusters-coverage', app=HostApp.ALL_CLUSTERS, use_coverage=True)
+    yield target_native.Extend('chip-tool-coverage', app=HostApp.CHIP_TOOL, use_coverage=True)
+
     # Without extra build variants
     yield target_native.Extend('chip-cert', app=HostApp.CERT_TOOL)
     yield target_native.Extend('address-resolve-tool', app=HostApp.ADDRESS_RESOLVE)
@@ -304,9 +308,11 @@ def HostTargets():
     yield target_native.Extend('address-resolve-tool-platform-mdns-ipv6only', app=HostApp.ADDRESS_RESOLVE,
                                use_platform_mdns=True, enable_ipv4=False).GlobBlacklist("Reduce default build variants")
 
+    yield target_native.Extend('tests', app=HostApp.TESTS)
+    yield target_native.Extend('tests-coverage', app=HostApp.TESTS, use_coverage=True)
+    yield target_native.Extend('tests-clang', app=HostApp.TESTS, use_clang=True)
+
     test_target = Target(HostBoard.NATIVE.PlatformName(), HostBuilder)
-    yield test_target.Extend(HostBoard.NATIVE.BoardName() + '-tests', board=HostBoard.NATIVE, app=HostApp.TESTS)
-    yield test_target.Extend(HostBoard.NATIVE.BoardName() + '-tests-clang', board=HostBoard.NATIVE, app=HostApp.TESTS, use_clang=True)
     yield test_target.Extend(HostBoard.FAKE.BoardName() + '-tests', board=HostBoard.FAKE, app=HostApp.TESTS)
 
 

diff --git a/scripts/build/builders/gn.py b/scripts/build/builders/gn.py
@@ -45,6 +45,14 @@ def GnBuildEnv(self):
         """
         return None
 
+    def PreBuildCommand(self):
+        """Extra steps to run before 'build'"""
+        pass
+
+    def PostBuildCommand(self):
+        """Extra steps to run after 'build'"""
+        pass
+
     def generate(self):
         if not os.path.exists(self.output_dir):
             cmd = [
@@ -75,8 +83,12 @@ def generate(self):
             self._Execute(cmd, title=title)
 
     def _build(self):
+        self.PreBuildCommand()
+
         cmd = ['ninja', '-C', self.output_dir]
         if self.build_command:
             cmd.append(self.build_command)
 
         self._Execute(cmd, title='Building ' + self.identifier)
+
+        self.PostBuildCommand()
diff --git a/scripts/build/builders/host.py b/scripts/build/builders/host.py
@@ -197,7 +197,8 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE,
                  enable_thread=True, use_tsan=False, use_asan=False,
                  separate_event_loop=True, use_libfuzzer=False, use_clang=False,
                  interactive_mode=True, extra_tests=False,
-                 use_platform_mdns=False, enable_rpcs=False):
+                 use_platform_mdns=False, enable_rpcs=False,
+                 use_coverage=False):
         super(HostBuilder, self).__init__(
             root=os.path.join(root, 'examples', app.ExamplePath()),
             runner=runner)
@@ -236,6 +237,10 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE,
         if use_libfuzzer:
             self.extra_gn_options.append('is_libfuzzer=true')
 
+        self.use_coverage = use_coverage
+        if use_coverage:
+            self.extra_gn_options.append('use_coverage=true')
+
         if use_clang:
             self.extra_gn_options.append('is_clang=true')
 
@@ -321,6 +326,32 @@ def SysRootPath(self, name):
             raise Exception('Missing environment variable "%s"' % name)
         return os.environ[name]
 
+    def generate(self):
+        super(HostBuilder, self).generate()
+
+        if self.app == HostApp.TESTS and self.use_coverage:
+            self.coverage_dir = os.path.join(self.output_dir, 'coverage')
+            self._Execute(['mkdir', '-p', self.coverage_dir], title="Create coverage output location")
+            self._Execute(['lcov', '--initial', '--capture', '--directory', os.path.join(self.output_dir, 'obj'),
+                          '--output-file', os.path.join(self.coverage_dir, 'lcov_base.info')], title="Initial coverage baseline")
+
+    def PreBuildCommand(self):
+        if self.app == HostApp.TESTS and self.use_coverage:
+            self._Execute(['ninja', '-C', self.output_dir, 'default'], title="Build-only")
+            self._Execute(['lcov', '--initial', '--capture', '--directory', os.path.join(self.output_dir, 'obj'),
+                          '--output-file', os.path.join(self.coverage_dir, 'lcov_base.info')], title="Initial coverage baseline")
+
+    def PostBuildCommand(self):
+        if self.app == HostApp.TESTS and self.use_coverage:
+            self._Execute(['lcov', '--capture', '--directory', os.path.join(self.output_dir, 'obj'), '--output-file',
+                          os.path.join(self.coverage_dir, 'lcov_test.info')], title="Update coverage")
+            self._Execute(['lcov', '--add-tracefile', os.path.join(self.coverage_dir, 'lcov_base.info'),
+                           '--add-tracefile', os.path.join(self.coverage_dir, 'lcov_test.info'),
+                           '--output-file', os.path.join(self.coverage_dir, 'lcov_final.info')
+                           ], title="Final coverage info")
+            self._Execute(['genhtml', os.path.join(self.coverage_dir, 'lcov_final.info'), '--output-directory',
+                          os.path.join(self.coverage_dir, 'html')], title="HTML coverage")
+
     def build_outputs(self):
         outputs = {}
 

diff --git a/scripts/build/testdata/build_linux_on_x64.txt b/scripts/build/testdata/build_linux_on_x64.txt
@@ -181,6 +181,9 @@ gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/exa
 # Generating linux-x64-all-clusters-app-nodeps-ipv6only
 gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/all-clusters-app/linux '--args=chip_inet_config_enable_ipv4=false chip_config_network_layer_ble=false chip_enable_wifi=false chip_enable_openthread=false is_clang=true' {out}/linux-x64-all-clusters-app-nodeps-ipv6only
 
+# Generating linux-x64-all-clusters-coverage
+gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/all-clusters-app/linux --args=use_coverage=true {out}/linux-x64-all-clusters-coverage
+
 # Generating linux-x64-all-clusters-ipv6only
 gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/all-clusters-app/linux --args=chip_inet_config_enable_ipv4=false {out}/linux-x64-all-clusters-ipv6only
 
@@ -202,6 +205,9 @@ gn gen --check --fail-on-unused-args --export-compile-commands --root={root} '--
 # Generating linux-x64-chip-tool
 gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/chip-tool {out}/linux-x64-chip-tool
 
+# Generating linux-x64-chip-tool-coverage
+gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/chip-tool --args=use_coverage=true {out}/linux-x64-chip-tool-coverage
+
 # Generating linux-x64-chip-tool-ipv6only
 gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/chip-tool --args=chip_inet_config_enable_ipv4=false {out}/linux-x64-chip-tool-ipv6only
 
@@ -268,6 +274,15 @@ gn gen --check --fail-on-unused-args --export-compile-commands --root={root} --a
 # Generating linux-x64-tests-clang
 gn gen --check --fail-on-unused-args --export-compile-commands --root={root} '--args=is_clang=true chip_build_tests=true' {out}/linux-x64-tests-clang
 
+# Generating linux-x64-tests-coverage
+gn gen --check --fail-on-unused-args --export-compile-commands --root={root} '--args=use_coverage=true chip_build_tests=true' {out}/linux-x64-tests-coverage
+
+# Create coverage output location
+mkdir -p {out}/linux-x64-tests-coverage/coverage
+
+# Initial coverage baseline
+lcov --initial --capture --directory {out}/linux-x64-tests-coverage/obj --output-file {out}/linux-x64-tests-coverage/coverage/lcov_base.info
+
 # Generating linux-x64-thermostat
 gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/thermostat/linux {out}/linux-x64-thermostat
 
@@ -400,6 +415,9 @@ ninja -C {out}/linux-x64-all-clusters-app-nodeps
 # Building linux-x64-all-clusters-app-nodeps-ipv6only
 ninja -C {out}/linux-x64-all-clusters-app-nodeps-ipv6only
 
+# Building linux-x64-all-clusters-coverage
+ninja -C {out}/linux-x64-all-clusters-coverage
+
 # Building linux-x64-all-clusters-ipv6only
 ninja -C {out}/linux-x64-all-clusters-ipv6only
 
@@ -421,6 +439,9 @@ ninja -C {out}/linux-x64-chip-cert src/tools/chip-cert
 # Building linux-x64-chip-tool
 ninja -C {out}/linux-x64-chip-tool
 
+# Building linux-x64-chip-tool-coverage
+ninja -C {out}/linux-x64-chip-tool-coverage
+
 # Building linux-x64-chip-tool-ipv6only
 ninja -C {out}/linux-x64-chip-tool-ipv6only
 
@@ -487,6 +508,24 @@ ninja -C {out}/linux-x64-tests check
 # Building linux-x64-tests-clang
 ninja -C {out}/linux-x64-tests-clang check
 
+# Build-only
+ninja -C {out}/linux-x64-tests-coverage default
+
+# Initial coverage baseline
+lcov --initial --capture --directory {out}/linux-x64-tests-coverage/obj --output-file {out}/linux-x64-tests-coverage/coverage/lcov_base.info
+
+# Building linux-x64-tests-coverage
+ninja -C {out}/linux-x64-tests-coverage check
+
+# Update coverage
+lcov --capture --directory {out}/linux-x64-tests-coverage/obj --output-file {out}/linux-x64-tests-coverage/coverage/lcov_test.info
+
+# Final coverage info
+lcov --add-tracefile {out}/linux-x64-tests-coverage/coverage/lcov_base.info --add-tracefile {out}/linux-x64-tests-coverage/coverage/lcov_test.info --output-file {out}/linux-x64-tests-coverage/coverage/lcov_final.info
+
+# HTML coverage
+genhtml {out}/linux-x64-tests-coverage/coverage/lcov_final.info --output-directory {out}/linux-x64-tests-coverage/coverage/html
+
 # Building linux-x64-thermostat
 ninja -C {out}/linux-x64-thermostat