From 14621362faeb09adb22f3bde88dee55ed4666fd8 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 25 Aug 2022 16:14:49 -0400 Subject: [PATCH 1/4] Add dmalloc support to build examples test run --- scripts/build/build/targets.py | 3 + scripts/build/builders/gn.py | 11 ++ scripts/build/builders/host.py | 32 +++- scripts/build/testdata/build_linux_on_x64.txt | 148 +++++++++++++----- 4 files changed, 150 insertions(+), 44 deletions(-) diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py index c9edb01794bca9..6c199be484e884 100755 --- a/scripts/build/build/targets.py +++ b/scripts/build/build/targets.py @@ -336,6 +336,9 @@ def HostTargets(): yield target_native.Extend('tests-boringssl', app=HostApp.TESTS, crypto_library=HostCryptoLibrary.BORINGSSL).GlobBlacklist("Non-default test") yield target_native.Extend('tests-coverage', app=HostApp.TESTS, use_coverage=True).GlobBlacklist("Non-default test") yield target_native.Extend('tests-clang', app=HostApp.TESTS, use_clang=True).GlobBlacklist("Non-default test") + yield target_native.Extend('tests-clang-asan', app=HostApp.TESTS, use_clang=True, use_asan=True).GlobBlacklist("Non-default test") + yield target_native.Extend('tests-dmalloc', app=HostApp.TESTS, use_dmalloc=True).GlobBlacklist("Non-default test") + yield target_native.Extend('tests-clang-dmalloc-asan', app=HostApp.TESTS, use_clang=True, use_asan=True, use_dmalloc=True).GlobBlacklist("Non-default test") test_target = Target(HostBoard.NATIVE.PlatformName(), HostBuilder) yield test_target.Extend(HostBoard.FAKE.BoardName() + '-tests', board=HostBoard.FAKE, app=HostApp.TESTS) diff --git a/scripts/build/builders/gn.py b/scripts/build/builders/gn.py index 031546ae388873..298a9e9c90145b 100644 --- a/scripts/build/builders/gn.py +++ b/scripts/build/builders/gn.py @@ -88,6 +88,17 @@ def _build(self): if self.build_command: cmd.append(self.build_command) + extra_env = self.GnBuildEnv() + if extra_env: + # convert the command into a bash command that includes + # setting environment variables + cmd = [ + 'bash', '-c', '\n' + ' '.join( + ['%s="%s" \\\n' % (key, value) for key, value in extra_env.items()] + + [shlex.join(cmd)] + ) + ] + self._Execute(cmd, title='Building ' + self.identifier) self.PostBuildCommand() diff --git a/scripts/build/builders/host.py b/scripts/build/builders/host.py index 54df761d3dd3ca..1703c175be604d 100644 --- a/scripts/build/builders/host.py +++ b/scripts/build/builders/host.py @@ -214,7 +214,8 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE, separate_event_loop=True, use_libfuzzer=False, use_clang=False, interactive_mode=True, extra_tests=False, use_platform_mdns=False, enable_rpcs=False, - use_coverage=False, crypto_library: HostCryptoLibrary = None): + use_coverage=False, use_dmalloc=False, + crypto_library: HostCryptoLibrary = None): super(HostBuilder, self).__init__( root=os.path.join(root, 'examples', app.ExamplePath()), runner=runner) @@ -244,6 +245,11 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE, if use_asan: self.extra_gn_options.append('is_asan=true') + self.use_dmalloc = use_dmalloc + if use_dmalloc: + self.extra_gn_options.append('chip_config_memory_debug_checks=true') + self.extra_gn_options.append('chip_config_memory_debug_dmalloc=true') + if not separate_event_loop: self.extra_gn_options.append('config_use_separate_eventloop=false') @@ -327,19 +333,31 @@ def GnBuildArgs(self): raise Exception('Unknown host board type: %r' % self) def GnBuildEnv(self): + env = {} + + if self.use_dmalloc: + # this is from `dmalloc -b -l DMALLOC_LOG -i 1 high` + env['DMALLOC_OPTIONS']='debug=0x4f4ed03,inter=1,log=DMALLOC_LOG' + + # glib interop with dmalloc + env['G_SLICE']='always-malloc' + if self.board == HostBoard.NATIVE: - return None + pass elif self.board == HostBoard.FAKE: - return None + pass elif self.board == HostBoard.ARM64: - return { - 'PKG_CONFIG_PATH': os.path.join( + env['PKG_CONFIG_PATH'] = os.path.join( self.SysRootPath('SYSROOT_AARCH64'), - 'lib/aarch64-linux-gnu/pkgconfig'), - } + 'lib/aarch64-linux-gnu/pkgconfig') else: raise Exception('Unknown host board type: %r' % self) + if not env: + return None + + return env + def SysRootPath(self, name): if name not in os.environ: raise Exception('Missing environment variable "%s"' % name) diff --git a/scripts/build/testdata/build_linux_on_x64.txt b/scripts/build/testdata/build_linux_on_x64.txt index 9a926ebce07b96..0f4069fbdab319 100644 --- a/scripts/build/testdata/build_linux_on_x64.txt +++ b/scripts/build/testdata/build_linux_on_x64.txt @@ -322,115 +322,189 @@ gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/exa gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/tv-casting-app/linux --args=chip_inet_config_enable_ipv4=false {out}/linux-x64-tv-casting-app-ipv6only # Building linux-arm64-clang-all-clusters -ninja -C {out}/linux-arm64-clang-all-clusters +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-all-clusters' # Building linux-arm64-clang-all-clusters-app-nodeps -ninja -C {out}/linux-arm64-clang-all-clusters-app-nodeps +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-all-clusters-app-nodeps' # Building linux-arm64-clang-all-clusters-app-nodeps-ipv6only -ninja -C {out}/linux-arm64-clang-all-clusters-app-nodeps-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-all-clusters-app-nodeps-ipv6only' # Building linux-arm64-clang-all-clusters-ipv6only -ninja -C {out}/linux-arm64-clang-all-clusters-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-all-clusters-ipv6only' # Building linux-arm64-clang-all-clusters-minimal -ninja -C {out}/linux-arm64-clang-all-clusters-minimal +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-all-clusters-minimal' # Building linux-arm64-clang-all-clusters-minimal-ipv6only -ninja -C {out}/linux-arm64-clang-all-clusters-minimal-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-all-clusters-minimal-ipv6only' # Building linux-arm64-clang-bridge -ninja -C {out}/linux-arm64-clang-bridge +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-bridge' # Building linux-arm64-clang-bridge-ipv6only -ninja -C {out}/linux-arm64-clang-bridge-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-bridge-ipv6only' # Building linux-arm64-clang-chip-tool -ninja -C {out}/linux-arm64-clang-chip-tool +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-chip-tool' # Building linux-arm64-clang-chip-tool-ipv6only -ninja -C {out}/linux-arm64-clang-chip-tool-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-chip-tool-ipv6only' # Building linux-arm64-clang-chip-tool-nodeps -ninja -C {out}/linux-arm64-clang-chip-tool-nodeps +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-chip-tool-nodeps' # Building linux-arm64-clang-chip-tool-nodeps-ipv6only -ninja -C {out}/linux-arm64-clang-chip-tool-nodeps-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-chip-tool-nodeps-ipv6only' # Building linux-arm64-clang-light -ninja -C {out}/linux-arm64-clang-light +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-light' # Building linux-arm64-clang-light-ipv6only -ninja -C {out}/linux-arm64-clang-light-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-light-ipv6only' # Building linux-arm64-clang-light-rpc -ninja -C {out}/linux-arm64-clang-light-rpc +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-light-rpc' # Building linux-arm64-clang-light-rpc-ipv6only -ninja -C {out}/linux-arm64-clang-light-rpc-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-light-rpc-ipv6only' # Building linux-arm64-clang-lock -ninja -C {out}/linux-arm64-clang-lock +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-lock' # Building linux-arm64-clang-lock-ipv6only -ninja -C {out}/linux-arm64-clang-lock-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-lock-ipv6only' # Building linux-arm64-clang-minmdns -ninja -C {out}/linux-arm64-clang-minmdns +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-minmdns' # Building linux-arm64-clang-minmdns-ipv6only -ninja -C {out}/linux-arm64-clang-minmdns-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-minmdns-ipv6only' # Building linux-arm64-clang-ota-provider -ninja -C {out}/linux-arm64-clang-ota-provider +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-ota-provider' # Building linux-arm64-clang-ota-provider-ipv6only -ninja -C {out}/linux-arm64-clang-ota-provider-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-ota-provider-ipv6only' # Building linux-arm64-clang-ota-provider-nodeps -ninja -C {out}/linux-arm64-clang-ota-provider-nodeps +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-ota-provider-nodeps' # Building linux-arm64-clang-ota-provider-nodeps-ipv6only -ninja -C {out}/linux-arm64-clang-ota-provider-nodeps-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-ota-provider-nodeps-ipv6only' # Building linux-arm64-clang-ota-requestor -ninja -C {out}/linux-arm64-clang-ota-requestor +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-ota-requestor' # Building linux-arm64-clang-ota-requestor-ipv6only -ninja -C {out}/linux-arm64-clang-ota-requestor-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-ota-requestor-ipv6only' # Building linux-arm64-clang-ota-requestor-nodeps -ninja -C {out}/linux-arm64-clang-ota-requestor-nodeps +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-ota-requestor-nodeps' # Building linux-arm64-clang-ota-requestor-nodeps-ipv6only -ninja -C {out}/linux-arm64-clang-ota-requestor-nodeps-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-ota-requestor-nodeps-ipv6only' # Building linux-arm64-clang-python-bindings -ninja -C {out}/linux-arm64-clang-python-bindings chip-repl +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-python-bindings chip-repl' # Building linux-arm64-clang-shell -ninja -C {out}/linux-arm64-clang-shell +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-shell' # Building linux-arm64-clang-shell-ipv6only -ninja -C {out}/linux-arm64-clang-shell-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-shell-ipv6only' # Building linux-arm64-clang-thermostat -ninja -C {out}/linux-arm64-clang-thermostat +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-thermostat' # Building linux-arm64-clang-thermostat-ipv6only -ninja -C {out}/linux-arm64-clang-thermostat-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-thermostat-ipv6only' # Building linux-arm64-clang-tv-app -ninja -C {out}/linux-arm64-clang-tv-app +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-tv-app' # Building linux-arm64-clang-tv-app-ipv6only -ninja -C {out}/linux-arm64-clang-tv-app-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-tv-app-ipv6only' # Building linux-arm64-clang-tv-casting-app -ninja -C {out}/linux-arm64-clang-tv-casting-app +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-tv-casting-app' # Building linux-arm64-clang-tv-casting-app-ipv6only -ninja -C {out}/linux-arm64-clang-tv-casting-app-ipv6only +bash -c ' +PKG_CONFIG_PATH="SYSROOT_AARCH64/lib/aarch64-linux-gnu/pkgconfig" \ + ninja -C {out}/linux-arm64-clang-tv-casting-app-ipv6only' # Building linux-fake-tests ninja -C {out}/linux-fake-tests check From 1c60c4a15d0c9d56912816b62617e639be6cc40f Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 25 Aug 2022 16:15:54 -0400 Subject: [PATCH 2/4] Remove extra target for now - no need for both dmalloc and asan --- scripts/build/build/targets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py index 6c199be484e884..7f9e788c41d58a 100755 --- a/scripts/build/build/targets.py +++ b/scripts/build/build/targets.py @@ -338,7 +338,6 @@ def HostTargets(): yield target_native.Extend('tests-clang', app=HostApp.TESTS, use_clang=True).GlobBlacklist("Non-default test") yield target_native.Extend('tests-clang-asan', app=HostApp.TESTS, use_clang=True, use_asan=True).GlobBlacklist("Non-default test") yield target_native.Extend('tests-dmalloc', app=HostApp.TESTS, use_dmalloc=True).GlobBlacklist("Non-default test") - yield target_native.Extend('tests-clang-dmalloc-asan', app=HostApp.TESTS, use_clang=True, use_asan=True, use_dmalloc=True).GlobBlacklist("Non-default test") test_target = Target(HostBoard.NATIVE.PlatformName(), HostBuilder) yield test_target.Extend(HostBoard.FAKE.BoardName() + '-tests', board=HostBoard.FAKE, app=HostApp.TESTS) From cf76414960d6e9d8a6e83cb80642586d736d9143 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 25 Aug 2022 16:24:29 -0400 Subject: [PATCH 3/4] Move things a bit to be easier to follow --- scripts/build/builders/host.py | 36 +++++++++++----------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/scripts/build/builders/host.py b/scripts/build/builders/host.py index 1703c175be604d..0b00449f09e74e 100644 --- a/scripts/build/builders/host.py +++ b/scripts/build/builders/host.py @@ -223,6 +223,10 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE, self.app = app self.board = board self.extra_gn_options = [] + self.build_env = {} + + if board == HostBoard.ARM64: + self.build_env['PKG_CONFIG_PATH'] = os.path.join(self.SysRootPath('SYSROOT_AARCH64'), 'lib/aarch64-linux-gnu/pkgconfig') if enable_rpcs: self.extra_gn_options.append('import("//with_pw_rpc.gni")') @@ -245,11 +249,16 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE, if use_asan: self.extra_gn_options.append('is_asan=true') - self.use_dmalloc = use_dmalloc if use_dmalloc: self.extra_gn_options.append('chip_config_memory_debug_checks=true') self.extra_gn_options.append('chip_config_memory_debug_dmalloc=true') + # this is from `dmalloc -b -l DMALLOC_LOG -i 1 high` + self.build_env['DMALLOC_OPTIONS']='debug=0x4f4ed03,inter=1,log=DMALLOC_LOG' + + # glib interop with dmalloc + self.build_env['G_SLICE']='always-malloc' + if not separate_event_loop: self.extra_gn_options.append('config_use_separate_eventloop=false') @@ -333,30 +342,7 @@ def GnBuildArgs(self): raise Exception('Unknown host board type: %r' % self) def GnBuildEnv(self): - env = {} - - if self.use_dmalloc: - # this is from `dmalloc -b -l DMALLOC_LOG -i 1 high` - env['DMALLOC_OPTIONS']='debug=0x4f4ed03,inter=1,log=DMALLOC_LOG' - - # glib interop with dmalloc - env['G_SLICE']='always-malloc' - - if self.board == HostBoard.NATIVE: - pass - elif self.board == HostBoard.FAKE: - pass - elif self.board == HostBoard.ARM64: - env['PKG_CONFIG_PATH'] = os.path.join( - self.SysRootPath('SYSROOT_AARCH64'), - 'lib/aarch64-linux-gnu/pkgconfig') - else: - raise Exception('Unknown host board type: %r' % self) - - if not env: - return None - - return env + return self.build_env def SysRootPath(self, name): if name not in os.environ: From 543a9b9ef0cd4cb18ff4403542a7f744207ea2ff Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 25 Aug 2022 16:24:53 -0400 Subject: [PATCH 4/4] Restyle --- scripts/build/builders/host.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build/builders/host.py b/scripts/build/builders/host.py index 0b00449f09e74e..a536d6650b5e8f 100644 --- a/scripts/build/builders/host.py +++ b/scripts/build/builders/host.py @@ -254,10 +254,10 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE, self.extra_gn_options.append('chip_config_memory_debug_dmalloc=true') # this is from `dmalloc -b -l DMALLOC_LOG -i 1 high` - self.build_env['DMALLOC_OPTIONS']='debug=0x4f4ed03,inter=1,log=DMALLOC_LOG' + self.build_env['DMALLOC_OPTIONS'] = 'debug=0x4f4ed03,inter=1,log=DMALLOC_LOG' # glib interop with dmalloc - self.build_env['G_SLICE']='always-malloc' + self.build_env['G_SLICE'] = 'always-malloc' if not separate_event_loop: self.extra_gn_options.append('config_use_separate_eventloop=false')