From a9467fbbe206e6a62547a12a11ae50099a2c1a54 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 19 Jul 2021 15:50:04 -0400 Subject: [PATCH] Separate out "GN builder" for the unified build script (#8381) * Imported a general chip builder script, currently covering a few platforms/boards/apps as a start to have a single build entrypoint * Move build requirements into global script/requirements.txt so that they get picked up by the bootstrap script * Update script to assume and require bootstrapping * Code review comments * Support building the lock app for ESP32 It turns out that ESP32 lock app is ONLY for devkitc, so I added application matching logic to include board restrictions. Tested: lock app compiles and only for devkitc if esp32 platform is used. * Remove obsolete todo * Fix the duplicated accept for efr32 lock app * Add a dry run option for the build runner, showing what commands would be executed * Add gn builder as a generic build class, to make gn configuration uniform * Add support for a "test" to validate that the build generator executes commands as expected * Update the command comparison: output directory of the build script has also to be considered for testing * Fix some naming and use `get_target_outputs` * Address some code review comments * Rename chipbuild to build_examples * Fixup naming for the unit tests after build script renaming * Fix names * Restyle * Use difflib instead of diff binary for checking changes * Fix diffs (generator vs lines) and logging * Fix unit tests * Start converting python logic from build into pw_python_module * Tests pass * Restyle * Code review comments * Add comment for all_platform_commands.txt * Move expected txt data into inputs --- scripts/build/builders/efr32.py | 29 ++++---------- scripts/build/builders/gn.py | 38 +++++++++++++++++++ scripts/build/builders/linux.py | 23 +++-------- scripts/build/builders/qpg.py | 22 +++-------- .../build/expected_all_platform_commands.txt | 34 +++++++---------- scripts/build/runner/shell.py | 2 +- 6 files changed, 71 insertions(+), 77 deletions(-) create mode 100644 scripts/build/builders/gn.py diff --git a/scripts/build/builders/efr32.py b/scripts/build/builders/efr32.py index b8d3668876d1ce..5cc44f5544f343 100644 --- a/scripts/build/builders/efr32.py +++ b/scripts/build/builders/efr32.py @@ -2,7 +2,7 @@ import os from enum import Enum, auto -from .builder import Builder +from .gn import GnBuilder class Efr32App(Enum): @@ -49,7 +49,7 @@ def GnArgName(self): return 'BRD4161A' -class Efr32Builder(Builder): +class Efr32Builder(GnBuilder): def __init__(self, root, @@ -57,28 +57,13 @@ def __init__(self, output_dir: str, app: Efr32App = Efr32App.LIGHT, board: Efr32Board = Efr32Board.BRD4161A): - super(Efr32Builder, self).__init__(root, runner, output_dir) + super(Efr32Builder, self).__init__( + root=os.path.join(root, 'examples', app.ExampleName(), 'efr32'), + runner=runner, + output_dir=output_dir) self.app = app - self.board = board - self.identifier = None - - def generate(self): - if not os.path.exists(self.output_dir): - self._Execute([ - 'gn', 'gen', '--check', '--fail-on-unused-args', - '--root=%s' % - os.path.join(self.root, 'examples', self.app.ExampleName(), 'efr32'), - '--args=efr32_board="%s"' % self.board.GnArgName(), self.output_dir - ], - title='Generate %s' % self.identifier) - - def build(self): - logging.info('Compiling EFR32 at %s', self.output_dir) - - self.generate() - self._Execute(['ninja', '-C', self.output_dir], - title='Build %s' % self.identifier) + self.gn_build_args = ['efr32_board="%s"' % board.GnArgName()] def outputs(self): items = { diff --git a/scripts/build/builders/gn.py b/scripts/build/builders/gn.py new file mode 100644 index 00000000000000..591610265da54c --- /dev/null +++ b/scripts/build/builders/gn.py @@ -0,0 +1,38 @@ +import logging +import os + +from .builder import Builder + + +class GnBuilder(Builder): + + def __init__(self, root, runner, output_dir): + """Creates a generic ninja builder. + + Args: + root: the root where to run GN into + runner: what to use to execute shell commands + output_dir: where ninja files are to be generated + """ + super(GnBuilder, self).__init__(root, runner, output_dir) + + self.gn_build_args = None + + def generate(self): + if not os.path.exists(self.output_dir): + cmd = [ + 'gn', 'gen', '--check', '--fail-on-unused-args', + '--root=%s' % self.root + ] + + if self.gn_build_args: + cmd += ['--args=%s' % ' '.join(self.gn_build_args)] + + cmd += [self.output_dir] + + self._Execute(cmd, title='Generating ' + self.identifier) + + def build(self): + self.generate() + self._Execute(['ninja', '-C', self.output_dir], + title='Building ' + self.identifier) diff --git a/scripts/build/builders/linux.py b/scripts/build/builders/linux.py index d528fdee9eb3ec..b37e32078b0c43 100644 --- a/scripts/build/builders/linux.py +++ b/scripts/build/builders/linux.py @@ -1,27 +1,16 @@ import logging import os -from .builder import Builder +from .gn import GnBuilder -class LinuxBuilder(Builder): +class LinuxBuilder(GnBuilder): def __init__(self, root, runner, output_dir): - super(LinuxBuilder, self).__init__(root, runner, output_dir) - - def generate(self): - if not os.path.exists(self.output_dir): - self._Execute(['gn', 'gen', self.output_dir], - cwd=os.path.join(self.root, - 'examples/all-clusters-app/linux/'), - title='Generating ' + self.identifier) - - def build(self): - logging.info('Compiling Linux at %s', self.output_dir) - - self.generate() - self._Execute(['ninja', '-C', self.output_dir], - title='Building ' + self.identifier) + super(LinuxBuilder, self).__init__( + root=os.path.join(root, 'examples/all-clusters-app/linux/'), + runner=runner, + output_dir=output_dir) def outputs(self): return { diff --git a/scripts/build/builders/qpg.py b/scripts/build/builders/qpg.py index 0c91ff6ee3b999..e11fbb0fa5210c 100644 --- a/scripts/build/builders/qpg.py +++ b/scripts/build/builders/qpg.py @@ -3,26 +3,16 @@ import logging import os -from .builder import Builder +from .gn import GnBuilder -class QpgBuilder(Builder): +class QpgBuilder(GnBuilder): def __init__(self, root, runner, output_dir): - super(QpgBuilder, self).__init__(root, runner, output_dir) - - def generate(self): - if not os.path.exists(self.output_dir): - self._Execute(['gn', 'gen', self.output_dir], - cwd=os.path.join(self.root, 'examples/lock-app/qpg/'), - title='Generating ' + self.identifier) - - def build(self): - logging.info('Compiling QPG at %s', self.output_dir) - - self.generate() - self._Execute(['ninja', '-C', self.output_dir], - title='Building ' + self.identifier) + super(QpgBuilder, self).__init__( + root=os.path.join(root, 'examples/lock-app/qpg/'), + runner=runner, + output_dir=output_dir) def outputs(self): return { diff --git a/scripts/build/expected_all_platform_commands.txt b/scripts/build/expected_all_platform_commands.txt index d852e580bd63a4..3220461d27fd08 100644 --- a/scripts/build/expected_all_platform_commands.txt +++ b/scripts/build/expected_all_platform_commands.txt @@ -1,12 +1,8 @@ # Generating linux-native-all_clusters -cd "{root}/examples/all-clusters-app/linux/" -gn gen {out}/linux-native-all_clusters -cd - +gn gen --check --fail-on-unused-args --root={root}/examples/all-clusters-app/linux {out}/linux-native-all_clusters # Generating qpg-qpg6100-lock -cd "{root}/examples/lock-app/qpg/" -gn gen {out}/qpg-qpg6100-lock -cd - +gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/qpg {out}/qpg-qpg6100-lock # Generating esp32-m5stack-all_clusters cd "{root}" @@ -23,27 +19,23 @@ cd "{root}" bash -c 'source $IDF_PATH/export.sh; idf.py -C examples/lock-app/esp32 -B {out}/esp32-devkitc-lock reconfigure' cd - -# Generate efr32-brd4161a-light +# Generating efr32-brd4161a-light gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-light -# Generate efr32-brd4161a-lock +# Generating efr32-brd4161a-lock gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-lock -# Generate efr32-brd4161a-window_covering +# Generating efr32-brd4161a-window_covering gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-window_covering # Generating linux-native-all_clusters -cd "{root}/examples/all-clusters-app/linux/" -gn gen {out}/linux-native-all_clusters -cd - +gn gen --check --fail-on-unused-args --root={root}/examples/all-clusters-app/linux {out}/linux-native-all_clusters # Building linux-native-all_clusters ninja -C {out}/linux-native-all_clusters # Generating qpg-qpg6100-lock -cd "{root}/examples/lock-app/qpg/" -gn gen {out}/qpg-qpg6100-lock -cd - +gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/qpg {out}/qpg-qpg6100-lock # Building qpg-qpg6100-lock ninja -C {out}/qpg-qpg6100-lock @@ -72,22 +64,22 @@ cd - # Building esp32-devkitc-lock bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{out}/esp32-devkitc-lock'"'"'' -# Generate efr32-brd4161a-light +# Generating efr32-brd4161a-light gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-light -# Build efr32-brd4161a-light +# Building efr32-brd4161a-light ninja -C {out}/efr32-brd4161a-light -# Generate efr32-brd4161a-lock +# Generating efr32-brd4161a-lock gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-lock -# Build efr32-brd4161a-lock +# Building efr32-brd4161a-lock ninja -C {out}/efr32-brd4161a-lock -# Generate efr32-brd4161a-window_covering +# Generating efr32-brd4161a-window_covering gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-window_covering -# Build efr32-brd4161a-window_covering +# Building efr32-brd4161a-window_covering ninja -C {out}/efr32-brd4161a-window_covering diff --git a/scripts/build/runner/shell.py b/scripts/build/runner/shell.py index d1122060801018..13bc5e4d36d386 100644 --- a/scripts/build/runner/shell.py +++ b/scripts/build/runner/shell.py @@ -50,4 +50,4 @@ def Run(self, cmd, cwd=None, title=None): if code != 0: raise Exception('Command %r failed: %d' % (cmd, code)) else: - logging.info('Command %r completed' % cmd) + logging.info('Command %r completed', cmd)