Separate out "GN builder" for the unified build script (#8381)

* Imported a general chip builder script, currently covering a few platforms/boards/apps as a start to have a single build entrypoint

* Move build requirements into global script/requirements.txt so that they get picked up by the bootstrap script

* Update script to assume and require bootstrapping

* Code review comments

* Support building the lock app for ESP32

It turns out that ESP32 lock app is ONLY for devkitc, so
I added application matching logic to include board restrictions.

Tested: lock app compiles and only for devkitc if esp32 platform is
used.

* Remove obsolete todo

* Fix the duplicated accept for efr32 lock app

* Add a dry run option for the build runner, showing what commands would be executed

* Add gn builder as a generic build class, to make gn configuration uniform

* Add support for a "test" to validate that the build generator executes commands as expected

* Update the command comparison: output directory of the build script has also to be considered for testing

* Fix some naming and use `get_target_outputs`

* Address some code review comments

* Rename chipbuild to build_examples

* Fixup naming for the unit tests after build script renaming

* Fix names

* Restyle

* Use difflib instead of diff binary for checking changes

* Fix diffs (generator vs lines) and logging

* Fix unit tests

* Start converting python logic from build into pw_python_module

* Tests pass

* Restyle

* Code review comments

* Add comment for all_platform_commands.txt

* Move expected txt data into inputs

scripts/build/builders/efr32.py

@@ -2,7 +2,7 @@
 import os
 from enum import Enum, auto
 
-from .builder import Builder
+from .gn import GnBuilder
 
 
 class Efr32App(Enum):
@@ -49,36 +49,21 @@ def GnArgName(self):
       return 'BRD4161A'
 
 
-class Efr32Builder(Builder):
+class Efr32Builder(GnBuilder):
 
   def __init__(self,
                root,
                runner,
                output_dir: str,
                app: Efr32App = Efr32App.LIGHT,
                board: Efr32Board = Efr32Board.BRD4161A):
-    super(Efr32Builder, self).__init__(root, runner, output_dir)
+    super(Efr32Builder, self).__init__(
+        root=os.path.join(root, 'examples', app.ExampleName(), 'efr32'),
+        runner=runner,
+        output_dir=output_dir)
 
     self.app = app
-    self.board = board
-    self.identifier = None
-
-  def generate(self):
-    if not os.path.exists(self.output_dir):
-      self._Execute([
-          'gn', 'gen', '--check', '--fail-on-unused-args',
-          '--root=%s' %
-          os.path.join(self.root, 'examples', self.app.ExampleName(), 'efr32'),
-          '--args=efr32_board="%s"' % self.board.GnArgName(), self.output_dir
-      ],
-                    title='Generate %s' % self.identifier)
-
-  def build(self):
-    logging.info('Compiling EFR32 at %s', self.output_dir)
-
-    self.generate()
-    self._Execute(['ninja', '-C', self.output_dir],
-                  title='Build %s' % self.identifier)
+    self.gn_build_args = ['efr32_board="%s"' % board.GnArgName()]
 
   def outputs(self):
     items = {

scripts/build/builders/gn.py

@@ -0,0 +1,38 @@
+import logging
+import os
+
+from .builder import Builder
+
+
+class GnBuilder(Builder):
+
+  def __init__(self, root, runner, output_dir):
+    """Creates  a generic ninja builder.
+
+    Args:
+       root: the root where to run GN into
+       runner: what to use to execute shell commands
+       output_dir: where ninja files are to be generated
+    """
+    super(GnBuilder, self).__init__(root, runner, output_dir)
+
+    self.gn_build_args = None
+
+  def generate(self):
+    if not os.path.exists(self.output_dir):
+      cmd = [
+          'gn', 'gen', '--check', '--fail-on-unused-args',
+          '--root=%s' % self.root
+      ]
+
+      if self.gn_build_args:
+        cmd += ['--args=%s' % ' '.join(self.gn_build_args)]
+
+      cmd += [self.output_dir]
+
+      self._Execute(cmd, title='Generating ' + self.identifier)
+
+  def build(self):
+    self.generate()
+    self._Execute(['ninja', '-C', self.output_dir],
+                  title='Building ' + self.identifier)

scripts/build/builders/linux.py

@@ -1,27 +1,16 @@
 import logging
 import os
 
-from .builder import Builder
+from .gn import GnBuilder
 
 
-class LinuxBuilder(Builder):
+class LinuxBuilder(GnBuilder):
 
   def __init__(self, root, runner, output_dir):
-    super(LinuxBuilder, self).__init__(root, runner, output_dir)
-
-  def generate(self):
-    if not os.path.exists(self.output_dir):
-      self._Execute(['gn', 'gen', self.output_dir],
-                    cwd=os.path.join(self.root,
-                                     'examples/all-clusters-app/linux/'),
-                    title='Generating ' + self.identifier)
-
-  def build(self):
-    logging.info('Compiling Linux at %s', self.output_dir)
-
-    self.generate()
-    self._Execute(['ninja', '-C', self.output_dir],
-                  title='Building ' + self.identifier)
+    super(LinuxBuilder, self).__init__(
+        root=os.path.join(root, 'examples/all-clusters-app/linux/'),
+        runner=runner,
+        output_dir=output_dir)
 
   def outputs(self):
     return {

scripts/build/builders/qpg.py

@@ -3,26 +3,16 @@
 import logging
 import os
 
-from .builder import Builder
+from .gn import GnBuilder
 
 
-class QpgBuilder(Builder):
+class QpgBuilder(GnBuilder):
 
   def __init__(self, root, runner, output_dir):
-    super(QpgBuilder, self).__init__(root, runner, output_dir)
-
-  def generate(self):
-    if not os.path.exists(self.output_dir):
-      self._Execute(['gn', 'gen', self.output_dir],
-                    cwd=os.path.join(self.root, 'examples/lock-app/qpg/'),
-                    title='Generating ' + self.identifier)
-
-  def build(self):
-    logging.info('Compiling QPG at %s', self.output_dir)
-
-    self.generate()
-    self._Execute(['ninja', '-C', self.output_dir],
-                  title='Building ' + self.identifier)
+    super(QpgBuilder, self).__init__(
+        root=os.path.join(root, 'examples/lock-app/qpg/'),
+        runner=runner,
+        output_dir=output_dir)
 
   def outputs(self):
     return {

scripts/build/expected_all_platform_commands.txt

@@ -1,12 +1,8 @@
 # Generating linux-native-all_clusters
-cd "{root}/examples/all-clusters-app/linux/"
-gn gen {out}/linux-native-all_clusters
-cd -
+gn gen --check --fail-on-unused-args --root={root}/examples/all-clusters-app/linux {out}/linux-native-all_clusters
 
 # Generating qpg-qpg6100-lock
-cd "{root}/examples/lock-app/qpg/"
-gn gen {out}/qpg-qpg6100-lock
-cd -
+gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/qpg {out}/qpg-qpg6100-lock
 
 # Generating esp32-m5stack-all_clusters
 cd "{root}"
@@ -23,27 +19,23 @@ cd "{root}"
 bash -c 'source $IDF_PATH/export.sh; idf.py -C examples/lock-app/esp32 -B {out}/esp32-devkitc-lock reconfigure'
 cd -
 
-# Generate efr32-brd4161a-light
+# Generating efr32-brd4161a-light
 gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-light
 
-# Generate efr32-brd4161a-lock
+# Generating efr32-brd4161a-lock
 gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-lock
 
-# Generate efr32-brd4161a-window_covering
+# Generating efr32-brd4161a-window_covering
 gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-window_covering
 
 # Generating linux-native-all_clusters
-cd "{root}/examples/all-clusters-app/linux/"
-gn gen {out}/linux-native-all_clusters
-cd -
+gn gen --check --fail-on-unused-args --root={root}/examples/all-clusters-app/linux {out}/linux-native-all_clusters
 
 # Building linux-native-all_clusters
 ninja -C {out}/linux-native-all_clusters
 
 # Generating qpg-qpg6100-lock
-cd "{root}/examples/lock-app/qpg/"
-gn gen {out}/qpg-qpg6100-lock
-cd -
+gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/qpg {out}/qpg-qpg6100-lock
 
 # Building qpg-qpg6100-lock
 ninja -C {out}/qpg-qpg6100-lock
@@ -72,22 +64,22 @@ cd -
 # Building esp32-devkitc-lock
 bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{out}/esp32-devkitc-lock'"'"''
 
-# Generate efr32-brd4161a-light
+# Generating efr32-brd4161a-light
 gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-light
 
-# Build efr32-brd4161a-light
+# Building efr32-brd4161a-light
 ninja -C {out}/efr32-brd4161a-light
 
-# Generate efr32-brd4161a-lock
+# Generating efr32-brd4161a-lock
 gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-lock
 
-# Build efr32-brd4161a-lock
+# Building efr32-brd4161a-lock
 ninja -C {out}/efr32-brd4161a-lock
 
-# Generate efr32-brd4161a-window_covering
+# Generating efr32-brd4161a-window_covering
 gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-window_covering
 
-# Build efr32-brd4161a-window_covering
+# Building efr32-brd4161a-window_covering
 ninja -C {out}/efr32-brd4161a-window_covering
 
 

scripts/build/runner/shell.py

@@ -50,4 +50,4 @@ def Run(self, cmd, cwd=None, title=None):
       if code != 0:
         raise Exception('Command %r failed: %d' % (cmd, code))
       else:
-        logging.info('Command %r completed' % cmd)
+        logging.info('Command %r completed', cmd)