Unified build script: dry run and unit test (project-chip#8295)

* Imported a general chip builder script, currently covering a few platforms/boards/apps as a start to have a single build entrypoint

* Move build requirements into global script/requirements.txt so that they get picked up by the bootstrap script

* Update script to assume and require bootstrapping

* Code review comments

* Support building the lock app for ESP32

It turns out that ESP32 lock app is ONLY for devkitc, so
I added application matching logic to include board restrictions.

Tested: lock app compiles and only for devkitc if esp32 platform is
used.

* Remove obsolete todo

* Fix the duplicated accept for efr32 lock app

* Add a dry run option for the build runner, showing what commands would be executed

* Add support for a "test" to validate that the build generator executes commands as expected

* Update the command comparison: output directory of the build script has also to be considered for testing

* Fix some naming and use `get_target_outputs`

* Address some code review comments

* Rename chipbuild to build_examples

* Fixup naming for the unit tests after build script renaming

* Fix names

* Restyle

* Use difflib instead of diff binary for checking changes

* Fix diffs (generator vs lines) and logging

* Start converting python logic from build into pw_python_module

* Tests pass

* Restyle

* Code review comments

* Add comment for all_platform_commands.txt

* Move expected txt data into inputs

BUILD.gn

@@ -138,7 +138,10 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
 
   group("check") {
     if (chip_link_tests) {
-      deps = [ "//src:tests_run" ]
+      deps = [
+        "//scripts/build:build_examples.tests",
+        "//src:tests_run",
+      ]
     }
   }
 

scripts/build/BUILD.gn

@@ -0,0 +1,44 @@
+# Copyright (c) 2021 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build_overrides/build.gni")
+import("//build_overrides/chip.gni")
+
+import("//build_overrides/pigweed.gni")
+import("$dir_pw_build/python.gni")
+
+pw_python_package("build_examples") {
+  setup = [ "setup.py" ]
+  inputs = [
+    # Dependency for the unit test
+    "expected_all_platform_commands.txt",
+  ]
+
+  sources = [
+    "build/__init__.py",
+    "build/factory.py",
+    "build/targets.py",
+    "build_examples.py",
+    "builders/__init__.py",
+    "builders/builder.py",
+    "builders/efr32.py",
+    "builders/esp32.py",
+    "builders/linux.py",
+    "builders/qpg.py",
+    "runner/__init__.py",
+    "runner/printonly.py",
+    "runner/shell.py",
+  ]
+  tests = [ "test.py" ]
+}

scripts/build/build/__init__.py

@@ -31,10 +31,10 @@ class Context:
        to generate make/ninja instructions and to compile.
     """
 
-  def __init__(self, repository_path:str, output_prefix:str):
+  def __init__(self, runner, repository_path:str, output_prefix:str):
     self.builders = []
-    self.repository_path = repository_path
-    self.output_prefix = output_prefix
+    self.builder_factory = BuilderFactory(runner, repository_path,
+                                          output_prefix)
     self.completed_steps = set()
 
   def SetupBuilders(self, platforms: Sequence[Platform],
@@ -85,12 +85,10 @@ def SetupBuilders(self, platforms: Sequence[Platform],
     boards_with_builders = set()
     applications_with_builders = set()
 
-    factory = BuilderFactory(self.repository_path, self.output_prefix)
-
-    for platform in platforms:
-      for board in boards:
-        for application in applications:
-          builder = factory.Create(platform, board, application)
+    for platform in sorted(platforms):
+      for board in sorted(boards):
+        for application in sorted(applications):
+          builder = self.builder_factory.Create(platform, board, application)
           if not builder:
             logging.debug('Builder not supported for tuple %s/%s/%s', platform,
                           board, application)

scripts/build/build/factory.py

@@ -41,8 +41,8 @@ def AcceptApplicationForBoard(self, __app_key: Application, __board: Board,
   def AcceptBoard(self, __board_key: Board, **kargs):
     self.board_arguments[__board_key] = kargs
 
-  def Create(self, __board_key: Board, __app_key: Application, repo_path: str,
-             **kargs):
+  def Create(self, runner, __board_key: Board, __app_key: Application,
+             repo_path: str, **kargs):
     """Creates a new builder for the given board/app. """
     if not __board_key in self.board_arguments:
       return None
@@ -59,7 +59,7 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str,
     kargs.update(self.board_arguments[__board_key])
     kargs.update(extra_app_args)
 
-    return self.builder_class(repo_path, **kargs)
+    return self.builder_class(repo_path, runner=runner, **kargs)
 
 
 # Builds a list of acceptable application/board combination for every platform
@@ -96,7 +96,8 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str,
 class BuilderFactory:
   """Creates application builders."""
 
-  def __init__(self, repository_path: str, output_prefix: str):
+  def __init__(self, runner, repository_path: str, output_prefix: str):
+    self.runner = runner
     self.repository_path = repository_path
     self.output_prefix = output_prefix
 
@@ -108,7 +109,11 @@ def Create(self, platform: Platform, board: Board, app: Application):
 
     output_directory = os.path.join(self.output_prefix, identifier)
     builder = _MATCHERS[platform].Create(
-        board, app, self.repository_path, output_dir=output_directory)
+        self.runner,
+        board,
+        app,
+        self.repository_path,
+        output_dir=output_directory)
 
     if builder:
       builder.identifier = identifier

scripts/build/build/targets.py

@@ -2,10 +2,10 @@
 import os
 import shutil
 
-from enum import Enum, auto
+from enum import IntEnum, auto
 
 
-class Platform(Enum):
+class Platform(IntEnum):
   """Represents a supported build platform for compilation."""
   LINUX = auto()
   QPG = auto()
@@ -24,7 +24,7 @@ def FromArgName(name):
     raise KeyError()
 
 
-class Board(Enum):
+class Board(IntEnum):
   """Represents Specific boards within a platform."""
   # Host builds
   NATIVE = auto()
@@ -51,7 +51,7 @@ def FromArgName(name):
     raise KeyError()
 
 
-class Application(Enum):
+class Application(IntEnum):
   """Example applications that can be built."""
   ALL_CLUSTERS = auto()
   LIGHT = auto()

scripts/build/build_examples.py

@@ -9,6 +9,7 @@
 sys.path.append(os.path.abspath(os.path.dirname(__file__)))
 
 import build
+from runner import PrintOnlyRunner, ShellRunner
 
 # Supported log levels, mapping string values required for argument
 # parsing into logging constants
@@ -22,6 +23,10 @@
 
 def ValidateRepoPath(context, parameter, value):
   """Validates that the given path looks like a valid chip repository checkout."""
+  if value.startswith('/TEST/'):
+    # Hackish command to allow for unit testing
+    return value
+
   for name in ['BUILD.gn', '.gn', os.path.join('scripts', 'bootstrap.sh')]:
     expected_file = os.path.join(value, name)
     if not os.path.exists(expected_file):
@@ -73,8 +78,19 @@ def ValidateRepoPath(context, parameter, value):
     default=False,
     is_flag=True,
     help='Clean output directory before running the command')
+@click.option(
+    '--dry-run',
+    default=False,
+    is_flag=True,
+    help='Only print out shell commands that would be executed')
+@click.option(
+    '--dry-run-output',
+    default="-",
+    type=click.File("wt"),
+    help='Where to write the dry run output')
 @click.pass_context
-def main(context, log_level, platform, board, app, repo, out_prefix, clean):
+def main(context, log_level, platform, board, app, repo, out_prefix, clean,
+         dry_run, dry_run_output):
   # Ensures somewhat pretty logging of what is going on
   coloredlogs.install(
       level=__LOG_LEVELS__[log_level],
@@ -92,7 +108,13 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean):
   if 'all' in platform:
     platform = build.PLATFORMS
 
-  context.obj = build.Context(repository_path=repo, output_prefix=out_prefix)
+  if dry_run:
+    runner = PrintOnlyRunner(dry_run_output)
+  else:
+    runner = ShellRunner()
+
+  context.obj = build.Context(
+      repository_path=repo, output_prefix=out_prefix, runner=runner)
   context.obj.SetupBuilders(
       platforms=[build.Platform.FromArgName(name) for name in platform],
       boards=[build.Board.FromArgName(name) for name in board],

scripts/build/builders/builder.py

@@ -5,8 +5,6 @@
 import shutil
 from abc import ABC, abstractmethod
 
-from shellrunner import ShellRunner
-
 
 class Builder(ABC):
   """Generic builder base class for CHIP.
@@ -16,10 +14,11 @@ class Builder(ABC):
 
   """
 
-  def __init__(self, root, output_dir='out'):
+  def __init__(self, root, runner, output_dir='out'):
     self.root = os.path.abspath(root)
-    self._runner = ShellRunner()
+    self._runner = runner
     self.output_dir = output_dir
+    self.identifier = None
 
   @abstractmethod
   def generate(self):
@@ -40,8 +39,8 @@ def outputs(self):
     """
     raise NotImplementedError()
 
-  def _Execute(self, cmdarray, **args):
-    self._runner.Run(cmdarray, **args)
+  def _Execute(self, cmdarray, cwd=None, title=None):
+    self._runner.Run(cmdarray, cwd=cwd, title=title)
 
   def CopyArtifacts(self, target_dir: str):
     for target_name, source_name in self.outputs().items():

scripts/build/builders/efr32.py

@@ -53,13 +53,15 @@ class Efr32Builder(Builder):
 
   def __init__(self,
                root,
+               runner,
                output_dir: str,
                app: Efr32App = Efr32App.LIGHT,
                board: Efr32Board = Efr32Board.BRD4161A):
-    super(Efr32Builder, self).__init__(root, output_dir)
+    super(Efr32Builder, self).__init__(root, runner, output_dir)
 
     self.app = app
     self.board = board
+    self.identifier = None
 
   def generate(self):
     if not os.path.exists(self.output_dir):
@@ -68,13 +70,15 @@ def generate(self):
           '--root=%s' %
           os.path.join(self.root, 'examples', self.app.ExampleName(), 'efr32'),
           '--args=efr32_board="%s"' % self.board.GnArgName(), self.output_dir
-      ])
+      ],
+                    title='Generate %s' % self.identifier)
 
   def build(self):
     logging.info('Compiling EFR32 at %s', self.output_dir)
 
     self.generate()
-    self._Execute(['ninja', '-C', self.output_dir])
+    self._Execute(['ninja', '-C', self.output_dir],
+                  title='Build %s' % self.identifier)
 
   def outputs(self):
     items = {

scripts/build/builders/esp32.py

@@ -52,16 +52,19 @@ class Esp32Builder(Builder):
 
   def __init__(self,
                root,
+               runner,
                output_dir: str,
                board: Esp32Board = Esp32Board.M5Stack,
                app: Esp32App = Esp32App.ALL_CLUSTERS):
-    super(Esp32Builder, self).__init__(root, output_dir)
+    super(Esp32Builder, self).__init__(root, runner, output_dir)
     self.board = board
     self.app = app
 
-  def _IdfEnvExecute(self, cmd, **kargs):
+  def _IdfEnvExecute(self, cmd, cwd=None, title=None):
     self._Execute(
-        ['bash', '-c', 'source $IDF_PATH/export.sh; %s' % cmd], **kargs)
+        ['bash', '-c', 'source $IDF_PATH/export.sh; %s' % cmd],
+        cwd=cwd,
+        title=title)
 
   def generate(self):
     if os.path.exists(os.path.join(self.output_dir, 'build.ninja')):
@@ -77,13 +80,15 @@ def generate(self):
     cmd += ' -C examples/%s/esp32 -B %s reconfigure' % (self.app.ExampleName, shlex.quote(self.output_dir))
 
     # This will do a 'cmake reconfigure' which will create ninja files without rebuilding
-    self._IdfEnvExecute(cmd, cwd=self.root)
+    self._IdfEnvExecute(
+        cmd, cwd=self.root, title='Generating ' + self.identifier)
 
   def build(self):
     logging.info('Compiling Esp32 at %s', self.output_dir)
 
     self.generate()
-    self._IdfEnvExecute("ninja -C '%s'" % self.output_dir)
+    self._IdfEnvExecute(
+        "ninja -C '%s'" % self.output_dir, title='Building ' + self.identifier)
 
   def outputs(self):
     return {

scripts/build/builders/linux.py

@@ -6,20 +6,22 @@
 
 class LinuxBuilder(Builder):
 
-  def __init__(self, root, output_dir):
-    super(LinuxBuilder, self).__init__(root, output_dir)
+  def __init__(self, root, runner, output_dir):
+    super(LinuxBuilder, self).__init__(root, runner, output_dir)
 
   def generate(self):
     if not os.path.exists(self.output_dir):
       self._Execute(['gn', 'gen', self.output_dir],
                     cwd=os.path.join(self.root,
-                                     'examples/all-clusters-app/linux/'))
+                                     'examples/all-clusters-app/linux/'),
+                    title='Generating ' + self.identifier)
 
   def build(self):
     logging.info('Compiling Linux at %s', self.output_dir)
 
     self.generate()
-    self._Execute(['ninja', '-C', self.output_dir])
+    self._Execute(['ninja', '-C', self.output_dir],
+                  title='Building ' + self.identifier)
 
   def outputs(self):
     return {

scripts/build/builders/qpg.py

@@ -8,19 +8,21 @@
 
 class QpgBuilder(Builder):
 
-  def __init__(self, root, output_dir):
-    super(QpgBuilder, self).__init__(root, output_dir)
+  def __init__(self, root, runner, output_dir):
+    super(QpgBuilder, self).__init__(root, runner, output_dir)
 
   def generate(self):
     if not os.path.exists(self.output_dir):
       self._Execute(['gn', 'gen', self.output_dir],
-                    cwd=os.path.join(self.root, 'examples/lock-app/qpg/'))
+                    cwd=os.path.join(self.root, 'examples/lock-app/qpg/'),
+                    title='Generating ' + self.identifier)
 
   def build(self):
     logging.info('Compiling QPG at %s', self.output_dir)
 
     self.generate()
-    self._Execute(['ninja', '-C', self.output_dir])
+    self._Execute(['ninja', '-C', self.output_dir],
+                  title='Building ' + self.identifier)
 
   def outputs(self):
     return {