From 380bdf93d92952022576b2d4036e4def1733a820 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 8 Jul 2021 13:41:28 -0400 Subject: [PATCH 01/21] Imported a general chip builder script, currently covering a few platforms/boards/apps as a start to have a single build entrypoint --- scripts/build/README.md | 58 +++++++++++ scripts/build/build/__init__.py | 157 +++++++++++++++++++++++++++++ scripts/build/build/factory.py | 128 +++++++++++++++++++++++ scripts/build/build/targets.py | 69 +++++++++++++ scripts/build/builders/__init__.py | 0 scripts/build/builders/builder.py | 79 +++++++++++++++ scripts/build/builders/efr32.py | 92 +++++++++++++++++ scripts/build/builders/esp32.py | 52 ++++++++++ scripts/build/builders/linux.py | 30 ++++++ scripts/build/builders/qpg.py | 32 ++++++ scripts/build/chipbuild.py | 130 ++++++++++++++++++++++++ scripts/build/requirements.txt | 2 + scripts/build/shellrunner.py | 50 +++++++++ 13 files changed, 879 insertions(+) create mode 100644 scripts/build/README.md create mode 100644 scripts/build/build/__init__.py create mode 100644 scripts/build/build/factory.py create mode 100644 scripts/build/build/targets.py create mode 100644 scripts/build/builders/__init__.py create mode 100644 scripts/build/builders/builder.py create mode 100644 scripts/build/builders/efr32.py create mode 100644 scripts/build/builders/esp32.py create mode 100644 scripts/build/builders/linux.py create mode 100644 scripts/build/builders/qpg.py create mode 100755 scripts/build/chipbuild.py create mode 100644 scripts/build/requirements.txt create mode 100644 scripts/build/shellrunner.py diff --git a/scripts/build/README.md b/scripts/build/README.md new file mode 100644 index 00000000000000..782ef1aa791041 --- /dev/null +++ b/scripts/build/README.md @@ -0,0 +1,58 @@ +# CHIP automated builds scripts + +CHIP compilation is generally split into two steps + +1. Generate ninja/makefile for out-of-source builds +2. Compilation using ninja/makefiles + +## Building manually + +Manual building is generally platform-dependent. All build steps would require a +bootstrapped environment (loads a pigweed build environment) and will then be +followed by platform-specific instructions. + +The file BUILDING.md describes general requirements and examples. Typical usage +is: + +``` +source scripts/activate +gn gen out/host +ninja -C out/host +``` + +## Unified build script + +The script `chipbuild.py` provides a single entry point for generating and +executing the build. + +Build environment _MUST_ be properly configured for chipbuild to succeed. For +example ESP32 builds requite IDF_PATH to be set. Building in the corresponding +build image or the chip vscode image satisfy the build environment requirement. + +Usage examples: + +1. Compiles the Lock app on all supported platforms + + ``` + ./scripts/build/chipbuild.py --app lock build + ``` + +2. Compile the all clusters app for a ESP32 DevKitC + + ``` + ./scripts/build/chipbuild.py --app all_clusters_app --board devkitc build + ``` + +3. Generate all the makefiles (but do not compile) all native apps + + ``` + ./scripts/build/chipbuild.py --platform native generate --out-prefix out + ``` + +4. Compile the qpg lock app and copy the output in a 'artifact' folder. Note the + argument order (artifact copying is an argument for the build command) + + ``` + ./scripts/build/chipbuild.py --board qpg6100 --app lock build \ + --copy-artifacts-to /tmp/artifacts + ``` diff --git a/scripts/build/build/__init__.py b/scripts/build/build/__init__.py new file mode 100644 index 00000000000000..ff6f959de78f1c --- /dev/null +++ b/scripts/build/build/__init__.py @@ -0,0 +1,157 @@ +import logging +import os +import shutil + +from enum import Enum, auto +from typing import Sequence + +from .targets import Platform, Board, Application +from .factory import BuilderFactory, TargetRelations + + +def CommaSeparate(items) -> str: + return ', '.join([x.ArgName for x in items]) + + +# Supported platforms/boards/apps for generation/compilation +# note that not all combinations are supported, however command-line will +# accept all combinations +PLATFORMS = [x.ArgName for x in Platform] +BOARDS = [x.ArgName for x in Board] +APPLICATIONS = [x.ArgName for x in Application] + +class BuildSteps(Enum): + BOOTSTRAPED = auto() + GENERATED = auto() + + +class Context: + """Represents a grouped list of platform/board/app builders to use + + to generate make/ninja instructions and to compile. + """ + + def __init__(self, repository_path, output_prefix): + self.builders = [] + self.repository_path = repository_path + self.output_prefix = output_prefix + self.completed_steps = set() + + def SetupBuilders(self, platforms: Sequence[Platform], boards: Sequence[Board], + applications: Sequence[Application]): + """Configures internal builders for the given platform/board/app combionation. + + Handles smart default selection, so that users only need to specify + part of platform/board/application information and the method tries + to automatically deduce the rest of the arguments. + """ + if not platforms and not boards: + if applications: + platforms = set().union(*[TargetRelations.PlatformsForApplication(app) for app in applications]) + else: + # when nothing is specified, start with a default host build + platforms = [Platform.LINUX] + + # at this point, at least one of 'platforms' or 'boards' is non-empty + if not boards: + boards = set().union(*[TargetRelations.BoardsForPlatform(platform) for platform in platforms]) + elif not platforms: + platforms = set().union(*[TargetRelations.PlatformsForBoard(board) for board in boards]) + + if not applications: + applications = set().union(*[TargetRelations.ApplicationsForPlatform(platform) for platform in platforms]) + + platforms = set(platforms) + boards = set(boards) + applications = set(applications) + + logging.info('Platforms being built: %s', CommaSeparate(platforms)) + logging.info('Boards being built: %s', CommaSeparate(boards)) + logging.info('Applications being built: %s', CommaSeparate(applications)) + + # Sanity check: ensure all input arguments generate at least an output + platforms_with_builders = set() + boards_with_builders = set() + applications_with_builders = set() + + + factory = BuilderFactory(self.repository_path, self.output_prefix) + + for platform in platforms: + for board in boards: + for application in applications: + builder = factory.Create(platform, board, application) + if not builder: + logging.debug('Builder not supported for tuple %s/%s/%s', platform, + board, application) + continue + + self.builders.append(builder) + platforms_with_builders.add(platform) + boards_with_builders.add(board) + applications_with_builders.add(application) + + if platforms != platforms_with_builders: + logging.warn('Platforms without build output: %s', + CommaSeparate(platforms.difference(platforms_with_builders))) + + if boards != boards_with_builders: + logging.warn('Boards without build output: %s', + CommaSeparate(boards.difference(boards_with_builders))) + + if applications != applications_with_builders: + logging.warn( + 'Applications without build output: %s', + CommaSeparate(applications.difference(applications_with_builders))) + + # whenever builders change, assume generation is required again + self.completed_steps.discard(BuildSteps.GENERATED) + + def Bootstrap(self): + """Performs a bootstrap IFF a bootstrap has not yet been performed.""" + if BuildSteps.BOOTSTRAPED in self.completed_steps: + return + + # Bootstrap is generic. assumption is that any builder can + # bootstrap just the same, so run only once + if self.builders: + self.builders[0].Bootstrap() + + self.completed_steps.add(BuildSteps.BOOTSTRAPED) + + def Generate(self): + """Performs a build generation IFF code generation has not yet been performed.""" + if BuildSteps.GENERATED in self.completed_steps: + return + + self.Bootstrap() + + for builder in self.builders: + logging.info('Generating %s', builder.output_dir) + builder.generate() + + self.completed_steps.add(BuildSteps.GENERATED) + + def Build(self): + self.Generate() + + for builder in self.builders: + logging.info('Building %s', builder.output_dir) + builder.build() + + def CleanOutputDirectories(self): + for builder in self.builders: + logging.warn('Cleaning %s', builder.output_dir) + shutil.rmtree(builder.output_dir) + + # any generated output was cleaned + self.completed_steps.discard(BuildSteps.GENERATED) + + def CopyArtifactsTo(self, path: str): + logging.info("Copying build artifacts to %s" % path) + if not os.path.exists(path): + os.makedirs(path) + + for builder in self.builders: + # FIXME: builder subdir... + builder.CopyArtifacts(os.path.join(path, builder.identifier)) diff --git a/scripts/build/build/factory.py b/scripts/build/build/factory.py new file mode 100644 index 00000000000000..80cc9753d6f47c --- /dev/null +++ b/scripts/build/build/factory.py @@ -0,0 +1,128 @@ +import os + +from typing import Set + +from builders.builder import Builder +from builders.linux import LinuxBuilder +from builders.qpg import QpgBuilder +from builders.esp32 import Esp32Builder, Esp32Board +from builders.efr32 import Efr32Builder, Efr32App, Efr32Board + +from .targets import Application, Board, Platform + + +class Matcher(): + """Figures out if a proper builder can be created for a platform/board/app combination.""" + + def __init__(self, builder_class): + self.builder_class = builder_class + self.app_arguments = {} + self.board_arguments = {} + + def AcceptApplication(self, __app_key: Application, **kargs): + self.app_arguments[__app_key] = kargs + + def AcceptBoard(self, __board_key: Board, **kargs): + self.board_arguments[__board_key] = kargs + + def Create(self, __board_key: Board, __app_key: Application, repo_path: str, **kargs): + """Creates a new builder for the given board/app. """ + if not __board_key in self.board_arguments: + return None + + if not __app_key in self.app_arguments: + return None + + kargs.update(self.board_arguments[__board_key]) + kargs.update(self.app_arguments[__app_key]) + + return self.builder_class(repo_path, **kargs) + + +# Builds a list of acceptable application/board combination for every platform +_MATCHERS = { + Platform.LINUX: Matcher(LinuxBuilder), + Platform.ESP32: Matcher(Esp32Builder), + Platform.QPG: Matcher(QpgBuilder), + Platform.EFR32: Matcher(Efr32Builder), +} + +# Matrix of what can be compiled using and what build options are required +# by such compilation +_MATCHERS[Platform.LINUX].AcceptApplication(Application.ALL_CLUSTERS) +_MATCHERS[Platform.LINUX].AcceptBoard(Board.NATIVE) + +_MATCHERS[Platform.ESP32].AcceptApplication(Application.ALL_CLUSTERS) +_MATCHERS[Platform.ESP32].AcceptBoard(Board.DEVKITC, board=Esp32Board.DevKitC) +_MATCHERS[Platform.ESP32].AcceptBoard(Board.M5STACK, board=Esp32Board.M5Stack) + +_MATCHERS[Platform.QPG].AcceptApplication(Application.LOCK) +_MATCHERS[Platform.QPG].AcceptBoard(Board.QPG6100) + +_MATCHERS[Platform.EFR32].AcceptApplication(Application.LOCK) +_MATCHERS[Platform.EFR32].AcceptBoard( + Board.BRD4161A, board=Efr32Board.BRD4161A) +_MATCHERS[Platform.EFR32].AcceptApplication( + Application.LIGHT, app=Efr32App.LIGHT) +_MATCHERS[Platform.EFR32].AcceptApplication( + Application.LOCK, app=Efr32App.LOCK) +_MATCHERS[Platform.EFR32].AcceptApplication( + Application.WINDOW_COVERING, app=Efr32App.WINDOW_COVERING) + + +class BuilderFactory: + """Creates application builders.""" + + def __init__(self, repository_path: str, output_prefix: str): + self.repository_path = repository_path + self.output_prefix = output_prefix + + def Create(self, platform: Platform, board: Board, app: Application): + """Creates a builder object for the specified arguments. """ + + identifier = '%s-%s-%s' % (platform.name.lower(), board.name.lower(), + app.name.lower()) + + output_directory = os.path.join(self.output_prefix, identifier) + builder = _MATCHERS[platform].Create( + board, app, self.repository_path, output_dir=output_directory) + + if builder: + builder.identifier = identifier + + return builder + + +class TargetRelations: + """Figures out valid combinations of boards/platforms/applications.""" + + @staticmethod + def BoardsForPlatform(platform: Platform) -> Set[Board]: + global _MATCHERS + return set(_MATCHERS[platform].board_arguments.keys()) + + @staticmethod + def PlatformsForBoard(board: Board) -> Set[Platform]: + """Return the platforms that are using the specified board.""" + global _MATCHERS + platforms = set() + for platform, matcher in _MATCHERS.items(): + if board in matcher.board_arguments: + platforms.add(platform) + return platforms + + @staticmethod + def ApplicationsForPlatform(platform: Platform) -> Set[Application]: + """What applications are buildable for a specific platform.""" + global _MATCHERS + return set(_MATCHERS[platform].app_arguments.keys()) + + @staticmethod + def PlatformsForApplication(application: Application) -> Set[Platform]: + """For what platforms can the given application be compiled.""" + global _MATCHERS + platforms = set() + for platform, matcher in _MATCHERS.items(): + if application in matcher.app_arguments: + platforms.add(platform) + return platforms diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py new file mode 100644 index 00000000000000..7b8303604a5953 --- /dev/null +++ b/scripts/build/build/targets.py @@ -0,0 +1,69 @@ +import logging +import os +import shutil + +from enum import Enum, auto + +class Platform(Enum): + """Represents a supported build platform for compilation.""" + LINUX = auto() + QPG = auto() + ESP32 = auto() + EFR32 = auto() + + @property + def ArgName(self): + return self.name.lower() + + @staticmethod + def FromArgName(name): + for value in Platform: + if name == value.ArgName: + return value + raise KeyError() + + +class Board(Enum): + """Represents Specific boards within a platform.""" + # Host builds + NATIVE = auto() + + # QPG platform + QPG6100 = auto() + + # ESP32 platform + M5STACK = auto() + DEVKITC = auto() + + # EFR32 platform + BRD4161A = auto() + + @property + def ArgName(self): + return self.name.lower() + + @staticmethod + def FromArgName(name): + for value in Board: + if name == value.ArgName: + return value + raise KeyError() + + +class Application(Enum): + """Example applications that can be built.""" + ALL_CLUSTERS = auto() + LIGHT = auto() + LOCK = auto() + WINDOW_COVERING = auto() + + @property + def ArgName(self): + return self.name.lower().replace('_', '-') + + @staticmethod + def FromArgName(name): + for value in Application: + if name == value.ArgName: + return value + raise KeyError() diff --git a/scripts/build/builders/__init__.py b/scripts/build/builders/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/scripts/build/builders/builder.py b/scripts/build/builders/builder.py new file mode 100644 index 00000000000000..c10e848b8f25b9 --- /dev/null +++ b/scripts/build/builders/builder.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +import logging +import os +import shutil +from abc import ABC, abstractmethod + +from shellrunner import ShellRunner + + +class Builder(ABC): + """Generic builder base class for CHIP. + + Provides ability to boostrap and copy output artifacts and subclasses can use + a generic shell runner. + + """ + + def __init__(self, root, output_dir='out'): + self.root = os.path.abspath(root) + self._runner = ShellRunner() + self.output_dir = output_dir + + @abstractmethod + def generate(self): + """Generate the build files - generally the ninja/makefiles""" + raise NotImplementedError() + + @abstractmethod + def build(self): + """Perform an actual build""" + raise NotImplementedError() + + @abstractmethod + def outputs(self): + """Return a list of relevant output files after a build. + + May use build output data (e.g. manifests), so this should be invoked + only after a build has succeeded. + """ + raise NotImplementedError() + + def _Execute(self, cmdarray, **args): + self._runner.Run(cmdarray, **args) + + def _ActivatedExecute(self, cmdstring: str, **args): + """Switches to top level to activate then runs the given command in the original run directory.""" + self._runner.Run([ + 'bash', '-c', + 'cd "%s"; source ./scripts/activate.sh; cd -; %s' % + (self.root, cmdstring) + ], **args) + + def Bootstrap(self): + envpath = os.environ.get('PW_ENVIRONMENT_ROOT', + os.path.join(self.root, '.environment')) + + if not os.path.exists(os.path.join(envpath, 'activate.sh')): + logging.info('Bootstrapping in %s (%s does not look valid )', self.root, + envpath) + self._Execute(['bash', '-c', 'source ./scripts/bootstrap.sh'], + cwd=self.root) + else: + logging.info('Project already bootstrapped in %s (environment in %s)', + self.root, envpath) + + def CopyArtifacts(self, target_dir: str): + for target_name, source_name in self.outputs().items(): + target_full_name = os.path.join(target_dir, target_name) + + logging.info('Copying %s into %s', source_name, target_name) + + target_dir_full_name = os.path.dirname(target_full_name) + + if not os.path.exists(target_dir_full_name): + logging.info(' Creating subdirectory %s first', target_dir_full_name) + os.makedirs(target_dir_full_name) + + shutil.copyfile(source_name, target_full_name) diff --git a/scripts/build/builders/efr32.py b/scripts/build/builders/efr32.py new file mode 100644 index 00000000000000..2fdb52d49d55b8 --- /dev/null +++ b/scripts/build/builders/efr32.py @@ -0,0 +1,92 @@ +import logging +import os +from enum import Enum, auto + +from .builder import Builder + + +class Efr32App(Enum): + LIGHT = auto() + LOCK = auto() + WINDOW_COVERING = auto() + + def ExampleName(self): + if self == Efr32App.LIGHT: + return 'lighting-app' + elif self == Efr32App.LOCK: + return 'lock-app' + elif self == Efr32App.WINDOW_COVERING: + return 'window-app' + else: + raise Exception('Unknown app type: %r' % self) + + def AppNamePrefix(self): + if self == Efr32App.LIGHT: + return 'chip-efr32-lighting-example' + elif self == Efr32App.LOCK: + return 'chip-efr32-lock-example' + elif self == Efr32App.WINDOW_COVERING: + return 'chip-efr32-window-example' + else: + raise Exception('Unknown app type: %r' % self) + + def FlashBundleName(self): + if self == Efr32App.LIGHT: + return 'lighting_app.flashbundle.txt' + elif self == Efr32App.LOCK: + return 'lock_app.flashbundle.txt' + elif self == Efr32App.WINDOW_COVERING: + return 'window_app.flashbundle.txt' + else: + raise Exception('Unknown app type: %r' % self) + + +class Efr32Board(Enum): + BRD4161A = 1 + + def GnArgName(self): + if self == Efr32Board.BRD4161A: + return 'BRD4161A' + + +class Efr32Builder(Builder): + + def __init__(self, + root, + output_dir: str, + app: Efr32App = Efr32App.LIGHT, + board: Efr32Board = Efr32Board.BRD4161A): + super(Efr32Builder, self).__init__(root, output_dir) + + self.app = app + self.board = board + + def generate(self): + if not os.path.exists(self.output_dir): + self._ActivatedExecute( + 'gn gen --check --fail-on-unused-args --root="%s" --args="efr32_board=\\"%s\\"" %s' + % (os.path.join(self.root, 'examples', self.app.ExampleName(), + 'efr32'), self.board.GnArgName(), self.output_dir)) + + def build(self): + logging.info('Compiling EFR32 at %s', self.output_dir) + + self.generate() + self._ActivatedExecute('ninja -C %s' % self.output_dir) + + def outputs(self): + items = { + '%s.out' % self.app.AppNamePrefix(): + os.path.join(self.output_dir, '%s.out' % self.app.AppNamePrefix()), + '%s.out.map' % self.app.AppNamePrefix(): + os.path.join(self.output_dir, + '%s.out.map' % self.app.AppNamePrefix()), + } + + # Figure out flash bundle files and build accordingly + with open(os.path.join(self.output_dir, self.app.FlashBundleName())) as f: + for line in f.readlines(): + name = line.strip() + items['flashbundle/%s' % name] = os.path.join(self.output_dir, name) + + return items diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py new file mode 100644 index 00000000000000..a47a5c91e2641f --- /dev/null +++ b/scripts/build/builders/esp32.py @@ -0,0 +1,52 @@ +import logging +import os +from enum import Enum + +from .builder import Builder + + +class Esp32Board(Enum): + DevKitC = 1 + M5Stack = 2 + + def DefaultsFileName(self): + if self == Esp32Board.DevKitC: + return 'sdkconfig_devkit.defaults' + elif self == Esp32Board.M5Stack: + return 'sdkconfig_m5stack.defaults' + raise Exception('Unknown board type') + + +class Esp32Builder(Builder): + + def __init__(self, + root, + output_dir: str, + board: Esp32Board = Esp32Board.M5Stack): + super(Esp32Builder, self).__init__(root, output_dir) + self.board = board + + def _IdfEnvExecute(self, cmd, **kargs): + self._ActivatedExecute('source "$IDF_PATH/export.sh"; %s' % cmd, **kargs) + + def generate(self): + if not os.path.exists(os.path.join(self.output_dir, 'build.ninja')): + # This will do a 'cmake reconfigure' which will create ninja files without rebuilding + self._IdfEnvExecute( + "idf.py -D SDKCONFIG_DEFAULTS='%s' -C examples/all-clusters-app/esp32 -B %s reconfigure" + % (self.board.DefaultsFileName(), self.output_dir), + cwd=self.root) + + def build(self): + logging.info('Compiling Esp32 at %s', self.output_dir) + + self.generate() + self._IdfEnvExecute("ninja -C '%s'" % self.output_dir) + + def outputs(self): + return { + 'chip-all-clusters-app.elf': + os.path.join(self.output_dir, 'chip-all-clusters-app.elf'), + 'chip-all-clusters-app.map': + os.path.join(self.output_dir, 'chip-all-clusters-app.map'), + } diff --git a/scripts/build/builders/linux.py b/scripts/build/builders/linux.py new file mode 100644 index 00000000000000..fe85ef5c1d17fe --- /dev/null +++ b/scripts/build/builders/linux.py @@ -0,0 +1,30 @@ +import logging +import os + +from .builder import Builder + + +class LinuxBuilder(Builder): + + def __init__(self, root, output_dir): + super(LinuxBuilder, self).__init__(root, output_dir) + + def generate(self): + if not os.path.exists(self.output_dir): + self._ActivatedExecute( + 'gn gen %s' % self.output_dir, + cwd=os.path.join(self.root, 'examples/all-clusters-app/linux/')) + + def build(self): + logging.info('Compiling Linux at %s', self.output_dir) + + self.generate() + self._ActivatedExecute('ninja -C %s' % self.output_dir) + + def outputs(self): + return { + 'chip-all-clusters-app': + os.path.join(self.output_dir, 'chip-all-clusters-app'), + 'chip-all-clusters-app.map': + os.path.join(self.output_dir, 'chip-all-clusters-app.map'), + } diff --git a/scripts/build/builders/qpg.py b/scripts/build/builders/qpg.py new file mode 100644 index 00000000000000..ed10099a8017f6 --- /dev/null +++ b/scripts/build/builders/qpg.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import logging +import os + +from .builder import Builder + + +class QpgBuilder(Builder): + + def __init__(self, root, output_dir): + super(QpgBuilder, self).__init__(root, output_dir) + + def generate(self): + if not os.path.exists(self.output_dir): + self._ActivatedExecute( + 'gn gen %s' % self.output_dir, + cwd=os.path.join(self.root, 'examples/lock-app/qpg/')) + + def build(self): + logging.info('Compiling QPG at %s', self.output_dir) + + self.generate() + self._ActivatedExecute('ninja -C %s' % self.output_dir) + + def outputs(self): + return { + 'chip-qpg-lock-example.out': + os.path.join(self.output_dir, 'chip-qpg6100-lock-example.out'), + 'chip-qpg-lock-example.out.map': + os.path.join(self.output_dir, 'chip-qpg6100-lock-example.out.map'), + } diff --git a/scripts/build/chipbuild.py b/scripts/build/chipbuild.py new file mode 100755 index 00000000000000..65a9565bea656a --- /dev/null +++ b/scripts/build/chipbuild.py @@ -0,0 +1,130 @@ +#!/usr/bin/env -S python3 -B + +import coloredlogs +import click +import logging +import os +import sys + +sys.path.append(os.path.abspath(os.path.dirname(__file__))) + +import build + +# Supported log levels, mapping string values required for argument +# parsing into logging constants +__LOG_LEVELS__ = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warn': logging.WARN, + 'fatal': logging.FATAL, +} + + +def ValidateRepoPath(context, parameter, value): + """Validates that the given path looks like a valid chip repository checkout.""" + for name in ['BUILD.gn', '.gn', os.path.join('scripts', 'bootstrap.sh')]: + expected_file = os.path.join(value, name) + if not os.path.exists(expected_file): + raise click.BadParameter( + "'%s' does not look like a valid repository path: %s not found." % + (value, expected_file)) + return value + + +@click.group(chain=True) +@click.option( + '--log-level', + default='INFO', + type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False), + help='Determines the verbosity of script output.') +@click.option( + '--platform', + default=[], + type=click.Choice(build.PLATFORMS + ['all'], case_sensitive=False), + multiple=True, + help='Platform to use for compilation. Empty will default to a linux host build' +) +@click.option( + '--board', + default=[], + type=click.Choice(build.BOARDS, case_sensitive=False), + multiple=True, + help='Specific board to compile for. Empty will use --platform to determine suitable boards.' +) +@click.option( + '--app', + default=[], + type=click.Choice(build.APPLICATIONS, case_sensitive=False), + multiple=True, + help='What example application to build. Empty will find suitable applications.' +) +@click.option( + '--repo', + default='.', + callback=ValidateRepoPath, + help='Path to the root of the CHIP SDK repository checkout.') +@click.option( + '--out-prefix', + default='./out', + type=click.Path(file_okay=False, resolve_path=True), + help='Prefix for the generated file output.') +@click.option( + '--clean', + default=False, + is_flag=True, + help='Clean output directory before running the command') +@click.pass_context +def main(context, log_level, platform, board, app, repo, out_prefix, clean): + # Ensures somewhat pretty logging of what is going on + coloredlogs.install( + level=__LOG_LEVELS__[log_level], + fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s') + + # Support an 'all platforms' choice + if 'all' in platform: + platform = build.PLATFORMS + + context.obj = build.Context(repository_path=repo, output_prefix=out_prefix) + context.obj.SetupBuilders( + platforms=[build.Platform.FromArgName(name) for name in platform], + boards=[build.Board.FromArgName(name) for name in board], + applications=[build.Application.FromArgName(name) for name in app]) + + if clean: + context.obj.CleanOutputDirectories() + + +@main.command( + 'bootstrap', + help='Run build environment bootstrapping/download required binaries') +@click.pass_context +def cmd_build(context): + context.obj.Bootstrap() + + +@main.command( + 'gen', + help='Bootstrap and generate ninja/makefiles (but does not run the compilation)' +) +@click.pass_context +def cmd_generate(context): + context.obj.Generate() + + +@main.command( + 'build', help='Bootstrap, generate and run ninja/make as needed to compile') +@click.option( + '--copy-artifacts-to', + default=None, + type=click.Path(file_okay=False, resolve_path=True), + help='Prefix for the generated file output.') +@click.pass_context +def cmd_build(context, copy_artifacts_to): + context.obj.Build() + + if copy_artifacts_to: + context.obj.CopyArtifactsTo(copy_artifacts_to) + + +if __name__ == '__main__': + main() diff --git a/scripts/build/requirements.txt b/scripts/build/requirements.txt new file mode 100644 index 00000000000000..01a87e07af3f6d --- /dev/null +++ b/scripts/build/requirements.txt @@ -0,0 +1,2 @@ +coloredlogs==15.0 +click==8.0.1 diff --git a/scripts/build/shellrunner.py b/scripts/build/shellrunner.py new file mode 100644 index 00000000000000..7d2b4bdee599cd --- /dev/null +++ b/scripts/build/shellrunner.py @@ -0,0 +1,50 @@ +import logging +import os +import subprocess +import threading + + +class LogPipe(threading.Thread): + + def __init__(self, level): + """Setup the object with a logger and a loglevel + + and start the thread + """ + threading.Thread.__init__(self) + self.daemon = False + self.level = level + self.fdRead, self.fdWrite = os.pipe() + self.pipeReader = os.fdopen(self.fdRead) + self.start() + + def fileno(self): + """Return the write file descriptor of the pipe""" + return self.fdWrite + + def run(self): + """Run the thread, logging everything.""" + for line in iter(self.pipeReader.readline, ''): + logging.log(self.level, line.strip('\n')) + + self.pipeReader.close() + + def close(self): + """Close the write end of the pipe.""" + os.close(self.fdWrite) + + +class ShellRunner: + + def Run(self, *args, **kargs): + outpipe = LogPipe(logging.INFO) + errpipe = LogPipe(logging.WARN) + + with subprocess.Popen(*args, **kargs, stdout=outpipe, stderr=errpipe) as s: + outpipe.close() + errpipe.close() + code = s.wait() + if code != 0: + raise Exception('Command %r failed: %d' % (args, code)) + else: + logging.info('Command %r completed', args) From 38c051296dec1e3e16809fd432988fda8cf47815 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 8 Jul 2021 16:54:17 -0400 Subject: [PATCH 02/21] Move build requirements into global script/requirements.txt so that they get picked up by the bootstrap script --- scripts/build/requirements.txt | 2 -- scripts/requirements.txt | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 scripts/build/requirements.txt diff --git a/scripts/build/requirements.txt b/scripts/build/requirements.txt deleted file mode 100644 index 01a87e07af3f6d..00000000000000 --- a/scripts/build/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -coloredlogs==15.0 -click==8.0.1 diff --git a/scripts/requirements.txt b/scripts/requirements.txt index eb43a3f5bffa0d..ccb50602759861 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -42,3 +42,6 @@ protobuf anytree cxxfilt pandas ; platform_machine != 'aarch64' + +# scripts/build +click From a7d619fc74c9bb3635b5402b263162db33853b1a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 8 Jul 2021 17:18:14 -0400 Subject: [PATCH 03/21] Update script to assume and require bootstrapping --- scripts/build/build/__init__.py | 44 +++++++++++++------------------ scripts/build/build/factory.py | 9 +++---- scripts/build/build/targets.py | 1 + scripts/build/builders/builder.py | 21 --------------- scripts/build/builders/efr32.py | 12 +++++---- scripts/build/builders/esp32.py | 3 ++- scripts/build/builders/linux.py | 8 +++--- scripts/build/builders/qpg.py | 7 +++-- scripts/build/chipbuild.py | 23 +++++++--------- 9 files changed, 50 insertions(+), 78 deletions(-) diff --git a/scripts/build/build/__init__.py b/scripts/build/build/__init__.py index ff6f959de78f1c..7e1f94dada8249 100644 --- a/scripts/build/build/__init__.py +++ b/scripts/build/build/__init__.py @@ -20,8 +20,8 @@ def CommaSeparate(items) -> str: BOARDS = [x.ArgName for x in Board] APPLICATIONS = [x.ArgName for x in Application] + class BuildSteps(Enum): - BOOTSTRAPED = auto() GENERATED = auto() @@ -37,7 +37,8 @@ def __init__(self, repository_path, output_prefix): self.output_prefix = output_prefix self.completed_steps = set() - def SetupBuilders(self, platforms: Sequence[Platform], boards: Sequence[Board], + def SetupBuilders(self, platforms: Sequence[Platform], + boards: Sequence[Board], applications: Sequence[Application]): """Configures internal builders for the given platform/board/app combionation. @@ -47,19 +48,27 @@ def SetupBuilders(self, platforms: Sequence[Platform], boards: Sequence[Board], """ if not platforms and not boards: if applications: - platforms = set().union(*[TargetRelations.PlatformsForApplication(app) for app in applications]) + platforms = set().union(*[ + TargetRelations.PlatformsForApplication(app) for app in applications + ]) else: # when nothing is specified, start with a default host build platforms = [Platform.LINUX] # at this point, at least one of 'platforms' or 'boards' is non-empty if not boards: - boards = set().union(*[TargetRelations.BoardsForPlatform(platform) for platform in platforms]) + boards = set().union(*[ + TargetRelations.BoardsForPlatform(platform) for platform in platforms + ]) elif not platforms: - platforms = set().union(*[TargetRelations.PlatformsForBoard(board) for board in boards]) + platforms = set().union( + *[TargetRelations.PlatformsForBoard(board) for board in boards]) if not applications: - applications = set().union(*[TargetRelations.ApplicationsForPlatform(platform) for platform in platforms]) + applications = set().union(*[ + TargetRelations.ApplicationsForPlatform(platform) + for platform in platforms + ]) platforms = set(platforms) boards = set(boards) @@ -74,7 +83,6 @@ def SetupBuilders(self, platforms: Sequence[Platform], boards: Sequence[Board], boards_with_builders = set() applications_with_builders = set() - factory = BuilderFactory(self.repository_path, self.output_prefix) for platform in platforms: @@ -107,25 +115,11 @@ def SetupBuilders(self, platforms: Sequence[Platform], boards: Sequence[Board], # whenever builders change, assume generation is required again self.completed_steps.discard(BuildSteps.GENERATED) - def Bootstrap(self): - """Performs a bootstrap IFF a bootstrap has not yet been performed.""" - if BuildSteps.BOOTSTRAPED in self.completed_steps: - return - - # Bootstrap is generic. assumption is that any builder can - # bootstrap just the same, so run only once - if self.builders: - self.builders[0].Bootstrap() - - self.completed_steps.add(BuildSteps.BOOTSTRAPED) - def Generate(self): """Performs a build generation IFF code generation has not yet been performed.""" if BuildSteps.GENERATED in self.completed_steps: return - self.Bootstrap() - for builder in self.builders: logging.info('Generating %s', builder.output_dir) builder.generate() @@ -148,10 +142,10 @@ def CleanOutputDirectories(self): self.completed_steps.discard(BuildSteps.GENERATED) def CopyArtifactsTo(self, path: str): - logging.info("Copying build artifacts to %s" % path) + logging.info('Copying build artifacts to %s', path) if not os.path.exists(path): - os.makedirs(path) + os.makedirs(path) for builder in self.builders: - # FIXME: builder subdir... - builder.CopyArtifacts(os.path.join(path, builder.identifier)) + # FIXME: builder subdir... + builder.CopyArtifacts(os.path.join(path, builder.identifier)) diff --git a/scripts/build/build/factory.py b/scripts/build/build/factory.py index 80cc9753d6f47c..9ceb85bd709b87 100644 --- a/scripts/build/build/factory.py +++ b/scripts/build/build/factory.py @@ -25,7 +25,8 @@ def AcceptApplication(self, __app_key: Application, **kargs): def AcceptBoard(self, __board_key: Board, **kargs): self.board_arguments[__board_key] = kargs - def Create(self, __board_key: Board, __app_key: Application, repo_path: str, **kargs): + def Create(self, __board_key: Board, __app_key: Application, repo_path: str, + **kargs): """Creates a new builder for the given board/app. """ if not __board_key in self.board_arguments: return None @@ -60,12 +61,10 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, **k _MATCHERS[Platform.QPG].AcceptBoard(Board.QPG6100) _MATCHERS[Platform.EFR32].AcceptApplication(Application.LOCK) -_MATCHERS[Platform.EFR32].AcceptBoard( - Board.BRD4161A, board=Efr32Board.BRD4161A) +_MATCHERS[Platform.EFR32].AcceptBoard(Board.BRD4161A, board=Efr32Board.BRD4161A) _MATCHERS[Platform.EFR32].AcceptApplication( Application.LIGHT, app=Efr32App.LIGHT) -_MATCHERS[Platform.EFR32].AcceptApplication( - Application.LOCK, app=Efr32App.LOCK) +_MATCHERS[Platform.EFR32].AcceptApplication(Application.LOCK, app=Efr32App.LOCK) _MATCHERS[Platform.EFR32].AcceptApplication( Application.WINDOW_COVERING, app=Efr32App.WINDOW_COVERING) diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py index 7b8303604a5953..9dc9dbddc89122 100644 --- a/scripts/build/build/targets.py +++ b/scripts/build/build/targets.py @@ -4,6 +4,7 @@ from enum import Enum, auto + class Platform(Enum): """Represents a supported build platform for compilation.""" LINUX = auto() diff --git a/scripts/build/builders/builder.py b/scripts/build/builders/builder.py index c10e848b8f25b9..f437ae62e71519 100644 --- a/scripts/build/builders/builder.py +++ b/scripts/build/builders/builder.py @@ -43,27 +43,6 @@ def outputs(self): def _Execute(self, cmdarray, **args): self._runner.Run(cmdarray, **args) - def _ActivatedExecute(self, cmdstring: str, **args): - """Switches to top level to activate then runs the given command in the original run directory.""" - self._runner.Run([ - 'bash', '-c', - 'cd "%s"; source ./scripts/activate.sh; cd -; %s' % - (self.root, cmdstring) - ], **args) - - def Bootstrap(self): - envpath = os.environ.get('PW_ENVIRONMENT_ROOT', - os.path.join(self.root, '.environment')) - - if not os.path.exists(os.path.join(envpath, 'activate.sh')): - logging.info('Bootstrapping in %s (%s does not look valid )', self.root, - envpath) - self._Execute(['bash', '-c', 'source ./scripts/bootstrap.sh'], - cwd=self.root) - else: - logging.info('Project already bootstrapped in %s (environment in %s)', - self.root, envpath) - def CopyArtifacts(self, target_dir: str): for target_name, source_name in self.outputs().items(): target_full_name = os.path.join(target_dir, target_name) diff --git a/scripts/build/builders/efr32.py b/scripts/build/builders/efr32.py index 2fdb52d49d55b8..978a15aeed5169 100644 --- a/scripts/build/builders/efr32.py +++ b/scripts/build/builders/efr32.py @@ -63,16 +63,18 @@ def __init__(self, def generate(self): if not os.path.exists(self.output_dir): - self._ActivatedExecute( - 'gn gen --check --fail-on-unused-args --root="%s" --args="efr32_board=\\"%s\\"" %s' - % (os.path.join(self.root, 'examples', self.app.ExampleName(), - 'efr32'), self.board.GnArgName(), self.output_dir)) + self._Execute([ + 'gn', 'gen', '--check', '--fail-on-unused-args', + '--root=%s' % + os.path.join(self.root, 'examples', self.app.ExampleName(), 'efr32'), + '--args=efr32_board="%s"' % self.board.GnArgName(), self.output_dir + ]) def build(self): logging.info('Compiling EFR32 at %s', self.output_dir) self.generate() - self._ActivatedExecute('ninja -C %s' % self.output_dir) + self._Execute(['ninja', '-C', self.output_dir]) def outputs(self): items = { diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py index a47a5c91e2641f..c197503eb1f5bd 100644 --- a/scripts/build/builders/esp32.py +++ b/scripts/build/builders/esp32.py @@ -27,7 +27,8 @@ def __init__(self, self.board = board def _IdfEnvExecute(self, cmd, **kargs): - self._ActivatedExecute('source "$IDF_PATH/export.sh"; %s' % cmd, **kargs) + self._Execute( + ['bash', '-c', 'source $IDF_PATH/export.sh; %s' % cmd], **kargs) def generate(self): if not os.path.exists(os.path.join(self.output_dir, 'build.ninja')): diff --git a/scripts/build/builders/linux.py b/scripts/build/builders/linux.py index fe85ef5c1d17fe..77713924626e9c 100644 --- a/scripts/build/builders/linux.py +++ b/scripts/build/builders/linux.py @@ -11,15 +11,15 @@ def __init__(self, root, output_dir): def generate(self): if not os.path.exists(self.output_dir): - self._ActivatedExecute( - 'gn gen %s' % self.output_dir, - cwd=os.path.join(self.root, 'examples/all-clusters-app/linux/')) + self._Execute(['gn', 'gen', self.output_dir], + cwd=os.path.join(self.root, + 'examples/all-clusters-app/linux/')) def build(self): logging.info('Compiling Linux at %s', self.output_dir) self.generate() - self._ActivatedExecute('ninja -C %s' % self.output_dir) + self._Execute(['ninja', '-C', self.output_dir]) def outputs(self): return { diff --git a/scripts/build/builders/qpg.py b/scripts/build/builders/qpg.py index ed10099a8017f6..dbf7c420380fed 100644 --- a/scripts/build/builders/qpg.py +++ b/scripts/build/builders/qpg.py @@ -13,15 +13,14 @@ def __init__(self, root, output_dir): def generate(self): if not os.path.exists(self.output_dir): - self._ActivatedExecute( - 'gn gen %s' % self.output_dir, - cwd=os.path.join(self.root, 'examples/lock-app/qpg/')) + self._Execute(['gn', 'gen', self.output_dir], + cwd=os.path.join(self.root, 'examples/lock-app/qpg/')) def build(self): logging.info('Compiling QPG at %s', self.output_dir) self.generate() - self._ActivatedExecute('ninja -C %s' % self.output_dir) + self._Execute(['ninja', '-C', self.output_dir]) def outputs(self): return { diff --git a/scripts/build/chipbuild.py b/scripts/build/chipbuild.py index 65a9565bea656a..de350f5fe8f9c5 100755 --- a/scripts/build/chipbuild.py +++ b/scripts/build/chipbuild.py @@ -80,6 +80,14 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean): level=__LOG_LEVELS__[log_level], fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s') + if not 'PW_PROJECT_ROOT' in os.environ: + raise click.UsageError(""" +PW_PROJECT_ROOT not in current environment. + +Please make sure you `source scripts/bootstra.sh` or `source scripts/activate.sh` +before running this script. +""".strip()) + # Support an 'all platforms' choice if 'all' in platform: platform = build.PLATFORMS @@ -95,24 +103,13 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean): @main.command( - 'bootstrap', - help='Run build environment bootstrapping/download required binaries') -@click.pass_context -def cmd_build(context): - context.obj.Bootstrap() - - -@main.command( - 'gen', - help='Bootstrap and generate ninja/makefiles (but does not run the compilation)' -) + 'gen', help='Generate ninja/makefiles (but does not run the compilation)') @click.pass_context def cmd_generate(context): context.obj.Generate() -@main.command( - 'build', help='Bootstrap, generate and run ninja/make as needed to compile') +@main.command('build', help='generate and run ninja/make as needed to compile') @click.option( '--copy-artifacts-to', default=None, From e0f7e91d80cdb9cd48534f7db9363d23815b7ac2 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 9 Jul 2021 09:27:53 -0400 Subject: [PATCH 04/21] Code review comments --- scripts/build/README.md | 10 ++++++++-- scripts/build/build/factory.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/build/README.md b/scripts/build/README.md index 782ef1aa791041..d4e464a6d2e06d 100644 --- a/scripts/build/README.md +++ b/scripts/build/README.md @@ -46,10 +46,16 @@ Usage examples: 3. Generate all the makefiles (but do not compile) all native apps ``` - ./scripts/build/chipbuild.py --platform native generate --out-prefix out + ./scripts/build/chipbuild.py --platform native generate ``` -4. Compile the qpg lock app and copy the output in a 'artifact' folder. Note the +4. Generate all the makefiles (but do not compile) using a specific output root + + ``` + ./scripts/build/chipbuild.py --platform native generate --out-prefix ./mydir + ``` + +5. Compile the qpg lock app and copy the output in a 'artifact' folder. Note the argument order (artifact copying is an argument for the build command) ``` diff --git a/scripts/build/build/factory.py b/scripts/build/build/factory.py index 9ceb85bd709b87..e1cbb984105266 100644 --- a/scripts/build/build/factory.py +++ b/scripts/build/build/factory.py @@ -48,7 +48,7 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, Platform.EFR32: Matcher(Efr32Builder), } -# Matrix of what can be compiled using and what build options are required +# Matrix of what can be compiled and what build options are required # by such compilation _MATCHERS[Platform.LINUX].AcceptApplication(Application.ALL_CLUSTERS) _MATCHERS[Platform.LINUX].AcceptBoard(Board.NATIVE) From ba4192041d606dc4e39aae380bd59ce95ca262f7 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 9 Jul 2021 10:51:11 -0400 Subject: [PATCH 05/21] Support building the lock app for ESP32 It turns out that ESP32 lock app is ONLY for devkitc, so I added application matching logic to include board restrictions. Tested: lock app compiles and only for devkitc if esp32 platform is used. --- scripts/build/build/factory.py | 46 +++++++++++++++---- scripts/build/builders/esp32.py | 80 ++++++++++++++++++++++++--------- 2 files changed, 98 insertions(+), 28 deletions(-) diff --git a/scripts/build/build/factory.py b/scripts/build/build/factory.py index e1cbb984105266..2b7fe1a9ce6030 100644 --- a/scripts/build/build/factory.py +++ b/scripts/build/build/factory.py @@ -5,12 +5,24 @@ from builders.builder import Builder from builders.linux import LinuxBuilder from builders.qpg import QpgBuilder -from builders.esp32 import Esp32Builder, Esp32Board +from builders.esp32 import Esp32Builder, Esp32Board, Esp32App from builders.efr32 import Efr32Builder, Efr32App, Efr32Board from .targets import Application, Board, Platform +class MatchApplication: + + def __init__(self, app, board=None): + self.app = app + self.board = board + + def Match(self, board: Board, app: Application): + if app != self.app: + return False + return self.board is None or board == self.board + + class Matcher(): """Figures out if a proper builder can be created for a platform/board/app combination.""" @@ -20,7 +32,11 @@ def __init__(self, builder_class): self.board_arguments = {} def AcceptApplication(self, __app_key: Application, **kargs): - self.app_arguments[__app_key] = kargs + self.app_arguments[MatchApplication(__app_key)] = kargs + + def AcceptApplicationForBoard(self, __app_key: Application, __board: Board, + **kargs): + self.app_arguments[MatchApplication(__app_key, __board)] = kargs def AcceptBoard(self, __board_key: Board, **kargs): self.board_arguments[__board_key] = kargs @@ -31,11 +47,17 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, if not __board_key in self.board_arguments: return None - if not __app_key in self.app_arguments: + extra_app_args = None + for key, value in self.app_arguments.items(): + if key.Match(__board_key, __app_key): + extra_app_args = value + break + + if extra_app_args is None: return None kargs.update(self.board_arguments[__board_key]) - kargs.update(self.app_arguments[__app_key]) + kargs.update(extra_app_args) return self.builder_class(repo_path, **kargs) @@ -53,9 +75,14 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, _MATCHERS[Platform.LINUX].AcceptApplication(Application.ALL_CLUSTERS) _MATCHERS[Platform.LINUX].AcceptBoard(Board.NATIVE) -_MATCHERS[Platform.ESP32].AcceptApplication(Application.ALL_CLUSTERS) _MATCHERS[Platform.ESP32].AcceptBoard(Board.DEVKITC, board=Esp32Board.DevKitC) _MATCHERS[Platform.ESP32].AcceptBoard(Board.M5STACK, board=Esp32Board.M5Stack) +_MATCHERS[Platform.ESP32].AcceptApplication( + Application.ALL_CLUSTERS, app=Esp32App.ALL_CLUSTERS) +_MATCHERS[Platform.ESP32].AcceptApplicationForBoard( + Application.LOCK, Board.DEVKITC, app=Esp32App.LOCK) + +# TODO: match lock only on devkitc !!! _MATCHERS[Platform.QPG].AcceptApplication(Application.LOCK) _MATCHERS[Platform.QPG].AcceptBoard(Board.QPG6100) @@ -114,7 +141,8 @@ def PlatformsForBoard(board: Board) -> Set[Platform]: def ApplicationsForPlatform(platform: Platform) -> Set[Application]: """What applications are buildable for a specific platform.""" global _MATCHERS - return set(_MATCHERS[platform].app_arguments.keys()) + return set( + [matcher.app for matcher in _MATCHERS[platform].app_arguments.keys()]) @staticmethod def PlatformsForApplication(application: Application) -> Set[Platform]: @@ -122,6 +150,8 @@ def PlatformsForApplication(application: Application) -> Set[Platform]: global _MATCHERS platforms = set() for platform, matcher in _MATCHERS.items(): - if application in matcher.app_arguments: - platforms.add(platform) + for app_matcher in matcher.app_arguments: + if application == app_matcher.app: + platforms.add(platform) + break return platforms diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py index c197503eb1f5bd..57ce1fa6684de3 100644 --- a/scripts/build/builders/esp32.py +++ b/scripts/build/builders/esp32.py @@ -1,19 +1,48 @@ import logging import os -from enum import Enum +from enum import Enum, auto from .builder import Builder class Esp32Board(Enum): - DevKitC = 1 - M5Stack = 2 - - def DefaultsFileName(self): - if self == Esp32Board.DevKitC: - return 'sdkconfig_devkit.defaults' - elif self == Esp32Board.M5Stack: - return 'sdkconfig_m5stack.defaults' + DevKitC = auto() + M5Stack = auto() + + +class Esp32App(Enum): + ALL_CLUSTERS = auto() + LOCK = auto() + + @property + def ExampleName(self): + if self == Esp32App.ALL_CLUSTERS: + return 'all-clusters-app' + elif self == Esp32App.LOCK: + return 'lock-app' + else: + raise Exception('Unknown app type: %r' % self) + + @property + def AppNamePrefix(self): + if self == Esp32App.ALL_CLUSTERS: + return 'chip-all-clusters-app' + elif self == Esp32App.LOCK: + return 'chip-lock-app' + else: + raise Exception('Unknown app type: %r' % self) + + +def DefaultsFileName(board: Esp32Board, app: Esp32App): + if app != Esp32App.ALL_CLUSTERS: + # only all-clusters has a specific defaults name + return None + + if board == Esp32Board.DevKitC: + return 'sdkconfig_devkit.defaults' + elif board == Esp32Board.M5Stack: + return 'sdkconfig_m5stack.defaults' + else: raise Exception('Unknown board type') @@ -22,21 +51,32 @@ class Esp32Builder(Builder): def __init__(self, root, output_dir: str, - board: Esp32Board = Esp32Board.M5Stack): + board: Esp32Board = Esp32Board.M5Stack, + app: Esp32App = Esp32App.ALL_CLUSTERS): super(Esp32Builder, self).__init__(root, output_dir) self.board = board + self.app = app def _IdfEnvExecute(self, cmd, **kargs): self._Execute( ['bash', '-c', 'source $IDF_PATH/export.sh; %s' % cmd], **kargs) def generate(self): - if not os.path.exists(os.path.join(self.output_dir, 'build.ninja')): - # This will do a 'cmake reconfigure' which will create ninja files without rebuilding - self._IdfEnvExecute( - "idf.py -D SDKCONFIG_DEFAULTS='%s' -C examples/all-clusters-app/esp32 -B %s reconfigure" - % (self.board.DefaultsFileName(), self.output_dir), - cwd=self.root) + if os.path.exists(os.path.join(self.output_dir, 'build.ninja')): + return + + defaults = DefaultsFileName(self.board, self.app) + + cmd = 'idf.py' + + if defaults: + cmd += " -D SDKCONFIG_DEFAULTS='%s'" % defaults + + cmd += ' -C examples/%s/esp32 -B %s reconfigure' % (self.app.ExampleName, + self.output_dir) + + # This will do a 'cmake reconfigure' which will create ninja files without rebuilding + self._IdfEnvExecute(cmd, cwd=self.root) def build(self): logging.info('Compiling Esp32 at %s', self.output_dir) @@ -46,8 +86,8 @@ def build(self): def outputs(self): return { - 'chip-all-clusters-app.elf': - os.path.join(self.output_dir, 'chip-all-clusters-app.elf'), - 'chip-all-clusters-app.map': - os.path.join(self.output_dir, 'chip-all-clusters-app.map'), + self.app.AppNamePrefix + '.elf': + os.path.join(self.output_dir, self.app.AppNamePrefix + '.elf'), + self.app.AppNamePrefix + '.map': + os.path.join(self.output_dir, self.app.AppNamePrefix + '.map'), } From 7161ca79719a914ba56fe991ca180f49e386b996 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 9 Jul 2021 15:38:12 -0400 Subject: [PATCH 06/21] Remove obsolete todo --- scripts/build/build/factory.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/build/build/factory.py b/scripts/build/build/factory.py index 2b7fe1a9ce6030..ddf426df2e1e48 100644 --- a/scripts/build/build/factory.py +++ b/scripts/build/build/factory.py @@ -82,8 +82,6 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, _MATCHERS[Platform.ESP32].AcceptApplicationForBoard( Application.LOCK, Board.DEVKITC, app=Esp32App.LOCK) -# TODO: match lock only on devkitc !!! - _MATCHERS[Platform.QPG].AcceptApplication(Application.LOCK) _MATCHERS[Platform.QPG].AcceptBoard(Board.QPG6100) From 249144f955d93012375190bcf762c93f1535fd7a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 9 Jul 2021 16:59:49 -0400 Subject: [PATCH 07/21] Fix the duplicated accept for efr32 lock app --- scripts/build/build/factory.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build/build/factory.py b/scripts/build/build/factory.py index ddf426df2e1e48..24e33484143587 100644 --- a/scripts/build/build/factory.py +++ b/scripts/build/build/factory.py @@ -85,7 +85,6 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, _MATCHERS[Platform.QPG].AcceptApplication(Application.LOCK) _MATCHERS[Platform.QPG].AcceptBoard(Board.QPG6100) -_MATCHERS[Platform.EFR32].AcceptApplication(Application.LOCK) _MATCHERS[Platform.EFR32].AcceptBoard(Board.BRD4161A, board=Efr32Board.BRD4161A) _MATCHERS[Platform.EFR32].AcceptApplication( Application.LIGHT, app=Efr32App.LIGHT) From 3a125961290680eb9cffa11cc766290723b19710 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 9 Jul 2021 16:46:31 -0400 Subject: [PATCH 08/21] Add a dry run option for the build runner, showing what commands would be executed --- scripts/build/build/__init__.py | 16 +++++++--------- scripts/build/build/factory.py | 15 ++++++++++----- scripts/build/build/targets.py | 8 ++++---- scripts/build/builders/builder.py | 11 +++++------ scripts/build/builders/efr32.py | 10 +++++++--- scripts/build/builders/esp32.py | 15 ++++++++++----- scripts/build/builders/linux.py | 10 ++++++---- scripts/build/builders/qpg.py | 10 ++++++---- scripts/build/chipbuild.py | 17 +++++++++++++++-- scripts/build/runner/__init__.py | 2 ++ scripts/build/runner/printonly.py | 18 ++++++++++++++++++ .../build/{shellrunner.py => runner/shell.py} | 11 +++++++---- 12 files changed, 97 insertions(+), 46 deletions(-) create mode 100644 scripts/build/runner/__init__.py create mode 100644 scripts/build/runner/printonly.py rename scripts/build/{shellrunner.py => runner/shell.py} (78%) diff --git a/scripts/build/build/__init__.py b/scripts/build/build/__init__.py index 7e1f94dada8249..1365b7a07fba56 100644 --- a/scripts/build/build/__init__.py +++ b/scripts/build/build/__init__.py @@ -31,10 +31,10 @@ class Context: to generate make/ninja instructions and to compile. """ - def __init__(self, repository_path, output_prefix): + def __init__(self, runner, repository_path, output_prefix): self.builders = [] - self.repository_path = repository_path - self.output_prefix = output_prefix + self.builder_factory = BuilderFactory(runner, repository_path, + output_prefix) self.completed_steps = set() def SetupBuilders(self, platforms: Sequence[Platform], @@ -83,12 +83,10 @@ def SetupBuilders(self, platforms: Sequence[Platform], boards_with_builders = set() applications_with_builders = set() - factory = BuilderFactory(self.repository_path, self.output_prefix) - - for platform in platforms: - for board in boards: - for application in applications: - builder = factory.Create(platform, board, application) + for platform in sorted(platforms): + for board in sorted(boards): + for application in sorted(applications): + builder = self.builder_factory.Create(platform, board, application) if not builder: logging.debug('Builder not supported for tuple %s/%s/%s', platform, board, application) diff --git a/scripts/build/build/factory.py b/scripts/build/build/factory.py index 24e33484143587..5231ff79a3ede9 100644 --- a/scripts/build/build/factory.py +++ b/scripts/build/build/factory.py @@ -41,8 +41,8 @@ def AcceptApplicationForBoard(self, __app_key: Application, __board: Board, def AcceptBoard(self, __board_key: Board, **kargs): self.board_arguments[__board_key] = kargs - def Create(self, __board_key: Board, __app_key: Application, repo_path: str, - **kargs): + def Create(self, runner, __board_key: Board, __app_key: Application, + repo_path: str, **kargs): """Creates a new builder for the given board/app. """ if not __board_key in self.board_arguments: return None @@ -59,7 +59,7 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, kargs.update(self.board_arguments[__board_key]) kargs.update(extra_app_args) - return self.builder_class(repo_path, **kargs) + return self.builder_class(repo_path, runner=runner, **kargs) # Builds a list of acceptable application/board combination for every platform @@ -96,7 +96,8 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, class BuilderFactory: """Creates application builders.""" - def __init__(self, repository_path: str, output_prefix: str): + def __init__(self, runner, repository_path: str, output_prefix: str): + self.runner = runner self.repository_path = repository_path self.output_prefix = output_prefix @@ -108,7 +109,11 @@ def Create(self, platform: Platform, board: Board, app: Application): output_directory = os.path.join(self.output_prefix, identifier) builder = _MATCHERS[platform].Create( - board, app, self.repository_path, output_dir=output_directory) + self.runner, + board, + app, + self.repository_path, + output_dir=output_directory) if builder: builder.identifier = identifier diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py index 9dc9dbddc89122..b2f40cb6f8aeb8 100644 --- a/scripts/build/build/targets.py +++ b/scripts/build/build/targets.py @@ -2,10 +2,10 @@ import os import shutil -from enum import Enum, auto +from enum import IntEnum, auto -class Platform(Enum): +class Platform(IntEnum): """Represents a supported build platform for compilation.""" LINUX = auto() QPG = auto() @@ -24,7 +24,7 @@ def FromArgName(name): raise KeyError() -class Board(Enum): +class Board(IntEnum): """Represents Specific boards within a platform.""" # Host builds NATIVE = auto() @@ -51,7 +51,7 @@ def FromArgName(name): raise KeyError() -class Application(Enum): +class Application(IntEnum): """Example applications that can be built.""" ALL_CLUSTERS = auto() LIGHT = auto() diff --git a/scripts/build/builders/builder.py b/scripts/build/builders/builder.py index f437ae62e71519..c2a89af837c81b 100644 --- a/scripts/build/builders/builder.py +++ b/scripts/build/builders/builder.py @@ -5,8 +5,6 @@ import shutil from abc import ABC, abstractmethod -from shellrunner import ShellRunner - class Builder(ABC): """Generic builder base class for CHIP. @@ -16,10 +14,11 @@ class Builder(ABC): """ - def __init__(self, root, output_dir='out'): + def __init__(self, root, runner, output_dir='out'): self.root = os.path.abspath(root) - self._runner = ShellRunner() + self._runner = runner self.output_dir = output_dir + self.identifier = None @abstractmethod def generate(self): @@ -40,8 +39,8 @@ def outputs(self): """ raise NotImplementedError() - def _Execute(self, cmdarray, **args): - self._runner.Run(cmdarray, **args) + def _Execute(self, cmdarray, cwd=None, title=None): + self._runner.Run(cmdarray, cwd=cwd, title=title) def CopyArtifacts(self, target_dir: str): for target_name, source_name in self.outputs().items(): diff --git a/scripts/build/builders/efr32.py b/scripts/build/builders/efr32.py index 978a15aeed5169..b8d3668876d1ce 100644 --- a/scripts/build/builders/efr32.py +++ b/scripts/build/builders/efr32.py @@ -53,13 +53,15 @@ class Efr32Builder(Builder): def __init__(self, root, + runner, output_dir: str, app: Efr32App = Efr32App.LIGHT, board: Efr32Board = Efr32Board.BRD4161A): - super(Efr32Builder, self).__init__(root, output_dir) + super(Efr32Builder, self).__init__(root, runner, output_dir) self.app = app self.board = board + self.identifier = None def generate(self): if not os.path.exists(self.output_dir): @@ -68,13 +70,15 @@ def generate(self): '--root=%s' % os.path.join(self.root, 'examples', self.app.ExampleName(), 'efr32'), '--args=efr32_board="%s"' % self.board.GnArgName(), self.output_dir - ]) + ], + title='Generate %s' % self.identifier) def build(self): logging.info('Compiling EFR32 at %s', self.output_dir) self.generate() - self._Execute(['ninja', '-C', self.output_dir]) + self._Execute(['ninja', '-C', self.output_dir], + title='Build %s' % self.identifier) def outputs(self): items = { diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py index 57ce1fa6684de3..01df6339f059ff 100644 --- a/scripts/build/builders/esp32.py +++ b/scripts/build/builders/esp32.py @@ -50,16 +50,19 @@ class Esp32Builder(Builder): def __init__(self, root, + runner, output_dir: str, board: Esp32Board = Esp32Board.M5Stack, app: Esp32App = Esp32App.ALL_CLUSTERS): - super(Esp32Builder, self).__init__(root, output_dir) + super(Esp32Builder, self).__init__(root, runner, output_dir) self.board = board self.app = app - def _IdfEnvExecute(self, cmd, **kargs): + def _IdfEnvExecute(self, cmd, cwd=None, title=None): self._Execute( - ['bash', '-c', 'source $IDF_PATH/export.sh; %s' % cmd], **kargs) + ['bash', '-c', 'source $IDF_PATH/export.sh; %s' % cmd], + cwd=cwd, + title=title) def generate(self): if os.path.exists(os.path.join(self.output_dir, 'build.ninja')): @@ -76,13 +79,15 @@ def generate(self): self.output_dir) # This will do a 'cmake reconfigure' which will create ninja files without rebuilding - self._IdfEnvExecute(cmd, cwd=self.root) + self._IdfEnvExecute( + cmd, cwd=self.root, title='Generating ' + self.identifier) def build(self): logging.info('Compiling Esp32 at %s', self.output_dir) self.generate() - self._IdfEnvExecute("ninja -C '%s'" % self.output_dir) + self._IdfEnvExecute( + "ninja -C '%s'" % self.output_dir, title='Building ' + self.identifier) def outputs(self): return { diff --git a/scripts/build/builders/linux.py b/scripts/build/builders/linux.py index 77713924626e9c..d528fdee9eb3ec 100644 --- a/scripts/build/builders/linux.py +++ b/scripts/build/builders/linux.py @@ -6,20 +6,22 @@ class LinuxBuilder(Builder): - def __init__(self, root, output_dir): - super(LinuxBuilder, self).__init__(root, output_dir) + def __init__(self, root, runner, output_dir): + super(LinuxBuilder, self).__init__(root, runner, output_dir) def generate(self): if not os.path.exists(self.output_dir): self._Execute(['gn', 'gen', self.output_dir], cwd=os.path.join(self.root, - 'examples/all-clusters-app/linux/')) + 'examples/all-clusters-app/linux/'), + title='Generating ' + self.identifier) def build(self): logging.info('Compiling Linux at %s', self.output_dir) self.generate() - self._Execute(['ninja', '-C', self.output_dir]) + self._Execute(['ninja', '-C', self.output_dir], + title='Building ' + self.identifier) def outputs(self): return { diff --git a/scripts/build/builders/qpg.py b/scripts/build/builders/qpg.py index dbf7c420380fed..0c91ff6ee3b999 100644 --- a/scripts/build/builders/qpg.py +++ b/scripts/build/builders/qpg.py @@ -8,19 +8,21 @@ class QpgBuilder(Builder): - def __init__(self, root, output_dir): - super(QpgBuilder, self).__init__(root, output_dir) + def __init__(self, root, runner, output_dir): + super(QpgBuilder, self).__init__(root, runner, output_dir) def generate(self): if not os.path.exists(self.output_dir): self._Execute(['gn', 'gen', self.output_dir], - cwd=os.path.join(self.root, 'examples/lock-app/qpg/')) + cwd=os.path.join(self.root, 'examples/lock-app/qpg/'), + title='Generating ' + self.identifier) def build(self): logging.info('Compiling QPG at %s', self.output_dir) self.generate() - self._Execute(['ninja', '-C', self.output_dir]) + self._Execute(['ninja', '-C', self.output_dir], + title='Building ' + self.identifier) def outputs(self): return { diff --git a/scripts/build/chipbuild.py b/scripts/build/chipbuild.py index de350f5fe8f9c5..4e6406fdb00940 100755 --- a/scripts/build/chipbuild.py +++ b/scripts/build/chipbuild.py @@ -9,6 +9,7 @@ sys.path.append(os.path.abspath(os.path.dirname(__file__))) import build +from runner import PrintOnlyRunner, ShellRunner # Supported log levels, mapping string values required for argument # parsing into logging constants @@ -73,8 +74,14 @@ def ValidateRepoPath(context, parameter, value): default=False, is_flag=True, help='Clean output directory before running the command') +@click.option( + '--dry-run', + default=False, + is_flag=True, + help='Only print out shell commands that would be executed') @click.pass_context -def main(context, log_level, platform, board, app, repo, out_prefix, clean): +def main(context, log_level, platform, board, app, repo, out_prefix, clean, + dry_run): # Ensures somewhat pretty logging of what is going on coloredlogs.install( level=__LOG_LEVELS__[log_level], @@ -92,7 +99,13 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean): if 'all' in platform: platform = build.PLATFORMS - context.obj = build.Context(repository_path=repo, output_prefix=out_prefix) + if dry_run: + runner = PrintOnlyRunner() + else: + runner = ShellRunner() + + context.obj = build.Context( + repository_path=repo, output_prefix=out_prefix, runner=runner) context.obj.SetupBuilders( platforms=[build.Platform.FromArgName(name) for name in platform], boards=[build.Board.FromArgName(name) for name in board], diff --git a/scripts/build/runner/__init__.py b/scripts/build/runner/__init__.py new file mode 100644 index 00000000000000..791753a838aa32 --- /dev/null +++ b/scripts/build/runner/__init__.py @@ -0,0 +1,2 @@ +from .shell import ShellRunner +from .printonly import PrintOnlyRunner diff --git a/scripts/build/runner/printonly.py b/scripts/build/runner/printonly.py new file mode 100644 index 00000000000000..2dfab1a46e03dd --- /dev/null +++ b/scripts/build/runner/printonly.py @@ -0,0 +1,18 @@ +import shlex + + +class PrintOnlyRunner: + + def Run(self, cmd, cwd=None, title=None): + if title: + print("# " + title) + + if cwd: + print('cd "%s"' % cwd) + + print(" ".join([shlex.quote(part) for part in cmd])) + + if cwd: + print("cd -") + + print() diff --git a/scripts/build/shellrunner.py b/scripts/build/runner/shell.py similarity index 78% rename from scripts/build/shellrunner.py rename to scripts/build/runner/shell.py index 7d2b4bdee599cd..d5e5e98c15ccc9 100644 --- a/scripts/build/shellrunner.py +++ b/scripts/build/runner/shell.py @@ -36,15 +36,18 @@ def close(self): class ShellRunner: - def Run(self, *args, **kargs): + def Run(self, cmd, cwd=None, title=None): outpipe = LogPipe(logging.INFO) errpipe = LogPipe(logging.WARN) - with subprocess.Popen(*args, **kargs, stdout=outpipe, stderr=errpipe) as s: + if title: + logging.info(title) + + with subprocess.Popen(cmd, cwd=cwd, stdout=outpipe, stderr=errpipe) as s: outpipe.close() errpipe.close() code = s.wait() if code != 0: - raise Exception('Command %r failed: %d' % (args, code)) + raise Exception('Command %r failed: %d' % (cmd, code)) else: - logging.info('Command %r completed', args) + logging.info('Command %r completed' % cmd) From 3ec0c9701baee64cf6b853c9208093000ac22661 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 12 Jul 2021 13:32:49 -0400 Subject: [PATCH 09/21] Add support for a "test" to validate that the build generator executes commands as expected --- BUILD.gn | 5 +- scripts/build/BUILD.gn | 87 ++++++++++++++++++ scripts/build/chipbuild.py | 9 +- .../build/expected_all_platform_commands.txt | 92 +++++++++++++++++++ scripts/build/runner/printonly.py | 12 ++- scripts/build/test.py | 52 +++++++++++ 6 files changed, 249 insertions(+), 8 deletions(-) create mode 100644 scripts/build/BUILD.gn create mode 100644 scripts/build/expected_all_platform_commands.txt create mode 100644 scripts/build/test.py diff --git a/BUILD.gn b/BUILD.gn index efdc7f5991cce5..d2d63ae8023d38 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -138,7 +138,10 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") { group("check") { if (chip_link_tests) { - deps = [ "//src:tests_run" ] + deps = [ + "//scripts/build:tests_run", + "//src:tests_run", + ] } } diff --git a/scripts/build/BUILD.gn b/scripts/build/BUILD.gn new file mode 100644 index 00000000000000..dc3af067a124a8 --- /dev/null +++ b/scripts/build/BUILD.gn @@ -0,0 +1,87 @@ +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build_overrides/build.gni") +import("//build_overrides/chip.gni") + +import("${chip_root}/build/chip/tests.gni") + +if (chip_build_tests) { + action("all_platform_build_example") { + _output_file = "$target_gen_dir/all_platform_build_commands.txt" + + script = "chipbuild.py" + outputs = [ _output_file ] + + # chipbuild script + inputs = [ + "./build/__init__.py", + "./build/factory.py", + "./build/targets.py", + "./builders/__init__.py", + "./builders/builder.py", + "./builders/efr32.py", + "./builders/esp32.py", + "./builders/linux.py", + "./builders/qpg.py", + "./chipbuild.py", + "expected_all_platform_commands.txt", + "./runner/__init__.py", + "./runner/printonly.py", + "./runner/shell.py", + ] + + args = [ + "--repo", + rebase_path("${chip_root}"), + "--platform", + "all", + "--log-level", + "fatal", + "--dry-run", + "--dry-run-output", + rebase_path(_output_file, root_build_dir), + "build", + ] + } + + action_foreach("tests_run") { + _stamp_file = "$target_gen_dir/$target_name.{{source_name_part}}.stamp" + _expected_file = "$target_gen_dir/{{source_name_part}}.expected" + _source_file = "$target_gen_dir/all_platform_build_commands.txt" + + script = "test.py" + outputs = [ + _stamp_file, + _expected_file, + ] + + sources = [ _source_file ] + + deps = [ ":all_platform_build_example" ] + + args = [ + "--repo", + rebase_path("${chip_root}"), + "--expected", + rebase_path("expected_all_platform_commands.txt", root_build_dir), + "--expected-out", + rebase_path(_expected_file, root_build_dir), + "--actual", + "{{source}}", + "--touch", + rebase_path(_stamp_file, root_build_dir), + ] + } +} diff --git a/scripts/build/chipbuild.py b/scripts/build/chipbuild.py index 4e6406fdb00940..f3c1d0583e6578 100755 --- a/scripts/build/chipbuild.py +++ b/scripts/build/chipbuild.py @@ -79,9 +79,14 @@ def ValidateRepoPath(context, parameter, value): default=False, is_flag=True, help='Only print out shell commands that would be executed') +@click.option( + '--dry-run-output', + default="-", + type=click.File("wt"), + help='Where to write the dry run output') @click.pass_context def main(context, log_level, platform, board, app, repo, out_prefix, clean, - dry_run): + dry_run, dry_run_output): # Ensures somewhat pretty logging of what is going on coloredlogs.install( level=__LOG_LEVELS__[log_level], @@ -100,7 +105,7 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean, platform = build.PLATFORMS if dry_run: - runner = PrintOnlyRunner() + runner = PrintOnlyRunner(dry_run_output) else: runner = ShellRunner() diff --git a/scripts/build/expected_all_platform_commands.txt b/scripts/build/expected_all_platform_commands.txt new file mode 100644 index 00000000000000..49cb518074223f --- /dev/null +++ b/scripts/build/expected_all_platform_commands.txt @@ -0,0 +1,92 @@ +# Generating linux-native-all_clusters +cd "{root}/examples/all-clusters-app/linux/" +gn gen {root}/out_test/out/linux-native-all_clusters +cd - + +# Generating qpg-qpg6100-lock +cd "{root}/examples/lock-app/qpg/" +gn gen {root}/out_test/out/qpg-qpg6100-lock +cd - + +# Generating esp32-m5stack-all_clusters +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_m5stack.defaults'"'"' -C examples/all-clusters-app/esp32 -B {root}/out_test/out/esp32-m5stack-all_clusters reconfigure' +cd - + +# Generating esp32-devkitc-all_clusters +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_devkit.defaults'"'"' -C examples/all-clusters-app/esp32 -B {root}/out_test/out/esp32-devkitc-all_clusters reconfigure' +cd - + +# Generating esp32-devkitc-lock +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -C examples/lock-app/esp32 -B {root}/out_test/out/esp32-devkitc-lock reconfigure' +cd - + +# Generate efr32-brd4161a-light +gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-light + +# Generate efr32-brd4161a-lock +gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-lock + +# Generate efr32-brd4161a-window_covering +gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-window_covering + +# Generating linux-native-all_clusters +cd "{root}/examples/all-clusters-app/linux/" +gn gen {root}/out_test/out/linux-native-all_clusters +cd - + +# Building linux-native-all_clusters +ninja -C {root}/out_test/out/linux-native-all_clusters + +# Generating qpg-qpg6100-lock +cd "{root}/examples/lock-app/qpg/" +gn gen {root}/out_test/out/qpg-qpg6100-lock +cd - + +# Building qpg-qpg6100-lock +ninja -C {root}/out_test/out/qpg-qpg6100-lock + +# Generating esp32-m5stack-all_clusters +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_m5stack.defaults'"'"' -C examples/all-clusters-app/esp32 -B {root}/out_test/out/esp32-m5stack-all_clusters reconfigure' +cd - + +# Building esp32-m5stack-all_clusters +bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{root}/out_test/out/esp32-m5stack-all_clusters'"'"'' + +# Generating esp32-devkitc-all_clusters +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_devkit.defaults'"'"' -C examples/all-clusters-app/esp32 -B {root}/out_test/out/esp32-devkitc-all_clusters reconfigure' +cd - + +# Building esp32-devkitc-all_clusters +bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{root}/out_test/out/esp32-devkitc-all_clusters'"'"'' + +# Generating esp32-devkitc-lock +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -C examples/lock-app/esp32 -B {root}/out_test/out/esp32-devkitc-lock reconfigure' +cd - + +# Building esp32-devkitc-lock +bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{root}/out_test/out/esp32-devkitc-lock'"'"'' + +# Generate efr32-brd4161a-light +gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-light + +# Build efr32-brd4161a-light +ninja -C {root}/out_test/out/efr32-brd4161a-light + +# Generate efr32-brd4161a-lock +gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-lock + +# Build efr32-brd4161a-lock +ninja -C {root}/out_test/out/efr32-brd4161a-lock + +# Generate efr32-brd4161a-window_covering +gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-window_covering + +# Build efr32-brd4161a-window_covering +ninja -C {root}/out_test/out/efr32-brd4161a-window_covering + diff --git a/scripts/build/runner/printonly.py b/scripts/build/runner/printonly.py index 2dfab1a46e03dd..7f6f70f8e8b448 100644 --- a/scripts/build/runner/printonly.py +++ b/scripts/build/runner/printonly.py @@ -2,17 +2,19 @@ class PrintOnlyRunner: + def __init__(self, output_file): + self.output_file = output_file def Run(self, cmd, cwd=None, title=None): if title: - print("# " + title) + self.output_file.write("# " + title + "\n") if cwd: - print('cd "%s"' % cwd) + self.output_file.write('cd "%s"\n' % cwd) - print(" ".join([shlex.quote(part) for part in cmd])) + self.output_file.write(" ".join([shlex.quote(part) for part in cmd]) + "\n") if cwd: - print("cd -") + self.output_file.write("cd -\n") - print() + self.output_file.write("\n") diff --git a/scripts/build/test.py b/scripts/build/test.py new file mode 100644 index 00000000000000..b7d139ecbb2ccd --- /dev/null +++ b/scripts/build/test.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +import coloredlogs +import click +import logging +import time +import os +import subprocess + +# Supported log levels, mapping string values required for argument +# parsing into logging constants +__LOG_LEVELS__ = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warn': logging.WARN, + 'fatal': logging.FATAL, +} + + +@click.command() +@click.option( + '--log-level', + default='INFO', + type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False), + help='Determines the verbosity of script output.') +@click.option("--touch", type=click.Path(resolve_path=True, dir_okay=False), help="Timestamp file to touch.") +@click.option("--repo", type=click.Path(resolve_path=True, dir_okay=True), help="Repository path used when generating.") +@click.option("--expected", type=click.File("rt"), help="Expected file content.") +@click.option("--expected-out", type=click.File("wt"), help="Where to write expected content.") +@click.option("--actual", type=click.Path(resolve_path=True, dir_okay=False), help="Actual file generated content.") +def main(log_level, touch, repo, expected, expected_out, actual): + coloredlogs.install( + level=__LOG_LEVELS__[log_level], + fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s') + + for l in expected.readlines(): + expected_out.write(l.replace("{root}", repo)) + expected_out.close() + + + logging.info('Diffing %s and %s' % (expected_out.name, actual)) + subprocess.run(['diff', expected_out.name, actual], check=True) + + logging.info('Touching %s' % touch) + + os.makedirs(os.path.dirname(touch), exist_ok=True) + with open(touch, 'wt') as f: + f.write("Executed at %s" % time.ctime()) + + +if __name__ == "__main__": + main() From 9f4e2adf309de9a388d8becf565bef291d11383d Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 12 Jul 2021 17:49:42 -0400 Subject: [PATCH 10/21] Update the command comparison: output directory of the build script has also to be considered for testing --- scripts/build/BUILD.gn | 4 ++ .../build/expected_all_platform_commands.txt | 48 +++++++++---------- scripts/build/test.py | 5 +- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/scripts/build/BUILD.gn b/scripts/build/BUILD.gn index dc3af067a124a8..2225e0fc8c9b0b 100644 --- a/scripts/build/BUILD.gn +++ b/scripts/build/BUILD.gn @@ -49,6 +49,8 @@ if (chip_build_tests) { "all", "--log-level", "fatal", + "--out-prefix", + "/buildout", "--dry-run", "--dry-run-output", rebase_path(_output_file, root_build_dir), @@ -74,6 +76,8 @@ if (chip_build_tests) { args = [ "--repo", rebase_path("${chip_root}"), + "--output-root", + "/buildout", "--expected", rebase_path("expected_all_platform_commands.txt", root_build_dir), "--expected-out", diff --git a/scripts/build/expected_all_platform_commands.txt b/scripts/build/expected_all_platform_commands.txt index 49cb518074223f..91399c9fcf0a7e 100644 --- a/scripts/build/expected_all_platform_commands.txt +++ b/scripts/build/expected_all_platform_commands.txt @@ -1,92 +1,92 @@ # Generating linux-native-all_clusters cd "{root}/examples/all-clusters-app/linux/" -gn gen {root}/out_test/out/linux-native-all_clusters +gn gen {out}/linux-native-all_clusters cd - # Generating qpg-qpg6100-lock cd "{root}/examples/lock-app/qpg/" -gn gen {root}/out_test/out/qpg-qpg6100-lock +gn gen {out}/qpg-qpg6100-lock cd - # Generating esp32-m5stack-all_clusters cd "{root}" -bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_m5stack.defaults'"'"' -C examples/all-clusters-app/esp32 -B {root}/out_test/out/esp32-m5stack-all_clusters reconfigure' +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_m5stack.defaults'"'"' -C examples/all-clusters-app/esp32 -B {out}/esp32-m5stack-all_clusters reconfigure' cd - # Generating esp32-devkitc-all_clusters cd "{root}" -bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_devkit.defaults'"'"' -C examples/all-clusters-app/esp32 -B {root}/out_test/out/esp32-devkitc-all_clusters reconfigure' +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_devkit.defaults'"'"' -C examples/all-clusters-app/esp32 -B {out}/esp32-devkitc-all_clusters reconfigure' cd - # Generating esp32-devkitc-lock cd "{root}" -bash -c 'source $IDF_PATH/export.sh; idf.py -C examples/lock-app/esp32 -B {root}/out_test/out/esp32-devkitc-lock reconfigure' +bash -c 'source $IDF_PATH/export.sh; idf.py -C examples/lock-app/esp32 -B {out}/esp32-devkitc-lock reconfigure' cd - # Generate efr32-brd4161a-light -gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-light +gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-light # Generate efr32-brd4161a-lock -gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-lock +gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-lock # Generate efr32-brd4161a-window_covering -gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-window_covering +gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-window_covering # Generating linux-native-all_clusters cd "{root}/examples/all-clusters-app/linux/" -gn gen {root}/out_test/out/linux-native-all_clusters +gn gen {out}/linux-native-all_clusters cd - # Building linux-native-all_clusters -ninja -C {root}/out_test/out/linux-native-all_clusters +ninja -C {out}/linux-native-all_clusters # Generating qpg-qpg6100-lock cd "{root}/examples/lock-app/qpg/" -gn gen {root}/out_test/out/qpg-qpg6100-lock +gn gen {out}/qpg-qpg6100-lock cd - # Building qpg-qpg6100-lock -ninja -C {root}/out_test/out/qpg-qpg6100-lock +ninja -C {out}/qpg-qpg6100-lock # Generating esp32-m5stack-all_clusters cd "{root}" -bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_m5stack.defaults'"'"' -C examples/all-clusters-app/esp32 -B {root}/out_test/out/esp32-m5stack-all_clusters reconfigure' +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_m5stack.defaults'"'"' -C examples/all-clusters-app/esp32 -B {out}/esp32-m5stack-all_clusters reconfigure' cd - # Building esp32-m5stack-all_clusters -bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{root}/out_test/out/esp32-m5stack-all_clusters'"'"'' +bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{out}/esp32-m5stack-all_clusters'"'"'' # Generating esp32-devkitc-all_clusters cd "{root}" -bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_devkit.defaults'"'"' -C examples/all-clusters-app/esp32 -B {root}/out_test/out/esp32-devkitc-all_clusters reconfigure' +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_devkit.defaults'"'"' -C examples/all-clusters-app/esp32 -B {out}/esp32-devkitc-all_clusters reconfigure' cd - # Building esp32-devkitc-all_clusters -bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{root}/out_test/out/esp32-devkitc-all_clusters'"'"'' +bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{out}/esp32-devkitc-all_clusters'"'"'' # Generating esp32-devkitc-lock cd "{root}" -bash -c 'source $IDF_PATH/export.sh; idf.py -C examples/lock-app/esp32 -B {root}/out_test/out/esp32-devkitc-lock reconfigure' +bash -c 'source $IDF_PATH/export.sh; idf.py -C examples/lock-app/esp32 -B {out}/esp32-devkitc-lock reconfigure' cd - # Building esp32-devkitc-lock -bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{root}/out_test/out/esp32-devkitc-lock'"'"'' +bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{out}/esp32-devkitc-lock'"'"'' # Generate efr32-brd4161a-light -gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-light +gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-light # Build efr32-brd4161a-light -ninja -C {root}/out_test/out/efr32-brd4161a-light +ninja -C {out}/efr32-brd4161a-light # Generate efr32-brd4161a-lock -gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-lock +gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-lock # Build efr32-brd4161a-lock -ninja -C {root}/out_test/out/efr32-brd4161a-lock +ninja -C {out}/efr32-brd4161a-lock # Generate efr32-brd4161a-window_covering -gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {root}/out_test/out/efr32-brd4161a-window_covering +gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-window_covering # Build efr32-brd4161a-window_covering -ninja -C {root}/out_test/out/efr32-brd4161a-window_covering +ninja -C {out}/efr32-brd4161a-window_covering diff --git a/scripts/build/test.py b/scripts/build/test.py index b7d139ecbb2ccd..d1351d5f5410ca 100644 --- a/scripts/build/test.py +++ b/scripts/build/test.py @@ -25,16 +25,17 @@ help='Determines the verbosity of script output.') @click.option("--touch", type=click.Path(resolve_path=True, dir_okay=False), help="Timestamp file to touch.") @click.option("--repo", type=click.Path(resolve_path=True, dir_okay=True), help="Repository path used when generating.") +@click.option("--output-root", type=click.Path(resolve_path=True, dir_okay=True), help="Build output directory") @click.option("--expected", type=click.File("rt"), help="Expected file content.") @click.option("--expected-out", type=click.File("wt"), help="Where to write expected content.") @click.option("--actual", type=click.Path(resolve_path=True, dir_okay=False), help="Actual file generated content.") -def main(log_level, touch, repo, expected, expected_out, actual): +def main(log_level, touch, repo, output_root, expected, expected_out, actual): coloredlogs.install( level=__LOG_LEVELS__[log_level], fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s') for l in expected.readlines(): - expected_out.write(l.replace("{root}", repo)) + expected_out.write(l.replace("{root}", repo).replace("{out}", output_root)) expected_out.close() From e80f9f6d7c9e4ce420b35b2ce2aeab80cc9cad62 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 12 Jul 2021 18:34:41 -0400 Subject: [PATCH 11/21] Fix some naming and use `get_target_outputs` --- scripts/build/BUILD.gn | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/build/BUILD.gn b/scripts/build/BUILD.gn index 2225e0fc8c9b0b..af7caf56d16cb8 100644 --- a/scripts/build/BUILD.gn +++ b/scripts/build/BUILD.gn @@ -60,16 +60,15 @@ if (chip_build_tests) { action_foreach("tests_run") { _stamp_file = "$target_gen_dir/$target_name.{{source_name_part}}.stamp" - _expected_file = "$target_gen_dir/{{source_name_part}}.expected" - _source_file = "$target_gen_dir/all_platform_build_commands.txt" + _expected_out = "$target_gen_dir/{{source_name_part}}.expected" script = "test.py" outputs = [ _stamp_file, - _expected_file, + _expected_out, ] - sources = [ _source_file ] + sources = get_target_outputs(":all_platform_build_example") deps = [ ":all_platform_build_example" ] @@ -81,7 +80,7 @@ if (chip_build_tests) { "--expected", rebase_path("expected_all_platform_commands.txt", root_build_dir), "--expected-out", - rebase_path(_expected_file, root_build_dir), + rebase_path(_expected_out, root_build_dir), "--actual", "{{source}}", "--touch", From c8c8e48cc04ed81e29a5bc6a4bca6ddfae6168d3 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 13 Jul 2021 09:03:38 -0400 Subject: [PATCH 12/21] Address some code review comments --- scripts/build/README.md | 2 +- scripts/build/build/__init__.py | 6 ++++-- scripts/build/builders/builder.py | 2 +- scripts/build/builders/esp32.py | 5 +++-- scripts/build/chipbuild.py | 2 +- scripts/build/shellrunner.py | 8 ++++---- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/scripts/build/README.md b/scripts/build/README.md index d4e464a6d2e06d..62b6cd7ba3e988 100644 --- a/scripts/build/README.md +++ b/scripts/build/README.md @@ -43,7 +43,7 @@ Usage examples: ./scripts/build/chipbuild.py --app all_clusters_app --board devkitc build ``` -3. Generate all the makefiles (but do not compile) all native apps +3. Generate all the build rules (but do not compile) all native apps ``` ./scripts/build/chipbuild.py --platform native generate diff --git a/scripts/build/build/__init__.py b/scripts/build/build/__init__.py index 7e1f94dada8249..190f1fa79f2ca1 100644 --- a/scripts/build/build/__init__.py +++ b/scripts/build/build/__init__.py @@ -31,7 +31,7 @@ class Context: to generate make/ninja instructions and to compile. """ - def __init__(self, repository_path, output_prefix): + def __init__(self, repository_path:str, output_prefix:str): self.builders = [] self.repository_path = repository_path self.output_prefix = output_prefix @@ -40,7 +40,7 @@ def __init__(self, repository_path, output_prefix): def SetupBuilders(self, platforms: Sequence[Platform], boards: Sequence[Board], applications: Sequence[Application]): - """Configures internal builders for the given platform/board/app combionation. + """Configures internal builders for the given platform/board/app combination. Handles smart default selection, so that users only need to specify part of platform/board/application information and the method tries @@ -53,6 +53,8 @@ def SetupBuilders(self, platforms: Sequence[Platform], ]) else: # when nothing is specified, start with a default host build + # TODO: this is only for linux. Should be moved to 'HOST' as a platform + # to also support building on MacOS platforms = [Platform.LINUX] # at this point, at least one of 'platforms' or 'boards' is non-empty diff --git a/scripts/build/builders/builder.py b/scripts/build/builders/builder.py index f437ae62e71519..4e91e5c63d5823 100644 --- a/scripts/build/builders/builder.py +++ b/scripts/build/builders/builder.py @@ -52,7 +52,7 @@ def CopyArtifacts(self, target_dir: str): target_dir_full_name = os.path.dirname(target_full_name) if not os.path.exists(target_dir_full_name): - logging.info(' Creating subdirectory %s first', target_dir_full_name) + logging.info('Creating subdirectory %s first', target_dir_full_name) os.makedirs(target_dir_full_name) shutil.copyfile(source_name, target_full_name) diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py index 57ce1fa6684de3..a459547cdd9af8 100644 --- a/scripts/build/builders/esp32.py +++ b/scripts/build/builders/esp32.py @@ -1,5 +1,7 @@ import logging import os +import shlex + from enum import Enum, auto from .builder import Builder @@ -72,8 +74,7 @@ def generate(self): if defaults: cmd += " -D SDKCONFIG_DEFAULTS='%s'" % defaults - cmd += ' -C examples/%s/esp32 -B %s reconfigure' % (self.app.ExampleName, - self.output_dir) + cmd += ' -C examples/%s/esp32 -B %s reconfigure' % (self.app.ExampleName, shlex.quote(self.output_dir)) # This will do a 'cmake reconfigure' which will create ninja files without rebuilding self._IdfEnvExecute(cmd, cwd=self.root) diff --git a/scripts/build/chipbuild.py b/scripts/build/chipbuild.py index de350f5fe8f9c5..09a8bc28f8cf51 100755 --- a/scripts/build/chipbuild.py +++ b/scripts/build/chipbuild.py @@ -84,7 +84,7 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean): raise click.UsageError(""" PW_PROJECT_ROOT not in current environment. -Please make sure you `source scripts/bootstra.sh` or `source scripts/activate.sh` +Please make sure you `source scripts/bootstrap.sh` or `source scripts/activate.sh` before running this script. """.strip()) diff --git a/scripts/build/shellrunner.py b/scripts/build/shellrunner.py index 7d2b4bdee599cd..0340cb5ed8c42f 100644 --- a/scripts/build/shellrunner.py +++ b/scripts/build/shellrunner.py @@ -14,13 +14,13 @@ def __init__(self, level): threading.Thread.__init__(self) self.daemon = False self.level = level - self.fdRead, self.fdWrite = os.pipe() - self.pipeReader = os.fdopen(self.fdRead) + self.fd_read, self.fd_write = os.pipe() + self.pipeReader = os.fdopen(self.fd_read) self.start() def fileno(self): """Return the write file descriptor of the pipe""" - return self.fdWrite + return self.fd_write def run(self): """Run the thread, logging everything.""" @@ -31,7 +31,7 @@ def run(self): def close(self): """Close the write end of the pipe.""" - os.close(self.fdWrite) + os.close(self.fd_write) class ShellRunner: From 24b0749da1bed6bea2398e54361249ecee0fd3b6 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 13 Jul 2021 09:04:46 -0400 Subject: [PATCH 13/21] Rename chipbuild to build_examples --- scripts/build/{chipbuild.py => build_examples.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/build/{chipbuild.py => build_examples.py} (100%) diff --git a/scripts/build/chipbuild.py b/scripts/build/build_examples.py similarity index 100% rename from scripts/build/chipbuild.py rename to scripts/build/build_examples.py From f8647152e4fb07ebc73e77793f4c6545eed971e5 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 13 Jul 2021 09:11:07 -0400 Subject: [PATCH 14/21] Fixup naming for the unit tests after build script renaming --- scripts/build/BUILD.gn | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/build/BUILD.gn b/scripts/build/BUILD.gn index af7caf56d16cb8..c5fd9bd32669db 100644 --- a/scripts/build/BUILD.gn +++ b/scripts/build/BUILD.gn @@ -21,10 +21,10 @@ if (chip_build_tests) { action("all_platform_build_example") { _output_file = "$target_gen_dir/all_platform_build_commands.txt" - script = "chipbuild.py" + script = "build_examples.py" outputs = [ _output_file ] - # chipbuild script + # build script source files inputs = [ "./build/__init__.py", "./build/factory.py", @@ -35,7 +35,6 @@ if (chip_build_tests) { "./builders/esp32.py", "./builders/linux.py", "./builders/qpg.py", - "./chipbuild.py", "expected_all_platform_commands.txt", "./runner/__init__.py", "./runner/printonly.py", From d6e154b0c7f0f7a00bd591645fe2319d7598b389 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 13 Jul 2021 17:16:58 -0400 Subject: [PATCH 15/21] Fix names --- scripts/build/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/build/README.md b/scripts/build/README.md index 62b6cd7ba3e988..39a52ec7ad5bf7 100644 --- a/scripts/build/README.md +++ b/scripts/build/README.md @@ -22,10 +22,10 @@ ninja -C out/host ## Unified build script -The script `chipbuild.py` provides a single entry point for generating and +The script `build_examples.py` provides a single entry point for generating and executing the build. -Build environment _MUST_ be properly configured for chipbuild to succeed. For +Build environment _MUST_ be properly configured for build_examples to succeed. For example ESP32 builds requite IDF_PATH to be set. Building in the corresponding build image or the chip vscode image satisfy the build environment requirement. @@ -34,31 +34,31 @@ Usage examples: 1. Compiles the Lock app on all supported platforms ``` - ./scripts/build/chipbuild.py --app lock build + ./scripts/build/build_examples.py --app lock build ``` 2. Compile the all clusters app for a ESP32 DevKitC ``` - ./scripts/build/chipbuild.py --app all_clusters_app --board devkitc build + ./scripts/build/build_examples.py --app all_clusters_app --board devkitc build ``` 3. Generate all the build rules (but do not compile) all native apps ``` - ./scripts/build/chipbuild.py --platform native generate + ./scripts/build/build_examples.py --platform native generate ``` 4. Generate all the makefiles (but do not compile) using a specific output root ``` - ./scripts/build/chipbuild.py --platform native generate --out-prefix ./mydir + ./scripts/build/build_examples.py --platform native generate --out-prefix ./mydir ``` 5. Compile the qpg lock app and copy the output in a 'artifact' folder. Note the argument order (artifact copying is an argument for the build command) ``` - ./scripts/build/chipbuild.py --board qpg6100 --app lock build \ + ./scripts/build/build_examples.py --board qpg6100 --app lock build \ --copy-artifacts-to /tmp/artifacts ``` From 5a66c7a090623aa988d7a22e01506304a5e66601 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 13 Jul 2021 17:41:21 -0400 Subject: [PATCH 16/21] Restyle --- scripts/build/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/build/README.md b/scripts/build/README.md index 39a52ec7ad5bf7..33837fb10ecd44 100644 --- a/scripts/build/README.md +++ b/scripts/build/README.md @@ -25,9 +25,10 @@ ninja -C out/host The script `build_examples.py` provides a single entry point for generating and executing the build. -Build environment _MUST_ be properly configured for build_examples to succeed. For -example ESP32 builds requite IDF_PATH to be set. Building in the corresponding -build image or the chip vscode image satisfy the build environment requirement. +Build environment _MUST_ be properly configured for build_examples to succeed. +For example ESP32 builds requite IDF_PATH to be set. Building in the +corresponding build image or the chip vscode image satisfy the build environment +requirement. Usage examples: From 8b3bb56afaf4d99ce4e17aeae6e7a4c5cad6b5db Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 14 Jul 2021 10:27:38 -0400 Subject: [PATCH 17/21] Use difflib instead of diff binary for checking changes --- scripts/build/test.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/build/test.py b/scripts/build/test.py index d1351d5f5410ca..94de129ecdb94a 100644 --- a/scripts/build/test.py +++ b/scripts/build/test.py @@ -5,6 +5,7 @@ import logging import time import os +import difflib import subprocess # Supported log levels, mapping string values required for argument @@ -16,6 +17,23 @@ 'fatal': logging.FATAL, } +def sameFile(a: str, b: str) -> bool: + with open(a, 'rt') as fa: + a_lines = fa.readlines() + + with open(b, 'rt') as fb: + b_lines = fb.readlines() + + diffs = difflib.unified_diff(a_lines, b_lines, fromfile=a, tofile=b) + if diffs: + logging.error("DIFFERENCE found between %s and %s" % (a, b)) + for l in diffs: + logging.warning(" " + l) + + return False + + return True + @click.command() @click.option( @@ -40,10 +58,10 @@ def main(log_level, touch, repo, output_root, expected, expected_out, actual): logging.info('Diffing %s and %s' % (expected_out.name, actual)) - subprocess.run(['diff', expected_out.name, actual], check=True) + if not sameFile(expected_out.name, actual): + sys.exit(1) logging.info('Touching %s' % touch) - os.makedirs(os.path.dirname(touch), exist_ok=True) with open(touch, 'wt') as f: f.write("Executed at %s" % time.ctime()) From 0c0133d414bcc690d564ab9656c60dcec1b78725 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 14 Jul 2021 10:30:43 -0400 Subject: [PATCH 18/21] Fix diffs (generator vs lines) and logging --- scripts/build/test.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/build/test.py b/scripts/build/test.py index 94de129ecdb94a..5facd192c2156d 100644 --- a/scripts/build/test.py +++ b/scripts/build/test.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import coloredlogs import click +import coloredlogs +import difflib import logging -import time import os -import difflib import subprocess +import sys +import time # Supported log levels, mapping string values required for argument # parsing into logging constants @@ -24,11 +25,12 @@ def sameFile(a: str, b: str) -> bool: with open(b, 'rt') as fb: b_lines = fb.readlines() - diffs = difflib.unified_diff(a_lines, b_lines, fromfile=a, tofile=b) + diffs = [line for line in difflib.unified_diff(a_lines, b_lines, fromfile=a, tofile=b)] + if diffs: logging.error("DIFFERENCE found between %s and %s" % (a, b)) for l in diffs: - logging.warning(" " + l) + logging.warning(" " + l.strip()) return False From 0c105ce95bf708ac057b1b3a3c61f0b443be0c6b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 14 Jul 2021 13:52:14 -0400 Subject: [PATCH 19/21] Start converting python logic from build into pw_python_module --- BUILD.gn | 2 +- scripts/build/BUILD.gn | 94 +++++++++++------------------------------- scripts/build/setup.py | 28 +++++++++++++ scripts/build/test.py | 66 +++++++++++------------------ 4 files changed, 77 insertions(+), 113 deletions(-) create mode 100644 scripts/build/setup.py diff --git a/BUILD.gn b/BUILD.gn index a097d414870224..098e9711d90d78 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -139,7 +139,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") { group("check") { if (chip_link_tests) { deps = [ - "//scripts/build:tests_run", + "//scripts/build:build_examples.tests", "//src:tests_run", ] } diff --git a/scripts/build/BUILD.gn b/scripts/build/BUILD.gn index c5fd9bd32669db..a46e3beb7a7491 100644 --- a/scripts/build/BUILD.gn +++ b/scripts/build/BUILD.gn @@ -14,76 +14,30 @@ import("//build_overrides/build.gni") import("//build_overrides/chip.gni") + +import("//build_overrides/pigweed.gni") +import("$dir_pw_build/python.gni") -import("${chip_root}/build/chip/tests.gni") - -if (chip_build_tests) { - action("all_platform_build_example") { - _output_file = "$target_gen_dir/all_platform_build_commands.txt" - - script = "build_examples.py" - outputs = [ _output_file ] - - # build script source files - inputs = [ - "./build/__init__.py", - "./build/factory.py", - "./build/targets.py", - "./builders/__init__.py", - "./builders/builder.py", - "./builders/efr32.py", - "./builders/esp32.py", - "./builders/linux.py", - "./builders/qpg.py", - "expected_all_platform_commands.txt", - "./runner/__init__.py", - "./runner/printonly.py", - "./runner/shell.py", - ] - - args = [ - "--repo", - rebase_path("${chip_root}"), - "--platform", - "all", - "--log-level", - "fatal", - "--out-prefix", - "/buildout", - "--dry-run", - "--dry-run-output", - rebase_path(_output_file, root_build_dir), - "build", - ] - } - action_foreach("tests_run") { - _stamp_file = "$target_gen_dir/$target_name.{{source_name_part}}.stamp" - _expected_out = "$target_gen_dir/{{source_name_part}}.expected" - - script = "test.py" - outputs = [ - _stamp_file, - _expected_out, - ] - - sources = get_target_outputs(":all_platform_build_example") - - deps = [ ":all_platform_build_example" ] +import("${chip_root}/build/chip/tests.gni") - args = [ - "--repo", - rebase_path("${chip_root}"), - "--output-root", - "/buildout", - "--expected", - rebase_path("expected_all_platform_commands.txt", root_build_dir), - "--expected-out", - rebase_path(_expected_out, root_build_dir), - "--actual", - "{{source}}", - "--touch", - rebase_path(_stamp_file, root_build_dir), - ] - } -} +pw_python_package("build_examples") { + setup = [ "setup.py" ] + sources = [ + "build_examples.py", + "build/__init__.py", + "build/factory.py", + "build/targets.py", + "builders/__init__.py", + "builders/builder.py", + "builders/efr32.py", + "builders/esp32.py", + "builders/linux.py", + "builders/qpg.py", + "runner/__init__.py", + "runner/printonly.py", + "runner/shell.py", + "expected_all_platform_commands.txt", + ] + tests = [ "test.py" ] +} diff --git a/scripts/build/setup.py b/scripts/build/setup.py new file mode 100644 index 00000000000000..b5c82aaf37f4dc --- /dev/null +++ b/scripts/build/setup.py @@ -0,0 +1,28 @@ +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""The build_examples package.""" + +import setuptools # type: ignore + +setuptools.setup( + name='build_examples', + version='0.0.1', + author='Project CHIP Authors', + description='Generate build commands for the CHIP SDK Examples', + packages=setuptools.find_packages(), + package_data={'build_examples': ['py.typed']}, + zip_safe=False, +) diff --git a/scripts/build/test.py b/scripts/build/test.py index 5facd192c2156d..edd93a7f56e54a 100644 --- a/scripts/build/test.py +++ b/scripts/build/test.py @@ -9,58 +9,40 @@ import sys import time -# Supported log levels, mapping string values required for argument -# parsing into logging constants -__LOG_LEVELS__ = { - 'debug': logging.DEBUG, - 'info': logging.INFO, - 'warn': logging.WARN, - 'fatal': logging.FATAL, -} +def buildExpected(root: str, out: str): + with open(os.path.join(os.path.dirname(__file__), 'expected_all_platform_commands.txt'), 'rt') as f: + for l in f.readlines(): + yield l.replace("{root}", root).replace("{out}", out) -def sameFile(a: str, b: str) -> bool: - with open(a, 'rt') as fa: - a_lines = fa.readlines() - with open(b, 'rt') as fb: - b_lines = fb.readlines() +def buildActual(root: str, out: str): + return [] - diffs = [line for line in difflib.unified_diff(a_lines, b_lines, fromfile=a, tofile=b)] - if diffs: - logging.error("DIFFERENCE found between %s and %s" % (a, b)) - for l in diffs: - logging.warning(" " + l.strip()) - - return False +def main(): + coloredlogs.install(level=logging.INFO, fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s') - return True + # @click.command() + # @click.option("--touch", type=click.Path(resolve_path=True, dir_okay=False), help="Timestamp file to touch.") + # @click.option("--repo", type=click.Path(resolve_path=True, dir_okay=True), help="Repository path used when generating.") + # @click.option("--output-root", type=click.Path(resolve_path=True, dir_okay=True), help="Build output directory") + # @click.option("--expected", type=click.File("rt"), help="Expected file content.") + # @click.option("--expected-out", type=click.File("wt"), help="Where to write expected content.") + # @click.option("--actual", type=click.Path(resolve_path=True, dir_okay=False), help="Actual file generated content.") + ROOT = '/BUILD/ROOT' + OUT = '/OUTPUT/DIR' -@click.command() -@click.option( - '--log-level', - default='INFO', - type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False), - help='Determines the verbosity of script output.') -@click.option("--touch", type=click.Path(resolve_path=True, dir_okay=False), help="Timestamp file to touch.") -@click.option("--repo", type=click.Path(resolve_path=True, dir_okay=True), help="Repository path used when generating.") -@click.option("--output-root", type=click.Path(resolve_path=True, dir_okay=True), help="Build output directory") -@click.option("--expected", type=click.File("rt"), help="Expected file content.") -@click.option("--expected-out", type=click.File("wt"), help="Where to write expected content.") -@click.option("--actual", type=click.Path(resolve_path=True, dir_okay=False), help="Actual file generated content.") -def main(log_level, touch, repo, output_root, expected, expected_out, actual): - coloredlogs.install( - level=__LOG_LEVELS__[log_level], - fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s') + expected = [l for l in buildExpected(ROOT, OUT)] + actual = [l for l in buildActual(ROOT, OUT)] - for l in expected.readlines(): - expected_out.write(l.replace("{root}", repo).replace("{out}", output_root)) - expected_out.close() + diffs = [line for line in difflib.unified_diff(expected, actual)] + if diffs: + logging.error("DIFFERENCE between expected and generated output") + for l in diffs: + logging.warning(" " + l.strip()) - logging.info('Diffing %s and %s' % (expected_out.name, actual)) - if not sameFile(expected_out.name, actual): sys.exit(1) logging.info('Touching %s' % touch) From d34834058e74c3ffc55bdfa12975922ab0e75301 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 14 Jul 2021 14:03:44 -0400 Subject: [PATCH 20/21] Tests pass --- scripts/build/build_examples.py | 4 ++ .../build/expected_all_platform_commands.txt | 1 + scripts/build/test.py | 37 +++++++++++-------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/scripts/build/build_examples.py b/scripts/build/build_examples.py index 0b203e68ae6d94..a9ef2eee89b282 100755 --- a/scripts/build/build_examples.py +++ b/scripts/build/build_examples.py @@ -23,6 +23,10 @@ def ValidateRepoPath(context, parameter, value): """Validates that the given path looks like a valid chip repository checkout.""" + if value.startswith('/TEST/'): + # Hackish command to allow for unit testing + return value + for name in ['BUILD.gn', '.gn', os.path.join('scripts', 'bootstrap.sh')]: expected_file = os.path.join(value, name) if not os.path.exists(expected_file): diff --git a/scripts/build/expected_all_platform_commands.txt b/scripts/build/expected_all_platform_commands.txt index 91399c9fcf0a7e..d852e580bd63a4 100644 --- a/scripts/build/expected_all_platform_commands.txt +++ b/scripts/build/expected_all_platform_commands.txt @@ -90,3 +90,4 @@ gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '-- # Build efr32-brd4161a-window_covering ninja -C {out}/efr32-brd4161a-window_covering + diff --git a/scripts/build/test.py b/scripts/build/test.py index edd93a7f56e54a..7bf35090bda9e1 100644 --- a/scripts/build/test.py +++ b/scripts/build/test.py @@ -9,28 +9,38 @@ import sys import time +SCRIPT_ROOT = os.path.dirname(__file__) + def buildExpected(root: str, out: str): - with open(os.path.join(os.path.dirname(__file__), 'expected_all_platform_commands.txt'), 'rt') as f: + with open(os.path.join(SCRIPT_ROOT, 'expected_all_platform_commands.txt'), 'rt') as f: for l in f.readlines(): yield l.replace("{root}", root).replace("{out}", out) def buildActual(root: str, out: str): - return [] + # Fake out that we have a project root + os.environ['PW_PROJECT_ROOT'] = root + + binary = os.path.join(SCRIPT_ROOT, 'build_examples.py') + + retval = subprocess.run([ + binary, + '--platform', 'all', + '--log-level', 'FATAL', + '--dry-run', + '--repo', root, + '--out-prefix', out, + 'build' + ], stdout=subprocess.PIPE, check=True, encoding='UTF-8') + + + return [l + '\n' for l in retval.stdout.split('\n')] def main(): coloredlogs.install(level=logging.INFO, fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s') - # @click.command() - # @click.option("--touch", type=click.Path(resolve_path=True, dir_okay=False), help="Timestamp file to touch.") - # @click.option("--repo", type=click.Path(resolve_path=True, dir_okay=True), help="Repository path used when generating.") - # @click.option("--output-root", type=click.Path(resolve_path=True, dir_okay=True), help="Build output directory") - # @click.option("--expected", type=click.File("rt"), help="Expected file content.") - # @click.option("--expected-out", type=click.File("wt"), help="Where to write expected content.") - # @click.option("--actual", type=click.Path(resolve_path=True, dir_okay=False), help="Actual file generated content.") - - ROOT = '/BUILD/ROOT' + ROOT = '/TEST/BUILD/ROOT' OUT = '/OUTPUT/DIR' expected = [l for l in buildExpected(ROOT, OUT)] @@ -42,13 +52,8 @@ def main(): logging.error("DIFFERENCE between expected and generated output") for l in diffs: logging.warning(" " + l.strip()) - sys.exit(1) - logging.info('Touching %s' % touch) - os.makedirs(os.path.dirname(touch), exist_ok=True) - with open(touch, 'wt') as f: - f.write("Executed at %s" % time.ctime()) if __name__ == "__main__": From dae24ce8a006e75244528e182f1a65cc99bfcae6 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 14 Jul 2021 18:04:09 +0000 Subject: [PATCH 21/21] Restyled by gn --- scripts/build/BUILD.gn | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/build/BUILD.gn b/scripts/build/BUILD.gn index a46e3beb7a7491..450fde954f39da 100644 --- a/scripts/build/BUILD.gn +++ b/scripts/build/BUILD.gn @@ -14,30 +14,29 @@ import("//build_overrides/build.gni") import("//build_overrides/chip.gni") - -import("//build_overrides/pigweed.gni") -import("$dir_pw_build/python.gni") +import("//build_overrides/pigweed.gni") +import("$dir_pw_build/python.gni") import("${chip_root}/build/chip/tests.gni") -pw_python_package("build_examples") { - setup = [ "setup.py" ] - sources = [ - "build_examples.py", +pw_python_package("build_examples") { + setup = [ "setup.py" ] + sources = [ "build/__init__.py", "build/factory.py", "build/targets.py", + "build_examples.py", "builders/__init__.py", "builders/builder.py", "builders/efr32.py", "builders/esp32.py", "builders/linux.py", "builders/qpg.py", + "expected_all_platform_commands.txt", "runner/__init__.py", "runner/printonly.py", "runner/shell.py", - "expected_all_platform_commands.txt", - ] - tests = [ "test.py" ] -} + ] + tests = [ "test.py" ] +}