diff --git a/BUILD.gn b/BUILD.gn index ee06f5a3240b87..098e9711d90d78 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -138,7 +138,10 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") { group("check") { if (chip_link_tests) { - deps = [ "//src:tests_run" ] + deps = [ + "//scripts/build:build_examples.tests", + "//src:tests_run", + ] } } diff --git a/scripts/build/BUILD.gn b/scripts/build/BUILD.gn new file mode 100644 index 00000000000000..f0244a8290cc66 --- /dev/null +++ b/scripts/build/BUILD.gn @@ -0,0 +1,44 @@ +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build_overrides/build.gni") +import("//build_overrides/chip.gni") + +import("//build_overrides/pigweed.gni") +import("$dir_pw_build/python.gni") + +pw_python_package("build_examples") { + setup = [ "setup.py" ] + inputs = [ + # Dependency for the unit test + "expected_all_platform_commands.txt", + ] + + sources = [ + "build/__init__.py", + "build/factory.py", + "build/targets.py", + "build_examples.py", + "builders/__init__.py", + "builders/builder.py", + "builders/efr32.py", + "builders/esp32.py", + "builders/linux.py", + "builders/qpg.py", + "runner/__init__.py", + "runner/printonly.py", + "runner/shell.py", + ] + tests = [ "test.py" ] +} diff --git a/scripts/build/build/__init__.py b/scripts/build/build/__init__.py index 190f1fa79f2ca1..73594a076ac32c 100644 --- a/scripts/build/build/__init__.py +++ b/scripts/build/build/__init__.py @@ -31,10 +31,10 @@ class Context: to generate make/ninja instructions and to compile. """ - def __init__(self, repository_path:str, output_prefix:str): + def __init__(self, runner, repository_path:str, output_prefix:str): self.builders = [] - self.repository_path = repository_path - self.output_prefix = output_prefix + self.builder_factory = BuilderFactory(runner, repository_path, + output_prefix) self.completed_steps = set() def SetupBuilders(self, platforms: Sequence[Platform], @@ -85,12 +85,10 @@ def SetupBuilders(self, platforms: Sequence[Platform], boards_with_builders = set() applications_with_builders = set() - factory = BuilderFactory(self.repository_path, self.output_prefix) - - for platform in platforms: - for board in boards: - for application in applications: - builder = factory.Create(platform, board, application) + for platform in sorted(platforms): + for board in sorted(boards): + for application in sorted(applications): + builder = self.builder_factory.Create(platform, board, application) if not builder: logging.debug('Builder not supported for tuple %s/%s/%s', platform, board, application) diff --git a/scripts/build/build/factory.py b/scripts/build/build/factory.py index 24e33484143587..5231ff79a3ede9 100644 --- a/scripts/build/build/factory.py +++ b/scripts/build/build/factory.py @@ -41,8 +41,8 @@ def AcceptApplicationForBoard(self, __app_key: Application, __board: Board, def AcceptBoard(self, __board_key: Board, **kargs): self.board_arguments[__board_key] = kargs - def Create(self, __board_key: Board, __app_key: Application, repo_path: str, - **kargs): + def Create(self, runner, __board_key: Board, __app_key: Application, + repo_path: str, **kargs): """Creates a new builder for the given board/app. """ if not __board_key in self.board_arguments: return None @@ -59,7 +59,7 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, kargs.update(self.board_arguments[__board_key]) kargs.update(extra_app_args) - return self.builder_class(repo_path, **kargs) + return self.builder_class(repo_path, runner=runner, **kargs) # Builds a list of acceptable application/board combination for every platform @@ -96,7 +96,8 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, class BuilderFactory: """Creates application builders.""" - def __init__(self, repository_path: str, output_prefix: str): + def __init__(self, runner, repository_path: str, output_prefix: str): + self.runner = runner self.repository_path = repository_path self.output_prefix = output_prefix @@ -108,7 +109,11 @@ def Create(self, platform: Platform, board: Board, app: Application): output_directory = os.path.join(self.output_prefix, identifier) builder = _MATCHERS[platform].Create( - board, app, self.repository_path, output_dir=output_directory) + self.runner, + board, + app, + self.repository_path, + output_dir=output_directory) if builder: builder.identifier = identifier diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py index 9dc9dbddc89122..b2f40cb6f8aeb8 100644 --- a/scripts/build/build/targets.py +++ b/scripts/build/build/targets.py @@ -2,10 +2,10 @@ import os import shutil -from enum import Enum, auto +from enum import IntEnum, auto -class Platform(Enum): +class Platform(IntEnum): """Represents a supported build platform for compilation.""" LINUX = auto() QPG = auto() @@ -24,7 +24,7 @@ def FromArgName(name): raise KeyError() -class Board(Enum): +class Board(IntEnum): """Represents Specific boards within a platform.""" # Host builds NATIVE = auto() @@ -51,7 +51,7 @@ def FromArgName(name): raise KeyError() -class Application(Enum): +class Application(IntEnum): """Example applications that can be built.""" ALL_CLUSTERS = auto() LIGHT = auto() diff --git a/scripts/build/build_examples.py b/scripts/build/build_examples.py index 09a8bc28f8cf51..a9ef2eee89b282 100755 --- a/scripts/build/build_examples.py +++ b/scripts/build/build_examples.py @@ -9,6 +9,7 @@ sys.path.append(os.path.abspath(os.path.dirname(__file__))) import build +from runner import PrintOnlyRunner, ShellRunner # Supported log levels, mapping string values required for argument # parsing into logging constants @@ -22,6 +23,10 @@ def ValidateRepoPath(context, parameter, value): """Validates that the given path looks like a valid chip repository checkout.""" + if value.startswith('/TEST/'): + # Hackish command to allow for unit testing + return value + for name in ['BUILD.gn', '.gn', os.path.join('scripts', 'bootstrap.sh')]: expected_file = os.path.join(value, name) if not os.path.exists(expected_file): @@ -73,8 +78,19 @@ def ValidateRepoPath(context, parameter, value): default=False, is_flag=True, help='Clean output directory before running the command') +@click.option( + '--dry-run', + default=False, + is_flag=True, + help='Only print out shell commands that would be executed') +@click.option( + '--dry-run-output', + default="-", + type=click.File("wt"), + help='Where to write the dry run output') @click.pass_context -def main(context, log_level, platform, board, app, repo, out_prefix, clean): +def main(context, log_level, platform, board, app, repo, out_prefix, clean, + dry_run, dry_run_output): # Ensures somewhat pretty logging of what is going on coloredlogs.install( level=__LOG_LEVELS__[log_level], @@ -92,7 +108,13 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean): if 'all' in platform: platform = build.PLATFORMS - context.obj = build.Context(repository_path=repo, output_prefix=out_prefix) + if dry_run: + runner = PrintOnlyRunner(dry_run_output) + else: + runner = ShellRunner() + + context.obj = build.Context( + repository_path=repo, output_prefix=out_prefix, runner=runner) context.obj.SetupBuilders( platforms=[build.Platform.FromArgName(name) for name in platform], boards=[build.Board.FromArgName(name) for name in board], diff --git a/scripts/build/builders/builder.py b/scripts/build/builders/builder.py index 4e91e5c63d5823..f2942a611f7f9b 100644 --- a/scripts/build/builders/builder.py +++ b/scripts/build/builders/builder.py @@ -5,8 +5,6 @@ import shutil from abc import ABC, abstractmethod -from shellrunner import ShellRunner - class Builder(ABC): """Generic builder base class for CHIP. @@ -16,10 +14,11 @@ class Builder(ABC): """ - def __init__(self, root, output_dir='out'): + def __init__(self, root, runner, output_dir='out'): self.root = os.path.abspath(root) - self._runner = ShellRunner() + self._runner = runner self.output_dir = output_dir + self.identifier = None @abstractmethod def generate(self): @@ -40,8 +39,8 @@ def outputs(self): """ raise NotImplementedError() - def _Execute(self, cmdarray, **args): - self._runner.Run(cmdarray, **args) + def _Execute(self, cmdarray, cwd=None, title=None): + self._runner.Run(cmdarray, cwd=cwd, title=title) def CopyArtifacts(self, target_dir: str): for target_name, source_name in self.outputs().items(): diff --git a/scripts/build/builders/efr32.py b/scripts/build/builders/efr32.py index 978a15aeed5169..b8d3668876d1ce 100644 --- a/scripts/build/builders/efr32.py +++ b/scripts/build/builders/efr32.py @@ -53,13 +53,15 @@ class Efr32Builder(Builder): def __init__(self, root, + runner, output_dir: str, app: Efr32App = Efr32App.LIGHT, board: Efr32Board = Efr32Board.BRD4161A): - super(Efr32Builder, self).__init__(root, output_dir) + super(Efr32Builder, self).__init__(root, runner, output_dir) self.app = app self.board = board + self.identifier = None def generate(self): if not os.path.exists(self.output_dir): @@ -68,13 +70,15 @@ def generate(self): '--root=%s' % os.path.join(self.root, 'examples', self.app.ExampleName(), 'efr32'), '--args=efr32_board="%s"' % self.board.GnArgName(), self.output_dir - ]) + ], + title='Generate %s' % self.identifier) def build(self): logging.info('Compiling EFR32 at %s', self.output_dir) self.generate() - self._Execute(['ninja', '-C', self.output_dir]) + self._Execute(['ninja', '-C', self.output_dir], + title='Build %s' % self.identifier) def outputs(self): items = { diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py index a459547cdd9af8..d56e9abb0830d2 100644 --- a/scripts/build/builders/esp32.py +++ b/scripts/build/builders/esp32.py @@ -52,16 +52,19 @@ class Esp32Builder(Builder): def __init__(self, root, + runner, output_dir: str, board: Esp32Board = Esp32Board.M5Stack, app: Esp32App = Esp32App.ALL_CLUSTERS): - super(Esp32Builder, self).__init__(root, output_dir) + super(Esp32Builder, self).__init__(root, runner, output_dir) self.board = board self.app = app - def _IdfEnvExecute(self, cmd, **kargs): + def _IdfEnvExecute(self, cmd, cwd=None, title=None): self._Execute( - ['bash', '-c', 'source $IDF_PATH/export.sh; %s' % cmd], **kargs) + ['bash', '-c', 'source $IDF_PATH/export.sh; %s' % cmd], + cwd=cwd, + title=title) def generate(self): if os.path.exists(os.path.join(self.output_dir, 'build.ninja')): @@ -77,13 +80,15 @@ def generate(self): cmd += ' -C examples/%s/esp32 -B %s reconfigure' % (self.app.ExampleName, shlex.quote(self.output_dir)) # This will do a 'cmake reconfigure' which will create ninja files without rebuilding - self._IdfEnvExecute(cmd, cwd=self.root) + self._IdfEnvExecute( + cmd, cwd=self.root, title='Generating ' + self.identifier) def build(self): logging.info('Compiling Esp32 at %s', self.output_dir) self.generate() - self._IdfEnvExecute("ninja -C '%s'" % self.output_dir) + self._IdfEnvExecute( + "ninja -C '%s'" % self.output_dir, title='Building ' + self.identifier) def outputs(self): return { diff --git a/scripts/build/builders/linux.py b/scripts/build/builders/linux.py index 77713924626e9c..d528fdee9eb3ec 100644 --- a/scripts/build/builders/linux.py +++ b/scripts/build/builders/linux.py @@ -6,20 +6,22 @@ class LinuxBuilder(Builder): - def __init__(self, root, output_dir): - super(LinuxBuilder, self).__init__(root, output_dir) + def __init__(self, root, runner, output_dir): + super(LinuxBuilder, self).__init__(root, runner, output_dir) def generate(self): if not os.path.exists(self.output_dir): self._Execute(['gn', 'gen', self.output_dir], cwd=os.path.join(self.root, - 'examples/all-clusters-app/linux/')) + 'examples/all-clusters-app/linux/'), + title='Generating ' + self.identifier) def build(self): logging.info('Compiling Linux at %s', self.output_dir) self.generate() - self._Execute(['ninja', '-C', self.output_dir]) + self._Execute(['ninja', '-C', self.output_dir], + title='Building ' + self.identifier) def outputs(self): return { diff --git a/scripts/build/builders/qpg.py b/scripts/build/builders/qpg.py index dbf7c420380fed..0c91ff6ee3b999 100644 --- a/scripts/build/builders/qpg.py +++ b/scripts/build/builders/qpg.py @@ -8,19 +8,21 @@ class QpgBuilder(Builder): - def __init__(self, root, output_dir): - super(QpgBuilder, self).__init__(root, output_dir) + def __init__(self, root, runner, output_dir): + super(QpgBuilder, self).__init__(root, runner, output_dir) def generate(self): if not os.path.exists(self.output_dir): self._Execute(['gn', 'gen', self.output_dir], - cwd=os.path.join(self.root, 'examples/lock-app/qpg/')) + cwd=os.path.join(self.root, 'examples/lock-app/qpg/'), + title='Generating ' + self.identifier) def build(self): logging.info('Compiling QPG at %s', self.output_dir) self.generate() - self._Execute(['ninja', '-C', self.output_dir]) + self._Execute(['ninja', '-C', self.output_dir], + title='Building ' + self.identifier) def outputs(self): return { diff --git a/scripts/build/expected_all_platform_commands.txt b/scripts/build/expected_all_platform_commands.txt new file mode 100644 index 00000000000000..d852e580bd63a4 --- /dev/null +++ b/scripts/build/expected_all_platform_commands.txt @@ -0,0 +1,93 @@ +# Generating linux-native-all_clusters +cd "{root}/examples/all-clusters-app/linux/" +gn gen {out}/linux-native-all_clusters +cd - + +# Generating qpg-qpg6100-lock +cd "{root}/examples/lock-app/qpg/" +gn gen {out}/qpg-qpg6100-lock +cd - + +# Generating esp32-m5stack-all_clusters +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_m5stack.defaults'"'"' -C examples/all-clusters-app/esp32 -B {out}/esp32-m5stack-all_clusters reconfigure' +cd - + +# Generating esp32-devkitc-all_clusters +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_devkit.defaults'"'"' -C examples/all-clusters-app/esp32 -B {out}/esp32-devkitc-all_clusters reconfigure' +cd - + +# Generating esp32-devkitc-lock +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -C examples/lock-app/esp32 -B {out}/esp32-devkitc-lock reconfigure' +cd - + +# Generate efr32-brd4161a-light +gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-light + +# Generate efr32-brd4161a-lock +gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-lock + +# Generate efr32-brd4161a-window_covering +gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-window_covering + +# Generating linux-native-all_clusters +cd "{root}/examples/all-clusters-app/linux/" +gn gen {out}/linux-native-all_clusters +cd - + +# Building linux-native-all_clusters +ninja -C {out}/linux-native-all_clusters + +# Generating qpg-qpg6100-lock +cd "{root}/examples/lock-app/qpg/" +gn gen {out}/qpg-qpg6100-lock +cd - + +# Building qpg-qpg6100-lock +ninja -C {out}/qpg-qpg6100-lock + +# Generating esp32-m5stack-all_clusters +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_m5stack.defaults'"'"' -C examples/all-clusters-app/esp32 -B {out}/esp32-m5stack-all_clusters reconfigure' +cd - + +# Building esp32-m5stack-all_clusters +bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{out}/esp32-m5stack-all_clusters'"'"'' + +# Generating esp32-devkitc-all_clusters +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -D SDKCONFIG_DEFAULTS='"'"'sdkconfig_devkit.defaults'"'"' -C examples/all-clusters-app/esp32 -B {out}/esp32-devkitc-all_clusters reconfigure' +cd - + +# Building esp32-devkitc-all_clusters +bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{out}/esp32-devkitc-all_clusters'"'"'' + +# Generating esp32-devkitc-lock +cd "{root}" +bash -c 'source $IDF_PATH/export.sh; idf.py -C examples/lock-app/esp32 -B {out}/esp32-devkitc-lock reconfigure' +cd - + +# Building esp32-devkitc-lock +bash -c 'source $IDF_PATH/export.sh; ninja -C '"'"'{out}/esp32-devkitc-lock'"'"'' + +# Generate efr32-brd4161a-light +gn gen --check --fail-on-unused-args --root={root}/examples/lighting-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-light + +# Build efr32-brd4161a-light +ninja -C {out}/efr32-brd4161a-light + +# Generate efr32-brd4161a-lock +gn gen --check --fail-on-unused-args --root={root}/examples/lock-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-lock + +# Build efr32-brd4161a-lock +ninja -C {out}/efr32-brd4161a-lock + +# Generate efr32-brd4161a-window_covering +gn gen --check --fail-on-unused-args --root={root}/examples/window-app/efr32 '--args=efr32_board="BRD4161A"' {out}/efr32-brd4161a-window_covering + +# Build efr32-brd4161a-window_covering +ninja -C {out}/efr32-brd4161a-window_covering + + diff --git a/scripts/build/runner/__init__.py b/scripts/build/runner/__init__.py new file mode 100644 index 00000000000000..791753a838aa32 --- /dev/null +++ b/scripts/build/runner/__init__.py @@ -0,0 +1,2 @@ +from .shell import ShellRunner +from .printonly import PrintOnlyRunner diff --git a/scripts/build/runner/printonly.py b/scripts/build/runner/printonly.py new file mode 100644 index 00000000000000..7f6f70f8e8b448 --- /dev/null +++ b/scripts/build/runner/printonly.py @@ -0,0 +1,20 @@ +import shlex + + +class PrintOnlyRunner: + def __init__(self, output_file): + self.output_file = output_file + + def Run(self, cmd, cwd=None, title=None): + if title: + self.output_file.write("# " + title + "\n") + + if cwd: + self.output_file.write('cd "%s"\n' % cwd) + + self.output_file.write(" ".join([shlex.quote(part) for part in cmd]) + "\n") + + if cwd: + self.output_file.write("cd -\n") + + self.output_file.write("\n") diff --git a/scripts/build/shellrunner.py b/scripts/build/runner/shell.py similarity index 78% rename from scripts/build/shellrunner.py rename to scripts/build/runner/shell.py index 0340cb5ed8c42f..d1122060801018 100644 --- a/scripts/build/shellrunner.py +++ b/scripts/build/runner/shell.py @@ -36,15 +36,18 @@ def close(self): class ShellRunner: - def Run(self, *args, **kargs): + def Run(self, cmd, cwd=None, title=None): outpipe = LogPipe(logging.INFO) errpipe = LogPipe(logging.WARN) - with subprocess.Popen(*args, **kargs, stdout=outpipe, stderr=errpipe) as s: + if title: + logging.info(title) + + with subprocess.Popen(cmd, cwd=cwd, stdout=outpipe, stderr=errpipe) as s: outpipe.close() errpipe.close() code = s.wait() if code != 0: - raise Exception('Command %r failed: %d' % (args, code)) + raise Exception('Command %r failed: %d' % (cmd, code)) else: - logging.info('Command %r completed', args) + logging.info('Command %r completed' % cmd) diff --git a/scripts/build/setup.py b/scripts/build/setup.py new file mode 100644 index 00000000000000..b5c82aaf37f4dc --- /dev/null +++ b/scripts/build/setup.py @@ -0,0 +1,28 @@ +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""The build_examples package.""" + +import setuptools # type: ignore + +setuptools.setup( + name='build_examples', + version='0.0.1', + author='Project CHIP Authors', + description='Generate build commands for the CHIP SDK Examples', + packages=setuptools.find_packages(), + package_data={'build_examples': ['py.typed']}, + zip_safe=False, +) diff --git a/scripts/build/test.py b/scripts/build/test.py new file mode 100644 index 00000000000000..ff5c9317715022 --- /dev/null +++ b/scripts/build/test.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import click +import coloredlogs +import difflib +import logging +import os +import subprocess +import sys +import time + +from typing import List + +SCRIPT_ROOT = os.path.dirname(__file__) + +def build_expected_output(root: str, out: str) -> List[str]: + with open(os.path.join(SCRIPT_ROOT, 'expected_all_platform_commands.txt'), 'rt') as f: + for l in f.readlines(): + yield l.replace("{root}", root).replace("{out}", out) + + +def build_actual_output(root: str, out: str) -> List[str]: + # Fake out that we have a project root + os.environ['PW_PROJECT_ROOT'] = root + + binary = os.path.join(SCRIPT_ROOT, 'build_examples.py') + + retval = subprocess.run([ + binary, + '--platform', 'all', + '--log-level', 'FATAL', + '--dry-run', + '--repo', root, + '--out-prefix', out, + 'build' + ], stdout=subprocess.PIPE, check=True, encoding='UTF-8') + + + return [l + '\n' for l in retval.stdout.split('\n')] + + +def main(): + coloredlogs.install(level=logging.INFO, fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s') + + ROOT = '/TEST/BUILD/ROOT' + OUT = '/OUTPUT/DIR' + + expected = [l for l in build_expected_output(ROOT, OUT)] + actual = [l for l in build_actual_output(ROOT, OUT)] + + diffs = [line for line in difflib.unified_diff(expected, actual)] + + if diffs: + logging.error("DIFFERENCE between expected and generated output") + for l in diffs: + logging.warning(" " + l.strip()) + sys.exit(1) + + + +if __name__ == "__main__": + main()