Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Restyle Unified build script: dry run and unit test #8388

Closed
wants to merge 26 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
380bdf9
Imported a general chip builder script, currently covering a few plat…
andy31415 Jul 8, 2021
38c0512
Move build requirements into global script/requirements.txt so that t…
andy31415 Jul 8, 2021
a7d619f
Update script to assume and require bootstrapping
andy31415 Jul 8, 2021
e0f7e91
Code review comments
andy31415 Jul 9, 2021
ba41920
Support building the lock app for ESP32
andy31415 Jul 9, 2021
7161ca7
Remove obsolete todo
andy31415 Jul 9, 2021
249144f
Fix the duplicated accept for efr32 lock app
andy31415 Jul 9, 2021
3a12596
Add a dry run option for the build runner, showing what commands woul…
andy31415 Jul 9, 2021
15690fd
Merge branch 'master' into unified_build_script
andy31415 Jul 12, 2021
0f5e71b
Merge branch 'unified_build_script' into 02_dry_run_build_script
andy31415 Jul 12, 2021
3ec0c97
Add support for a "test" to validate that the build generator execute…
andy31415 Jul 12, 2021
9f4e2ad
Update the command comparison: output directory of the build script h…
andy31415 Jul 12, 2021
e80f9f6
Fix some naming and use `get_target_outputs`
andy31415 Jul 12, 2021
c8c8e48
Address some code review comments
andy31415 Jul 13, 2021
24b0749
Rename chipbuild to build_examples
andy31415 Jul 13, 2021
04760bf
Merge branch 'unified_build_script' into 02_dry_run_build_script
andy31415 Jul 13, 2021
f864715
Fixup naming for the unit tests after build script renaming
andy31415 Jul 13, 2021
d6e154b
Fix names
andy31415 Jul 13, 2021
5a66c7a
Restyle
andy31415 Jul 13, 2021
7759751
Merge branch 'master' into unified_build_script
andy31415 Jul 14, 2021
c385777
Merge branch 'unified_build_script' into 02_dry_run_build_script
andy31415 Jul 14, 2021
8b3bb56
Use difflib instead of diff binary for checking changes
andy31415 Jul 14, 2021
0c0133d
Fix diffs (generator vs lines) and logging
andy31415 Jul 14, 2021
0c105ce
Start converting python logic from build into pw_python_module
andy31415 Jul 14, 2021
d348340
Tests pass
andy31415 Jul 14, 2021
dae24ce
Restyled by gn
restyled-commits Jul 14, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update script to assume and require bootstrapping
andy31415 committed Jul 8, 2021

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit a7d619fc74c9bb3635b5402b263162db33853b1a
44 changes: 19 additions & 25 deletions scripts/build/build/__init__.py
Original file line number Diff line number Diff line change
@@ -20,8 +20,8 @@ def CommaSeparate(items) -> str:
BOARDS = [x.ArgName for x in Board]
APPLICATIONS = [x.ArgName for x in Application]


class BuildSteps(Enum):
BOOTSTRAPED = auto()
GENERATED = auto()


@@ -37,7 +37,8 @@ def __init__(self, repository_path, output_prefix):
self.output_prefix = output_prefix
self.completed_steps = set()

def SetupBuilders(self, platforms: Sequence[Platform], boards: Sequence[Board],
def SetupBuilders(self, platforms: Sequence[Platform],
boards: Sequence[Board],
applications: Sequence[Application]):
"""Configures internal builders for the given platform/board/app combionation.

@@ -47,19 +48,27 @@ def SetupBuilders(self, platforms: Sequence[Platform], boards: Sequence[Board],
"""
if not platforms and not boards:
if applications:
platforms = set().union(*[TargetRelations.PlatformsForApplication(app) for app in applications])
platforms = set().union(*[
TargetRelations.PlatformsForApplication(app) for app in applications
])
else:
# when nothing is specified, start with a default host build
platforms = [Platform.LINUX]

# at this point, at least one of 'platforms' or 'boards' is non-empty
if not boards:
boards = set().union(*[TargetRelations.BoardsForPlatform(platform) for platform in platforms])
boards = set().union(*[
TargetRelations.BoardsForPlatform(platform) for platform in platforms
])
elif not platforms:
platforms = set().union(*[TargetRelations.PlatformsForBoard(board) for board in boards])
platforms = set().union(
*[TargetRelations.PlatformsForBoard(board) for board in boards])

if not applications:
applications = set().union(*[TargetRelations.ApplicationsForPlatform(platform) for platform in platforms])
applications = set().union(*[
TargetRelations.ApplicationsForPlatform(platform)
for platform in platforms
])

platforms = set(platforms)
boards = set(boards)
@@ -74,7 +83,6 @@ def SetupBuilders(self, platforms: Sequence[Platform], boards: Sequence[Board],
boards_with_builders = set()
applications_with_builders = set()


factory = BuilderFactory(self.repository_path, self.output_prefix)

for platform in platforms:
@@ -107,25 +115,11 @@ def SetupBuilders(self, platforms: Sequence[Platform], boards: Sequence[Board],
# whenever builders change, assume generation is required again
self.completed_steps.discard(BuildSteps.GENERATED)

def Bootstrap(self):
"""Performs a bootstrap IFF a bootstrap has not yet been performed."""
if BuildSteps.BOOTSTRAPED in self.completed_steps:
return

# Bootstrap is generic. assumption is that any builder can
# bootstrap just the same, so run only once
if self.builders:
self.builders[0].Bootstrap()

self.completed_steps.add(BuildSteps.BOOTSTRAPED)

def Generate(self):
"""Performs a build generation IFF code generation has not yet been performed."""
if BuildSteps.GENERATED in self.completed_steps:
return

self.Bootstrap()

for builder in self.builders:
logging.info('Generating %s', builder.output_dir)
builder.generate()
@@ -148,10 +142,10 @@ def CleanOutputDirectories(self):
self.completed_steps.discard(BuildSteps.GENERATED)

def CopyArtifactsTo(self, path: str):
logging.info("Copying build artifacts to %s" % path)
logging.info('Copying build artifacts to %s', path)
if not os.path.exists(path):
os.makedirs(path)
os.makedirs(path)

for builder in self.builders:
# FIXME: builder subdir...
builder.CopyArtifacts(os.path.join(path, builder.identifier))
# FIXME: builder subdir...
builder.CopyArtifacts(os.path.join(path, builder.identifier))
9 changes: 4 additions & 5 deletions scripts/build/build/factory.py
Original file line number Diff line number Diff line change
@@ -25,7 +25,8 @@ def AcceptApplication(self, __app_key: Application, **kargs):
def AcceptBoard(self, __board_key: Board, **kargs):
self.board_arguments[__board_key] = kargs

def Create(self, __board_key: Board, __app_key: Application, repo_path: str, **kargs):
def Create(self, __board_key: Board, __app_key: Application, repo_path: str,
**kargs):
"""Creates a new builder for the given board/app. """
if not __board_key in self.board_arguments:
return None
@@ -60,12 +61,10 @@ def Create(self, __board_key: Board, __app_key: Application, repo_path: str, **k
_MATCHERS[Platform.QPG].AcceptBoard(Board.QPG6100)

_MATCHERS[Platform.EFR32].AcceptApplication(Application.LOCK)
_MATCHERS[Platform.EFR32].AcceptBoard(
Board.BRD4161A, board=Efr32Board.BRD4161A)
_MATCHERS[Platform.EFR32].AcceptBoard(Board.BRD4161A, board=Efr32Board.BRD4161A)
_MATCHERS[Platform.EFR32].AcceptApplication(
Application.LIGHT, app=Efr32App.LIGHT)
_MATCHERS[Platform.EFR32].AcceptApplication(
Application.LOCK, app=Efr32App.LOCK)
_MATCHERS[Platform.EFR32].AcceptApplication(Application.LOCK, app=Efr32App.LOCK)
_MATCHERS[Platform.EFR32].AcceptApplication(
Application.WINDOW_COVERING, app=Efr32App.WINDOW_COVERING)

1 change: 1 addition & 0 deletions scripts/build/build/targets.py
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@

from enum import Enum, auto


class Platform(Enum):
"""Represents a supported build platform for compilation."""
LINUX = auto()
21 changes: 0 additions & 21 deletions scripts/build/builders/builder.py
Original file line number Diff line number Diff line change
@@ -43,27 +43,6 @@ def outputs(self):
def _Execute(self, cmdarray, **args):
self._runner.Run(cmdarray, **args)

def _ActivatedExecute(self, cmdstring: str, **args):
"""Switches to top level to activate then runs the given command in the original run directory."""
self._runner.Run([
'bash', '-c',
'cd "%s"; source ./scripts/activate.sh; cd -; %s' %
(self.root, cmdstring)
], **args)

def Bootstrap(self):
envpath = os.environ.get('PW_ENVIRONMENT_ROOT',
os.path.join(self.root, '.environment'))

if not os.path.exists(os.path.join(envpath, 'activate.sh')):
logging.info('Bootstrapping in %s (%s does not look valid )', self.root,
envpath)
self._Execute(['bash', '-c', 'source ./scripts/bootstrap.sh'],
cwd=self.root)
else:
logging.info('Project already bootstrapped in %s (environment in %s)',
self.root, envpath)

def CopyArtifacts(self, target_dir: str):
for target_name, source_name in self.outputs().items():
target_full_name = os.path.join(target_dir, target_name)
12 changes: 7 additions & 5 deletions scripts/build/builders/efr32.py
Original file line number Diff line number Diff line change
@@ -63,16 +63,18 @@ def __init__(self,

def generate(self):
if not os.path.exists(self.output_dir):
self._ActivatedExecute(
'gn gen --check --fail-on-unused-args --root="%s" --args="efr32_board=\\"%s\\"" %s'
% (os.path.join(self.root, 'examples', self.app.ExampleName(),
'efr32'), self.board.GnArgName(), self.output_dir))
self._Execute([
'gn', 'gen', '--check', '--fail-on-unused-args',
'--root=%s' %
os.path.join(self.root, 'examples', self.app.ExampleName(), 'efr32'),
'--args=efr32_board="%s"' % self.board.GnArgName(), self.output_dir
])

def build(self):
logging.info('Compiling EFR32 at %s', self.output_dir)

self.generate()
self._ActivatedExecute('ninja -C %s' % self.output_dir)
self._Execute(['ninja', '-C', self.output_dir])

def outputs(self):
items = {
3 changes: 2 additions & 1 deletion scripts/build/builders/esp32.py
Original file line number Diff line number Diff line change
@@ -27,7 +27,8 @@ def __init__(self,
self.board = board

def _IdfEnvExecute(self, cmd, **kargs):
self._ActivatedExecute('source "$IDF_PATH/export.sh"; %s' % cmd, **kargs)
self._Execute(
['bash', '-c', 'source $IDF_PATH/export.sh; %s' % cmd], **kargs)

def generate(self):
if not os.path.exists(os.path.join(self.output_dir, 'build.ninja')):
8 changes: 4 additions & 4 deletions scripts/build/builders/linux.py
Original file line number Diff line number Diff line change
@@ -11,15 +11,15 @@ def __init__(self, root, output_dir):

def generate(self):
if not os.path.exists(self.output_dir):
self._ActivatedExecute(
'gn gen %s' % self.output_dir,
cwd=os.path.join(self.root, 'examples/all-clusters-app/linux/'))
self._Execute(['gn', 'gen', self.output_dir],
cwd=os.path.join(self.root,
'examples/all-clusters-app/linux/'))

def build(self):
logging.info('Compiling Linux at %s', self.output_dir)

self.generate()
self._ActivatedExecute('ninja -C %s' % self.output_dir)
self._Execute(['ninja', '-C', self.output_dir])

def outputs(self):
return {
7 changes: 3 additions & 4 deletions scripts/build/builders/qpg.py
Original file line number Diff line number Diff line change
@@ -13,15 +13,14 @@ def __init__(self, root, output_dir):

def generate(self):
if not os.path.exists(self.output_dir):
self._ActivatedExecute(
'gn gen %s' % self.output_dir,
cwd=os.path.join(self.root, 'examples/lock-app/qpg/'))
self._Execute(['gn', 'gen', self.output_dir],
cwd=os.path.join(self.root, 'examples/lock-app/qpg/'))

def build(self):
logging.info('Compiling QPG at %s', self.output_dir)

self.generate()
self._ActivatedExecute('ninja -C %s' % self.output_dir)
self._Execute(['ninja', '-C', self.output_dir])

def outputs(self):
return {
23 changes: 10 additions & 13 deletions scripts/build/chipbuild.py
Original file line number Diff line number Diff line change
@@ -80,6 +80,14 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean):
level=__LOG_LEVELS__[log_level],
fmt='%(asctime)s %(name)s %(levelname)-7s %(message)s')

if not 'PW_PROJECT_ROOT' in os.environ:
raise click.UsageError("""
PW_PROJECT_ROOT not in current environment.

Please make sure you `source scripts/bootstra.sh` or `source scripts/activate.sh`
before running this script.
""".strip())

# Support an 'all platforms' choice
if 'all' in platform:
platform = build.PLATFORMS
@@ -95,24 +103,13 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean):


@main.command(
'bootstrap',
help='Run build environment bootstrapping/download required binaries')
@click.pass_context
def cmd_build(context):
context.obj.Bootstrap()


@main.command(
'gen',
help='Bootstrap and generate ninja/makefiles (but does not run the compilation)'
)
'gen', help='Generate ninja/makefiles (but does not run the compilation)')
@click.pass_context
def cmd_generate(context):
context.obj.Generate()


@main.command(
'build', help='Bootstrap, generate and run ninja/make as needed to compile')
@main.command('build', help='generate and run ninja/make as needed to compile')
@click.option(
'--copy-artifacts-to',
default=None,