From 09709bc10f4f679da349ee56fc141e81f57165c6 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 11:26:07 -0500 Subject: [PATCH 01/47] Start implementing a pregenerator --- docs/code_generation.md | 26 ++++++++++++++-- scripts/codepregen.py | 66 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 3 deletions(-) create mode 100755 scripts/codepregen.py diff --git a/docs/code_generation.md b/docs/code_generation.md index 51a615f9872a9e..47b66f7f212de8 100644 --- a/docs/code_generation.md +++ b/docs/code_generation.md @@ -176,12 +176,32 @@ enerated ### `*.matter` code generation -Currently `*.matter` code generation is done at compile time. +`*.matter` code generation can be done either at compile time or it can +use pre-generated output. Rules for how `codegen.py` is invoked and how includes/sources are set are defined at: - `src/app/chip_data_model.cmake` -- `build/chip/esp32/esp32_codegen.cmake` (support for 2-pass cmake builds used - by the Espressif `idf.py` build system) - `src/app/chip_data_model.gni` + +Additionally, `build/chip/esp32/esp32_codegen.cmake` adds processing support for +the 2-pass cmake builds used by the Espressif `idf.py` build system. + +## Pre-generation + +Code pre-generation can be used: + +- when compile-time code generation is not desirable. This may be for importing + into build systems that do not have the pre-requisites to run code generation + at build time or to save the code generation time at the expense of running + code generation for every possible zap/generation type +- To check changes in generated code across versions, beyond the comparisons + of golden image tests in `scripts/idl/tests` + +The script to trigger code pre-generation is `scripts/code_pregenerate.py` and +requires the pregeneration output directory as an argument + +```bash +scripts/code_pregenerate.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} +``` \ No newline at end of file diff --git a/scripts/codepregen.py b/scripts/codepregen.py new file mode 100755 index 00000000000000..85d162f36846a8 --- /dev/null +++ b/scripts/codepregen.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +import click +import logging +import enum +import os +import sys + +try: + import coloredlogs + _has_coloredlogs = True +except: + _has_coloredlogs = False + +# Supported log levels, mapping string values required for argument +# parsing into logging constants +__LOG_LEVELS__ = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warn': logging.WARN, + 'fatal': logging.FATAL, +} + +@click.command() +@click.option( + '--log-level', + default='INFO', + type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False), + help='Determines the verbosity of script output') +@click.option( + '--sdk-root', + default=None, + help='Path to the SDK root (where .zap/.matter files exist)') +@click.argument('output_dir') +def main(log_level, sdk_root, output_dir): + if _has_coloredlogs: + coloredlogs.install(level=__LOG_LEVELS__[ + log_level], fmt='%(asctime)s %(levelname)-7s %(message)s') + else: + logging.basicConfig( + level=__LOG_LEVELS__[log_level], + format='%(asctime)s %(levelname)-7s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + if not sdk_root: + sdk_root = os.path.join(os.path.realpath(__file__), '..') + sdk_root = os.path.abspath(sdk_root) + + if not output_dir: + raise Exception("Missing output directory") + + output_dir = os.path.abspath(output_dir) + + logging.info(f"Pre-generating {sdk_root} data into {output_dir}") + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + # FIXME: implement + + logging.info("Done") + + +if __name__ == '__main__': + main() From 3f441ab4e9fff877dbb729e84cb5a1478f1e19ec Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 11:30:12 -0500 Subject: [PATCH 02/47] Start moving pregenerate logic into a separate directory --- scripts/codepregen.py | 24 +++++++++++++++++++++++- scripts/pregenerate/__init__.py | 24 ++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 scripts/pregenerate/__init__.py diff --git a/scripts/codepregen.py b/scripts/codepregen.py index 85d162f36846a8..b0294c0e67c252 100755 --- a/scripts/codepregen.py +++ b/scripts/codepregen.py @@ -1,11 +1,32 @@ #!/usr/bin/env python +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import click import logging import enum import os import sys +try: + from pregenerate import FindPregenerationTargets +except: + import os + sys.path.append(os.path.abspath(os.path.dirname(__file__))) + from pregenerate import FindPregenerationTargets + try: import coloredlogs _has_coloredlogs = True @@ -57,7 +78,8 @@ def main(log_level, sdk_root, output_dir): if not os.path.exists(output_dir): os.makedirs(output_dir) - # FIXME: implement + for target in FindPregenerationTargets(sdk_root): + target.Generate() logging.info("Done") diff --git a/scripts/pregenerate/__init__.py b/scripts/pregenerate/__init__.py new file mode 100644 index 00000000000000..c8d15b11680dc5 --- /dev/null +++ b/scripts/pregenerate/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def FindPregenerationTargets(sdk_root): + """Finds all relevand pre-generation targets in the given + SDK root. + + Pre-generation targets are generally zap and matter files. + """ + + # TODO: implement + return [] From f1dc8a895895d90aa34efcd8829d393aeb21d315 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 11:53:58 -0500 Subject: [PATCH 03/47] Start adding some ability to figure out pregeneration output locations --- docs/code_generation.md | 20 +++++------ scripts/codepregen.py | 4 ++- scripts/pregenerate/__init__.py | 63 ++++++++++++++++++++++++++++++++- 3 files changed, 75 insertions(+), 12 deletions(-) diff --git a/docs/code_generation.md b/docs/code_generation.md index 47b66f7f212de8..d48006d95c7a98 100644 --- a/docs/code_generation.md +++ b/docs/code_generation.md @@ -176,8 +176,8 @@ enerated ### `*.matter` code generation -`*.matter` code generation can be done either at compile time or it can -use pre-generated output. +`*.matter` code generation can be done either at compile time or it can use +pre-generated output. Rules for how `codegen.py` is invoked and how includes/sources are set are defined at: @@ -185,23 +185,23 @@ defined at: - `src/app/chip_data_model.cmake` - `src/app/chip_data_model.gni` -Additionally, `build/chip/esp32/esp32_codegen.cmake` adds processing support for +Additionally, `build/chip/esp32/esp32_codegen.cmake` adds processing support for the 2-pass cmake builds used by the Espressif `idf.py` build system. ## Pre-generation Code pre-generation can be used: -- when compile-time code generation is not desirable. This may be for importing - into build systems that do not have the pre-requisites to run code generation - at build time or to save the code generation time at the expense of running - code generation for every possible zap/generation type -- To check changes in generated code across versions, beyond the comparisons - of golden image tests in `scripts/idl/tests` +- when compile-time code generation is not desirable. This may be for + importing into build systems that do not have the pre-requisites to run code + generation at build time or to save the code generation time at the expense + of running code generation for every possible zap/generation type +- To check changes in generated code across versions, beyond the comparisons + of golden image tests in `scripts/idl/tests` The script to trigger code pre-generation is `scripts/code_pregenerate.py` and requires the pregeneration output directory as an argument ```bash scripts/code_pregenerate.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} -``` \ No newline at end of file +``` diff --git a/scripts/codepregen.py b/scripts/codepregen.py index b0294c0e67c252..25e68894ab6a08 100755 --- a/scripts/codepregen.py +++ b/scripts/codepregen.py @@ -42,6 +42,7 @@ 'fatal': logging.FATAL, } + @click.command() @click.option( '--log-level', @@ -65,7 +66,8 @@ def main(log_level, sdk_root, output_dir): ) if not sdk_root: - sdk_root = os.path.join(os.path.realpath(__file__), '..') + sdk_root = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') + sdk_root = os.path.abspath(sdk_root) if not output_dir: diff --git a/scripts/pregenerate/__init__.py b/scripts/pregenerate/__init__.py index c8d15b11680dc5..43f474b0d31197 100644 --- a/scripts/pregenerate/__init__.py +++ b/scripts/pregenerate/__init__.py @@ -13,12 +13,73 @@ # limitations under the License. -def FindPregenerationTargets(sdk_root): +import logging +import os + +from dataclasses import dataclass +from enum import Enum, auto +from typing import Iterator + + +class IdlFileType(Enum): + ZAP = auto() + MATTER = auto() + + +@dataclass +class InputIdlFile: + file_type: IdlFileType + relative_path: str + + + @property + def pregen_subdir(self): + ''' + Returns the relative path inside the pregenerate directory where + data for this IDL file should be pregenerated. + ''' + top_dir = os.path.splitext(self.relative_path)[0] + + if self.file_type == IdlFileType.MATTER: + return os.path.join(top_dir, "codegen") + elif self.file_type == IdlFileType.ZAP: + return os.path.join(top_dir, "zap") + else: + raise Exception("Unknown file type for self") + + +def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]: + relevant_subdirs = [ + 'examples', # all example apps + 'src', # realistically only controller/data_model + ] + + while sdk_root.endswith('/'): + sdk_root = sdk_root[:-1] + sdk_root_length = len(sdk_root) + + for subdir_name in relevant_subdirs: + top_directory_name = os.path.join(sdk_root, subdir_name) + logging.debug(f"Searching {top_directory_name}") + for root, dirs, files in os.walk(top_directory_name): + for file in files: + if file.endswith('.zap'): + yield InputIdlFile(file_type=IdlFileType.ZAP, + relative_path=os.path.join(root[sdk_root_length+1:], file)) + if file.endswith('.matter'): + yield InputIdlFile(file_type=IdlFileType.MATTER, + relative_path=os.path.join(root[sdk_root_length+1:], file)) + + +def FindPregenerationTargets(sdk_root: str): """Finds all relevand pre-generation targets in the given SDK root. Pre-generation targets are generally zap and matter files. """ + for idl in FindAllIdls(sdk_root): + logging.debug(f"{idl.relative_path} => {idl.pregen_subdir}") + # TODO: implement return [] From f87b0849b2315c4ecc8b79f902db038b3e95caad Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 12:19:21 -0500 Subject: [PATCH 04/47] Pregeneration for bridge seems to work --- scripts/codepregen.py | 2 +- scripts/pregenerate/__init__.py | 46 +++++++++++---------------------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/scripts/codepregen.py b/scripts/codepregen.py index 25e68894ab6a08..7dc5a67fb00c0f 100755 --- a/scripts/codepregen.py +++ b/scripts/codepregen.py @@ -81,7 +81,7 @@ def main(log_level, sdk_root, output_dir): os.makedirs(output_dir) for target in FindPregenerationTargets(sdk_root): - target.Generate() + target.Generate(output_dir) logging.info("Done") diff --git a/scripts/pregenerate/__init__.py b/scripts/pregenerate/__init__.py index 43f474b0d31197..0b12a74da04443 100644 --- a/scripts/pregenerate/__init__.py +++ b/scripts/pregenerate/__init__.py @@ -16,36 +16,14 @@ import logging import os -from dataclasses import dataclass -from enum import Enum, auto from typing import Iterator +from .types import InputIdlFile, IdlFileType -class IdlFileType(Enum): - ZAP = auto() - MATTER = auto() - -@dataclass -class InputIdlFile: - file_type: IdlFileType - relative_path: str - - - @property - def pregen_subdir(self): - ''' - Returns the relative path inside the pregenerate directory where - data for this IDL file should be pregenerated. - ''' - top_dir = os.path.splitext(self.relative_path)[0] - - if self.file_type == IdlFileType.MATTER: - return os.path.join(top_dir, "codegen") - elif self.file_type == IdlFileType.ZAP: - return os.path.join(top_dir, "zap") - else: - raise Exception("Unknown file type for self") +from .pregenerators import CodegenJavaPregenerator +from .pregenerators import CodegenBridgePregenerator +from .pregenerators import CodegenCppAppPregenerator def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]: @@ -75,11 +53,17 @@ def FindPregenerationTargets(sdk_root: str): """Finds all relevand pre-generation targets in the given SDK root. - Pre-generation targets are generally zap and matter files. + Pre-generation targets are based on zap and matter files with options + on what rules to pregenerate and how. """ - for idl in FindAllIdls(sdk_root): - logging.debug(f"{idl.relative_path} => {idl.pregen_subdir}") + generators = [ + CodegenBridgePregenerator, + CodegenJavaPregenerator, + CodegenCppAppPregenerator, + ] - # TODO: implement - return [] + for idl in FindAllIdls(sdk_root): + for generator in generators: + if generator.Accept(idl): + yield generator.CreateTarget(sdk_root, idl) From 6967c58122484922c8e4a1e692d26b3ef27247f7 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 12:19:47 -0500 Subject: [PATCH 05/47] Add missing files --- scripts/pregenerate/pregenerators.py | 82 ++++++++++++++++++++++++++++ scripts/pregenerate/types.py | 45 +++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 scripts/pregenerate/pregenerators.py create mode 100644 scripts/pregenerate/types.py diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/pregenerators.py new file mode 100644 index 00000000000000..50a0782af1b7fb --- /dev/null +++ b/scripts/pregenerate/pregenerators.py @@ -0,0 +1,82 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import shlex +import subprocess + +from .types import InputIdlFile, IdlFileType + +CODEGEN_PY_PATH = os.path.join(os.path.dirname(__file__), '..', 'codegen.py') + + +class CodegenTarget: + def __init__(self, idl: InputIdlFile, generator: str, sdk_root: str): + self.idl = idl + self.generator = generator + self.sdk_root = sdk_root + + if idl.file_type != IdlFileType.MATTER: + raise Exception(f"Can only code generate for `*.matter` input files, not for {idl}") + + def Generate(self, output_root: str): + '''Runs codegen.py to generate in the specified directory''' + + output_dir = os.path.join(output_root, self.idl.pregen_subdir) + + logging.info(f"Generating: {self.generator}:{self.idl.relative_path} into {output_dir}") + + cmd = [ + CODEGEN_PY_PATH, + '--generator', self.generator, + '--output-dir', output_dir, + os.path.join(self.sdk_root, self.idl.relative_path) + ] + + logging.debug(f"Executing {cmd}") + subprocess.check_call(cmd) + + +class CodegenBridgePregenerator: + """Pregeneration logic for "bridge" codegen.py outputs""" + + @staticmethod + def Accept(idl: InputIdlFile): + # Bridgen is highly specific, a single path is acceptable for dynamic + # bridge codegen + if idl.file_type != IdlFileType.MATTER: + return False + + return idl.relative_path == "examples/dynamic-bridge-app/bridge-common/bridge-app.matter" + + @staticmethod + def CreateTarget(sdk_root: str, idl: InputIdlFile): + return CodegenTarget(sdk_root=sdk_root, idl=idl, generator="bridge") + + +class CodegenJavaPregenerator: + """Pregeneration logic for "java" codegen.py outputs""" + + @staticmethod + def Accept(idl: InputIdlFile): + return False + + +class CodegenCppAppPregenerator: + """Pregeneration logic for "cpp-app" codegen.py outputs""" + + @staticmethod + def Accept(idl: InputIdlFile): + return False diff --git a/scripts/pregenerate/types.py b/scripts/pregenerate/types.py new file mode 100644 index 00000000000000..1582a40d7dfeac --- /dev/null +++ b/scripts/pregenerate/types.py @@ -0,0 +1,45 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from dataclasses import dataclass +from enum import Enum, auto + + +class IdlFileType(Enum): + ZAP = auto() + MATTER = auto() + + +@dataclass +class InputIdlFile: + file_type: IdlFileType + relative_path: str + + @property + def pregen_subdir(self): + ''' + Returns the relative path inside the pregenerate directory where + data for this IDL file should be pregenerated. + ''' + top_dir = os.path.splitext(self.relative_path)[0] + + if self.file_type == IdlFileType.MATTER: + return os.path.join(top_dir, "codegen") + elif self.file_type == IdlFileType.ZAP: + return os.path.join(top_dir, "zap") + else: + raise Exception("Unknown file type for self") + From ee1d3ba8362705a1347765a4283e78930dd7e77b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 12:19:59 -0500 Subject: [PATCH 06/47] Restyle --- scripts/pregenerate/types.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/pregenerate/types.py b/scripts/pregenerate/types.py index 1582a40d7dfeac..3d8c0bca3034d8 100644 --- a/scripts/pregenerate/types.py +++ b/scripts/pregenerate/types.py @@ -42,4 +42,3 @@ def pregen_subdir(self): return os.path.join(top_dir, "zap") else: raise Exception("Unknown file type for self") - From 1764fa6ffdd14c31190d57bbb24280bd978a463a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 12:21:46 -0500 Subject: [PATCH 07/47] Better log level logic --- scripts/pregenerate/pregenerators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/pregenerators.py index 50a0782af1b7fb..46946ceffb156d 100644 --- a/scripts/pregenerate/pregenerators.py +++ b/scripts/pregenerate/pregenerators.py @@ -40,6 +40,7 @@ def Generate(self, output_root: str): cmd = [ CODEGEN_PY_PATH, + '--log-level', 'fatal', '--generator', self.generator, '--output-dir', output_dir, os.path.join(self.sdk_root, self.idl.relative_path) From 7a092785a78a2d997cd20606821e5afde0f0442d Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 12:24:29 -0500 Subject: [PATCH 08/47] Pregeneration of java also exists --- scripts/pregenerate/pregenerators.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/pregenerators.py index 46946ceffb156d..3979668df3e58c 100644 --- a/scripts/pregenerate/pregenerators.py +++ b/scripts/pregenerate/pregenerators.py @@ -55,11 +55,8 @@ class CodegenBridgePregenerator: @staticmethod def Accept(idl: InputIdlFile): - # Bridgen is highly specific, a single path is acceptable for dynamic + # Bridge is highly specific, a single path is acceptable for dynamic # bridge codegen - if idl.file_type != IdlFileType.MATTER: - return False - return idl.relative_path == "examples/dynamic-bridge-app/bridge-common/bridge-app.matter" @staticmethod @@ -72,7 +69,13 @@ class CodegenJavaPregenerator: @staticmethod def Accept(idl: InputIdlFile): - return False + # Java is highly specific, a single path is acceptable for dynamic + # bridge codegen + return idl.relative_path == "src/controller/data_model/controller-clusters.matter" + + @staticmethod + def CreateTarget(sdk_root: str, idl: InputIdlFile): + return CodegenTarget(sdk_root=sdk_root, idl=idl, generator="java") class CodegenCppAppPregenerator: From c2c1e60216b2f17db0c6d116dc3c4c059d0002b6 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 13:07:04 -0500 Subject: [PATCH 09/47] Allow pregeneration of cpp app data. Full codegen usage is now enabled --- scripts/pregenerate/pregenerators.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/pregenerators.py index 3979668df3e58c..5347c98641a858 100644 --- a/scripts/pregenerate/pregenerators.py +++ b/scripts/pregenerate/pregenerators.py @@ -83,4 +83,15 @@ class CodegenCppAppPregenerator: @staticmethod def Accept(idl: InputIdlFile): - return False + if idl.file_type != IdlFileType.MATTER: + return False + + # we should not be checked for these, but verify just in case + if '/tests/' in idl.relative_path: + return False + + return True + + @staticmethod + def CreateTarget(sdk_root: str, idl: InputIdlFile): + return CodegenTarget(sdk_root=sdk_root, idl=idl, generator="cpp-app") From 9817d82cfc3344939dc760e5b5a9ae31785a9078 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 13:09:06 -0500 Subject: [PATCH 10/47] Move sdk root around --- scripts/pregenerate/__init__.py | 8 ++++---- scripts/pregenerate/pregenerators.py | 28 ++++++++++++++++------------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/scripts/pregenerate/__init__.py b/scripts/pregenerate/__init__.py index 0b12a74da04443..eab3205dc9d981 100644 --- a/scripts/pregenerate/__init__.py +++ b/scripts/pregenerate/__init__.py @@ -58,12 +58,12 @@ def FindPregenerationTargets(sdk_root: str): """ generators = [ - CodegenBridgePregenerator, - CodegenJavaPregenerator, - CodegenCppAppPregenerator, + CodegenBridgePregenerator(sdk_root), + CodegenJavaPregenerator(sdk_root), + CodegenCppAppPregenerator(sdk_root), ] for idl in FindAllIdls(sdk_root): for generator in generators: if generator.Accept(idl): - yield generator.CreateTarget(sdk_root, idl) + yield generator.CreateTarget(idl) diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/pregenerators.py index 5347c98641a858..bc7e2abec8ae8d 100644 --- a/scripts/pregenerate/pregenerators.py +++ b/scripts/pregenerate/pregenerators.py @@ -23,6 +23,7 @@ class CodegenTarget: + """A target that uses `scripts/codegen.py` to generate files.""" def __init__(self, idl: InputIdlFile, generator: str, sdk_root: str): self.idl = idl self.generator = generator @@ -53,35 +54,39 @@ def Generate(self, output_root: str): class CodegenBridgePregenerator: """Pregeneration logic for "bridge" codegen.py outputs""" - @staticmethod + def __init__(self, sdk_root): + self.sdk_root = sdk_root + def Accept(idl: InputIdlFile): # Bridge is highly specific, a single path is acceptable for dynamic # bridge codegen return idl.relative_path == "examples/dynamic-bridge-app/bridge-common/bridge-app.matter" - @staticmethod - def CreateTarget(sdk_root: str, idl: InputIdlFile): - return CodegenTarget(sdk_root=sdk_root, idl=idl, generator="bridge") + def CreateTarget(idl: InputIdlFile): + return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="bridge") class CodegenJavaPregenerator: """Pregeneration logic for "java" codegen.py outputs""" - @staticmethod + def __init__(self, sdk_root): + self.sdk_root = sdk_root + def Accept(idl: InputIdlFile): # Java is highly specific, a single path is acceptable for dynamic # bridge codegen return idl.relative_path == "src/controller/data_model/controller-clusters.matter" - @staticmethod - def CreateTarget(sdk_root: str, idl: InputIdlFile): - return CodegenTarget(sdk_root=sdk_root, idl=idl, generator="java") + def CreateTarget(idl: InputIdlFile): + return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="java") class CodegenCppAppPregenerator: """Pregeneration logic for "cpp-app" codegen.py outputs""" - @staticmethod + def __init__(self, sdk_root): + self.sdk_root = sdk_root + def Accept(idl: InputIdlFile): if idl.file_type != IdlFileType.MATTER: return False @@ -92,6 +97,5 @@ def Accept(idl: InputIdlFile): return True - @staticmethod - def CreateTarget(sdk_root: str, idl: InputIdlFile): - return CodegenTarget(sdk_root=sdk_root, idl=idl, generator="cpp-app") + def CreateTarget(idl: InputIdlFile): + return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="cpp-app") From b9e6b225f80cb62a66c1d94d3650dd3d9218257e Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 13:10:28 -0500 Subject: [PATCH 11/47] Make things run --- scripts/pregenerate/pregenerators.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/pregenerators.py index bc7e2abec8ae8d..e00c8ba8569358 100644 --- a/scripts/pregenerate/pregenerators.py +++ b/scripts/pregenerate/pregenerators.py @@ -57,12 +57,12 @@ class CodegenBridgePregenerator: def __init__(self, sdk_root): self.sdk_root = sdk_root - def Accept(idl: InputIdlFile): + def Accept(self, idl: InputIdlFile): # Bridge is highly specific, a single path is acceptable for dynamic # bridge codegen return idl.relative_path == "examples/dynamic-bridge-app/bridge-common/bridge-app.matter" - def CreateTarget(idl: InputIdlFile): + def CreateTarget(self, idl: InputIdlFile): return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="bridge") @@ -72,12 +72,12 @@ class CodegenJavaPregenerator: def __init__(self, sdk_root): self.sdk_root = sdk_root - def Accept(idl: InputIdlFile): + def Accept(self, idl: InputIdlFile): # Java is highly specific, a single path is acceptable for dynamic # bridge codegen return idl.relative_path == "src/controller/data_model/controller-clusters.matter" - def CreateTarget(idl: InputIdlFile): + def CreateTarget(self, idl: InputIdlFile): return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="java") @@ -87,7 +87,7 @@ class CodegenCppAppPregenerator: def __init__(self, sdk_root): self.sdk_root = sdk_root - def Accept(idl: InputIdlFile): + def Accept(self, idl: InputIdlFile): if idl.file_type != IdlFileType.MATTER: return False @@ -97,5 +97,5 @@ def Accept(idl: InputIdlFile): return True - def CreateTarget(idl: InputIdlFile): + def CreateTarget(self, idl: InputIdlFile): return CodegenTarget(sdk_root=self.sdk_root, idl=idl, generator="cpp-app") From fdd3dad8fc2ab4284b6428cb5c41adfcf2deac2f Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 13:10:42 -0500 Subject: [PATCH 12/47] Restyle --- scripts/pregenerate/pregenerators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/pregenerators.py index e00c8ba8569358..582ab9b80b6524 100644 --- a/scripts/pregenerate/pregenerators.py +++ b/scripts/pregenerate/pregenerators.py @@ -24,6 +24,7 @@ class CodegenTarget: """A target that uses `scripts/codegen.py` to generate files.""" + def __init__(self, idl: InputIdlFile, generator: str, sdk_root: str): self.idl = idl self.generator = generator From 3bb901088c3e1ba2f9f6aaf389e7ee382f54d04d Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 13:25:47 -0500 Subject: [PATCH 13/47] Parallel code pregen - can do pregen in 1.08 seconds on my machine --- scripts/codepregen.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/scripts/codepregen.py b/scripts/codepregen.py index 7dc5a67fb00c0f..5c05509ef0d851 100755 --- a/scripts/codepregen.py +++ b/scripts/codepregen.py @@ -16,10 +16,13 @@ import click import logging +import multiprocessing +import itertools import enum import os import sys + try: from pregenerate import FindPregenerationTargets except: @@ -42,6 +45,13 @@ 'fatal': logging.FATAL, } +def _ParallelGenerateOne(arg): + """ + Helper method to be passed to multiprocessing parallel generation of + items. + """ + arg[0].Generate(arg[1]) + @click.command() @click.option( @@ -49,12 +59,16 @@ default='INFO', type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False), help='Determines the verbosity of script output') +@click.option( + '--parallel/--no-parallel', + default=True, + help='Do parallel/multiprocessing codege.') @click.option( '--sdk-root', default=None, - help='Path to the SDK root (where .zap/.matter files exist)') + help='Path to the SDK root (where .zap/.matter files exist).') @click.argument('output_dir') -def main(log_level, sdk_root, output_dir): +def main(log_level, parallel, sdk_root, output_dir): if _has_coloredlogs: coloredlogs.install(level=__LOG_LEVELS__[ log_level], fmt='%(asctime)s %(levelname)-7s %(message)s') @@ -80,8 +94,18 @@ def main(log_level, sdk_root, output_dir): if not os.path.exists(output_dir): os.makedirs(output_dir) - for target in FindPregenerationTargets(sdk_root): - target.Generate(output_dir) + if parallel: + target_and_dir = zip( + FindPregenerationTargets(sdk_root), + itertools.repeat(output_dir)) + + p = multiprocessing.Pool() + p.map(_ParallelGenerateOne, target_and_dir) + p.close() + p.join() + else: + for target in FindPregenerationTargets(sdk_root): + target.Generate(output_dir) logging.info("Done") From 3fe49ce465f1a28c0636f4ac60ab79c6bfe3ffb3 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 13:26:04 -0500 Subject: [PATCH 14/47] Restyle --- scripts/codepregen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/codepregen.py b/scripts/codepregen.py index 5c05509ef0d851..325803c64ceefa 100755 --- a/scripts/codepregen.py +++ b/scripts/codepregen.py @@ -45,6 +45,7 @@ 'fatal': logging.FATAL, } + def _ParallelGenerateOne(arg): """ Helper method to be passed to multiprocessing parallel generation of From a7b64d3b0484bf850432d04eeeb655d1d4586233 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 13:51:51 -0500 Subject: [PATCH 15/47] Make sure pregen folders are split --- scripts/pregenerate/pregenerators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pregenerate/pregenerators.py b/scripts/pregenerate/pregenerators.py index 582ab9b80b6524..3d6658dd3bb7b1 100644 --- a/scripts/pregenerate/pregenerators.py +++ b/scripts/pregenerate/pregenerators.py @@ -36,7 +36,7 @@ def __init__(self, idl: InputIdlFile, generator: str, sdk_root: str): def Generate(self, output_root: str): '''Runs codegen.py to generate in the specified directory''' - output_dir = os.path.join(output_root, self.idl.pregen_subdir) + output_dir = os.path.join(output_root, self.idl.pregen_subdir, self.generator) logging.info(f"Generating: {self.generator}:{self.idl.relative_path} into {output_dir}") From f33f7c2f88a0d0559fdc39fa0f5b6bef71bbddd7 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 14:58:06 -0500 Subject: [PATCH 16/47] Pregeneration compile when using GN works --- build/chip/chip_codegen.gni | 124 +++++++++++++++++++++++++++--------- build/chip/copy_tree.py | 19 ++++++ 2 files changed, 113 insertions(+), 30 deletions(-) create mode 100644 build/chip/copy_tree.py diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index bab79934bd37f5..136f00b6939c28 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -18,6 +18,56 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_build/python.gni") +declare_args() { + # Location where code has been pre-generated + chip_code_pre_gegenerated_directory = "" +} + +# Code generation that will happen at build time. +# +# +template("_chip_build_time_codegen") { + _name = target_name + _generator = invoker.generator + + config("${_name}_config") { + include_dirs = [ target_gen_dir ] + } + + pw_python_action(_name) { + script = "${chip_root}/scripts/codegen.py" + + _idl_file = invoker.input + _expected_outputs = "${target_gen_dir}/${_name}.expected.outputs" + + write_file(_expected_outputs, invoker.outputs, "list lines") + + args = [ + "--generator", + _generator, + "--output-dir", + rebase_path(target_gen_dir, root_build_dir), + "--expected-outputs", + rebase_path(_expected_outputs, root_build_dir), + rebase_path(_idl_file, root_build_dir), + ] + + deps = [ "${chip_root}/scripts/idl" ] + public_configs = [ ":${_name}_config" ] + + inputs = [ + _idl_file, + _expected_outputs, + ] + sources = [ _idl_file ] + + outputs = [] + foreach(name, invoker.outputs) { + outputs += [ "${target_gen_dir}/${name}" ] + } + } +} + # Defines a target that runs code generation based on # scripts/codegen.py # @@ -32,6 +82,16 @@ import("$dir_pw_build/python.gni") # Explicit names of the expected outputs. Enforced to validate that # expected outputs are generated when processing input files. # +# Command line parameters: +# +# chip_code_pre_gegenerated_directory: +# - If this is set, generation will NOT happen at compile time but rather +# the code generation is assumed to have already happened and reside in +# the given location. +# - The TOP LEVEL directory is assumed to be given. Actual location for +# individual generators is expected to be of the form +# // +# # NOTE: content of "outputs" is verified to match the output of codegen.py # exactly. It is not inferred on purpose, to make build-rules explicit # and verifiable (even though codege.py can at runtime report its outputs) @@ -53,43 +113,47 @@ import("$dir_pw_build/python.gni") # } # template("chip_codegen") { - _name = target_name - _generator = invoker.generator + if (chip_code_pre_gegenerated_directory == "") { + _chip_build_time_codegen(target_name) { + input = invoker.input + generator = invoker.generator + outputs = invoker.outputs + } + } else { + _name = target_name - config("${_name}_config") { - include_dirs = [ target_gen_dir ] - } + # This contstructs a path like: + # FROM all-clusters-app.matter (inside examples/all-clusters-app/all-clusters-common/) + # USING "cpp-app" for generator: + # => ${pregen_dir}/examples/all-clusters-app/all-clusters-common/all-clusters-app/codegen/cpp-app + _generation_dir = + chip_code_pre_gegenerated_directory + "/" + + string_replace(rebase_path(invoker.input, chip_root), ".matter", "") + + "/codegen/" + invoker.generator - pw_python_action(_name) { - script = "${chip_root}/scripts/codegen.py" - _idl_file = invoker.input - _expected_outputs = "${target_gen_dir}/${_name}.expected.outputs" + config("${_name}_config") { + include_dirs = [ target_gen_dir] + } - write_file(_expected_outputs, invoker.outputs, "list lines") + action("${_name}") { + script = "${chip_root}/build/chip/copy_tree.py" - args = [ - "--generator", - _generator, - "--output-dir", - rebase_path(target_gen_dir, root_build_dir), - "--expected-outputs", - rebase_path(_expected_outputs, root_build_dir), - rebase_path(_idl_file, root_build_dir), - ] + args = [ + "--src-dir", + rebase_path(_generation_dir), + "--dest-dir", + rebase_path(target_gen_dir), + ] - deps = [ "${chip_root}/scripts/idl" ] - public_configs = [ ":${_name}_config" ] + public_configs = [ ":${_name}_config" ] - inputs = [ - _idl_file, - _expected_outputs, - ] - sources = [ _idl_file ] - - outputs = [] - foreach(name, invoker.outputs) { - outputs += [ "${target_gen_dir}/${name}" ] + inputs = [] + outputs = [] + foreach(name, invoker.outputs) { + inputs += [ "${_generation_dir}/${name}" ] + outputs += [ "${target_gen_dir}/${name}" ] + } } } } diff --git a/build/chip/copy_tree.py b/build/chip/copy_tree.py new file mode 100644 index 00000000000000..5960698be8e528 --- /dev/null +++ b/build/chip/copy_tree.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +import argparse +import shutil +import sys + +def main(): + + parser = argparse.ArgumentParser(description="Copy a tree of files from a source directory to a destination one.") + + parser.add_argument('--src-dir', required=True) + parser.add_argument('--dest-dir', required=True) + + args = parser.parse_args() + + shutil.copytree(args.src_dir, args.dest_dir, dirs_exist_ok=True) + +if __name__ == '__main__': + sys.exit(main()) From 63b4614a5cc5f9fd4d1566acf0161f7e6dfe5d8b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 14:58:24 -0500 Subject: [PATCH 17/47] Restyle --- build/chip/chip_codegen.gni | 3 +-- build/chip/copy_tree.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index 136f00b6939c28..1df8effd7bb55e 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -131,9 +131,8 @@ template("chip_codegen") { string_replace(rebase_path(invoker.input, chip_root), ".matter", "") + "/codegen/" + invoker.generator - config("${_name}_config") { - include_dirs = [ target_gen_dir] + include_dirs = [ target_gen_dir ] } action("${_name}") { diff --git a/build/chip/copy_tree.py b/build/chip/copy_tree.py index 5960698be8e528..f2cfa56bb083d6 100644 --- a/build/chip/copy_tree.py +++ b/build/chip/copy_tree.py @@ -4,6 +4,7 @@ import shutil import sys + def main(): parser = argparse.ArgumentParser(description="Copy a tree of files from a source directory to a destination one.") @@ -12,8 +13,9 @@ def main(): parser.add_argument('--dest-dir', required=True) args = parser.parse_args() - + shutil.copytree(args.src_dir, args.dest_dir, dirs_exist_ok=True) + if __name__ == '__main__': sys.exit(main()) From 8bd98b678be9b237cc7a355a4e7d706e91b99c75 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 15:11:48 -0500 Subject: [PATCH 18/47] Direct pregen usage works now --- build/chip/chip_codegen.gni | 49 +++++++++++++++++++++++-------------- src/app/chip_data_model.gni | 23 +++++++++++------ 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index 1df8effd7bb55e..927b9b22fd12ad 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -34,7 +34,7 @@ template("_chip_build_time_codegen") { include_dirs = [ target_gen_dir ] } - pw_python_action(_name) { + pw_python_action("${_name}_codegen") { script = "${chip_root}/scripts/codegen.py" _idl_file = invoker.input @@ -53,7 +53,6 @@ template("_chip_build_time_codegen") { ] deps = [ "${chip_root}/scripts/idl" ] - public_configs = [ ":${_name}_config" ] inputs = [ _idl_file, @@ -66,6 +65,23 @@ template("_chip_build_time_codegen") { outputs += [ "${target_gen_dir}/${name}" ] } } + + source_set(_name) { + sources = [] + foreach(name, invoker.outputs) { + sources += [ "${target_gen_dir}/${name}" ] + } + + public_configs = [ ":${_name}_config" ] + + if (defined(invoker.public_configs)) { + public_configs += invoker.public_configs + } + + if (defined(invoker.deps)) { + deps = invoker.deps + } + } } # Defines a target that runs code generation based on @@ -115,9 +131,9 @@ template("_chip_build_time_codegen") { template("chip_codegen") { if (chip_code_pre_gegenerated_directory == "") { _chip_build_time_codegen(target_name) { - input = invoker.input - generator = invoker.generator - outputs = invoker.outputs + forward_variables_from(invoker, [ + "input", "generator", "output", "public_configs", "deps" + ]) } } else { _name = target_name @@ -135,23 +151,20 @@ template("chip_codegen") { include_dirs = [ target_gen_dir ] } - action("${_name}") { - script = "${chip_root}/build/chip/copy_tree.py" + source_set(_name) { + public_configs = [ ":${_name}_config" ] - args = [ - "--src-dir", - rebase_path(_generation_dir), - "--dest-dir", - rebase_path(target_gen_dir), - ] + if (defined(invoker.public_configs)) { + public_configs += invoker.public_configs + } - public_configs = [ ":${_name}_config" ] + if (defined(invoker.deps)) { + deps = invoker.deps + } - inputs = [] - outputs = [] + sources = [] foreach(name, invoker.outputs) { - inputs += [ "${_generation_dir}/${name}" ] - outputs += [ "${target_gen_dir}/${name}" ] + sources += [ "${_generation_dir}/${name}" ] } } } diff --git a/src/app/chip_data_model.gni b/src/app/chip_data_model.gni index 68aabd7463dda8..6dd3648583d553 100644 --- a/src/app/chip_data_model.gni +++ b/src/app/chip_data_model.gni @@ -51,21 +51,30 @@ template("chip_data_model") { _idl = string_replace(invoker.zap_file, ".zap", ".matter") } + config("${_data_model_name}_config") { + include_dirs = [] + + if (defined(invoker.zap_pregenerated_dir)) { + include_dirs += [ "${invoker.zap_pregenerated_dir}/.." ] + } + } + chip_codegen("${_data_model_name}_codegen") { input = _idl generator = "cpp-app" + outputs = [ "app/PluginApplicationCallbacks.h", "app/callback-stub.cpp", ] - } - config("${_data_model_name}_config") { - include_dirs = [] + public_configs = [ + ":${_data_model_name}_config", + ] - if (defined(invoker.zap_pregenerated_dir)) { - include_dirs += [ "${invoker.zap_pregenerated_dir}/.." ] - } + deps = [ + "${chip_root}/src/app/common:cluster-objects", + ] } _use_default_im_dispatch = !defined(invoker.use_default_im_dispatch) || @@ -84,7 +93,7 @@ template("chip_data_model") { sources = [] } - sources += get_target_outputs(":${_data_model_name}_codegen") + deps = [ ":${_data_model_name}_codegen" ] sources += [ "${_app_root}/clusters/barrier-control-server/barrier-control-server.h", From df15a152d417987a4ab6636784ae1cf9278dd53b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 15:12:05 -0500 Subject: [PATCH 19/47] Restyle --- build/chip/chip_codegen.gni | 11 ++++++++--- src/app/chip_data_model.gni | 8 ++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index 927b9b22fd12ad..97e6385218ba33 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -131,9 +131,14 @@ template("_chip_build_time_codegen") { template("chip_codegen") { if (chip_code_pre_gegenerated_directory == "") { _chip_build_time_codegen(target_name) { - forward_variables_from(invoker, [ - "input", "generator", "output", "public_configs", "deps" - ]) + forward_variables_from(invoker, + [ + "input", + "generator", + "output", + "public_configs", + "deps", + ]) } } else { _name = target_name diff --git a/src/app/chip_data_model.gni b/src/app/chip_data_model.gni index 6dd3648583d553..76b8e9c3379116 100644 --- a/src/app/chip_data_model.gni +++ b/src/app/chip_data_model.gni @@ -68,13 +68,9 @@ template("chip_data_model") { "app/callback-stub.cpp", ] - public_configs = [ - ":${_data_model_name}_config", - ] + public_configs = [ ":${_data_model_name}_config" ] - deps = [ - "${chip_root}/src/app/common:cluster-objects", - ] + deps = [ "${chip_root}/src/app/common:cluster-objects" ] } _use_default_im_dispatch = !defined(invoker.use_default_im_dispatch) || From 29e204c212e296236745489cceca76b1058f39a9 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 15:12:24 -0500 Subject: [PATCH 20/47] Minor sort --- build/chip/chip_codegen.gni | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index 97e6385218ba33..24ed533facaf26 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -133,11 +133,11 @@ template("chip_codegen") { _chip_build_time_codegen(target_name) { forward_variables_from(invoker, [ - "input", + "deps", "generator", + "input", "output", "public_configs", - "deps", ]) } } else { From 5fe3df6f7824c1accb8da5545e7fb588c72c5f47 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 15:21:36 -0500 Subject: [PATCH 21/47] Add support for both pregen and no pregen --- build/chip/chip_codegen.gni | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index 24ed533facaf26..5f201d1e06cc06 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -78,8 +78,12 @@ template("_chip_build_time_codegen") { public_configs += invoker.public_configs } - if (defined(invoker.deps)) { - deps = invoker.deps + forward_variables_from(invoker, ["deps"]) + + if (defined(deps)) { + deps += [ ":${_name}_codegen" ] + } else { + deps = [ ":${_name}_codegen" ] } } } @@ -136,7 +140,7 @@ template("chip_codegen") { "deps", "generator", "input", - "output", + "outputs", "public_configs", ]) } @@ -163,9 +167,7 @@ template("chip_codegen") { public_configs += invoker.public_configs } - if (defined(invoker.deps)) { - deps = invoker.deps - } + forward_variables_from(invoker, ["deps"]) sources = [] foreach(name, invoker.outputs) { From fdc6ae26bc15063fdf74240db94d8aa2c5094559 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 15:22:01 -0500 Subject: [PATCH 22/47] Restyle --- build/chip/chip_codegen.gni | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index 5f201d1e06cc06..1f627912296976 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -78,7 +78,7 @@ template("_chip_build_time_codegen") { public_configs += invoker.public_configs } - forward_variables_from(invoker, ["deps"]) + forward_variables_from(invoker, [ "deps" ]) if (defined(deps)) { deps += [ ":${_name}_codegen" ] @@ -167,7 +167,7 @@ template("chip_codegen") { public_configs += invoker.public_configs } - forward_variables_from(invoker, ["deps"]) + forward_variables_from(invoker, [ "deps" ]) sources = [] foreach(name, invoker.outputs) { From 79ccec04bb4e7eefbb3080224c99ec2921a9019b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 15:34:03 -0500 Subject: [PATCH 23/47] Fix pregen dir output logic --- build/chip/chip_codegen.gni | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index 1f627912296976..188199adba7e47 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -102,6 +102,9 @@ template("_chip_build_time_codegen") { # Explicit names of the expected outputs. Enforced to validate that # expected outputs are generated when processing input files. # +# deps, public_configs +# Forwarded to the resulting source set +# # Command line parameters: # # chip_code_pre_gegenerated_directory: @@ -119,6 +122,9 @@ template("_chip_build_time_codegen") { # To find the list of generated files, you can run codegen.py with the # "--name-only" argument # +# NOTE: +# the result of the target_name WILL BE a `source_set`. Treat it as such. +# # Example usage: # # chip_codegen("java-jni-generate") { @@ -157,7 +163,7 @@ template("chip_codegen") { "/codegen/" + invoker.generator config("${_name}_config") { - include_dirs = [ target_gen_dir ] + include_dirs = [ "${_generation_dir}" ] } source_set(_name) { From d7ad0d983eb3271c5d796acd7ea4de23c9bfc9e0 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 16:03:15 -0500 Subject: [PATCH 24/47] Support pregenerated directory in build_examples.py --- scripts/build/build_examples.py | 8 +++++++- scripts/build/builders/builder.py | 4 ++++ scripts/build/builders/gn.py | 5 +++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/build/build_examples.py b/scripts/build/build_examples.py index 5792bfaec64274..780241f089a35f 100755 --- a/scripts/build/build_examples.py +++ b/scripts/build/build_examples.py @@ -87,6 +87,11 @@ def ValidateRepoPath(context, parameter, value): default='./out', type=click.Path(file_okay=False, resolve_path=True), help='Prefix for the generated file output.') +@click.option( + '--pregen-dir', + default=None, + type=click.Path(file_okay=False, resolve_path=True), + help='Directory where generated files have been pre-generated.') @click.option( '--clean', default=False, @@ -114,7 +119,7 @@ def ValidateRepoPath(context, parameter, value): 'for using ccache when building examples.')) @click.pass_context def main(context, log_level, target, repo, - out_prefix, clean, dry_run, dry_run_output, enable_flashbundle, + out_prefix, pregen_dir, clean, dry_run, dry_run_output, enable_flashbundle, no_log_timestamps, pw_command_launcher): # Ensures somewhat pretty logging of what is going on log_fmt = '%(asctime)s %(levelname)-7s %(message)s' @@ -143,6 +148,7 @@ def main(context, log_level, target, repo, context.obj.SetupBuilders(targets=requested_targets, options=BuilderOptions( enable_flashbundle=enable_flashbundle, pw_command_launcher=pw_command_launcher, + pregen_dir=pregen_dir, )) if clean: diff --git a/scripts/build/builders/builder.py b/scripts/build/builders/builder.py index 99bc9255ee5ef9..dc420fce5e6b41 100644 --- a/scripts/build/builders/builder.py +++ b/scripts/build/builders/builder.py @@ -24,9 +24,13 @@ class BuilderOptions: # Enable flashbundle generation stage enable_flashbundle: bool = False + # Allow to wrap default build command pw_command_launcher: str = None + # Locations where files are pre-generated + pregen_dir: str = None + class Builder(ABC): """Generic builder base class for CHIP. diff --git a/scripts/build/builders/gn.py b/scripts/build/builders/gn.py index bacd87d2399e02..c2a00df2952bc5 100644 --- a/scripts/build/builders/gn.py +++ b/scripts/build/builders/gn.py @@ -61,8 +61,13 @@ def generate(self): ] extra_args = [] + if self.options.pw_command_launcher: extra_args.append('pw_command_launcher="%s"' % self.options.pw_command_launcher) + + if self.pregen_subdir: + extra_args.append('chip_code_pre_gegenerated_directory="%s"' % self.options.pregen_subdir) + extra_args.extend(self.GnBuildArgs() or []) if extra_args: cmd += ['--args=%s' % ' '.join(extra_args)] From e37b00f423a6830a5cdd8cf227982b07e04184e2 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 16:05:44 -0500 Subject: [PATCH 25/47] Fix gn build logic --- scripts/build/builders/gn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build/builders/gn.py b/scripts/build/builders/gn.py index c2a00df2952bc5..396001ff582460 100644 --- a/scripts/build/builders/gn.py +++ b/scripts/build/builders/gn.py @@ -65,8 +65,8 @@ def generate(self): if self.options.pw_command_launcher: extra_args.append('pw_command_launcher="%s"' % self.options.pw_command_launcher) - if self.pregen_subdir: - extra_args.append('chip_code_pre_gegenerated_directory="%s"' % self.options.pregen_subdir) + if self.options.pregen_dir: + extra_args.append('chip_code_pre_gegenerated_directory="%s"' % self.options.pregen_dir) extra_args.extend(self.GnBuildArgs() or []) if extra_args: From 2fadc56bb3f2f9f8b2b2eecd1d1ad812caaa24e0 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 16:08:49 -0500 Subject: [PATCH 26/47] Somewhat simpler code for parallel vs serial codegen --- scripts/codepregen.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/codepregen.py b/scripts/codepregen.py index 325803c64ceefa..511034af244665 100755 --- a/scripts/codepregen.py +++ b/scripts/codepregen.py @@ -95,17 +95,14 @@ def main(log_level, parallel, sdk_root, output_dir): if not os.path.exists(output_dir): os.makedirs(output_dir) + targets = FindPregenerationTargets(sdk_root) + if parallel: - target_and_dir = zip( - FindPregenerationTargets(sdk_root), - itertools.repeat(output_dir)) - - p = multiprocessing.Pool() - p.map(_ParallelGenerateOne, target_and_dir) - p.close() - p.join() + target_and_dir = zip(targets, itertools.repeat(output_dir)) + with multiprocessing.Pool() as pool: + pool.map(_ParallelGenerateOne, target_and_dir) else: - for target in FindPregenerationTargets(sdk_root): + for target in targets: target.Generate(output_dir) logging.info("Done") From 8b92391517df47369e7a99301f882e42728780af Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 16:11:51 -0500 Subject: [PATCH 27/47] Use imap_unordered to not care about actual parallel generation order --- scripts/codepregen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/codepregen.py b/scripts/codepregen.py index 511034af244665..3d2d7781b40312 100755 --- a/scripts/codepregen.py +++ b/scripts/codepregen.py @@ -100,7 +100,8 @@ def main(log_level, parallel, sdk_root, output_dir): if parallel: target_and_dir = zip(targets, itertools.repeat(output_dir)) with multiprocessing.Pool() as pool: - pool.map(_ParallelGenerateOne, target_and_dir) + for _ in pool.imap_unordered(_ParallelGenerateOne, target_and_dir): + pass else: for target in targets: target.Generate(output_dir) From 1ec8ea55fec5628ccb7bb399a342c4937df0d0f3 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 17:22:20 -0500 Subject: [PATCH 28/47] Also fix java jni codegen with pregenerated data --- src/controller/data_model/BUILD.gn | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/controller/data_model/BUILD.gn b/src/controller/data_model/BUILD.gn index 011669db09524a..09f52c9da74829 100644 --- a/src/controller/data_model/BUILD.gn +++ b/src/controller/data_model/BUILD.gn @@ -170,8 +170,6 @@ if (current_os == "android" || build_java_matter_controller) { } source_set("java-jni-sources") { - sources = get_target_outputs(":java-jni-generate") - public_configs = [ "${chip_root}/src:includes" ] deps = [ ":data_model", From 59388bec10b5a61b18c4c072a3407318cd45dc80 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 24 Nov 2022 17:26:25 -0500 Subject: [PATCH 29/47] Fix java compilation deps: java codegen uses data model files --- src/controller/data_model/BUILD.gn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/controller/data_model/BUILD.gn b/src/controller/data_model/BUILD.gn index 09f52c9da74829..99765d3c165fbc 100644 --- a/src/controller/data_model/BUILD.gn +++ b/src/controller/data_model/BUILD.gn @@ -167,6 +167,8 @@ if (current_os == "android" || build_java_matter_controller) { "jni/UnitTestingClient-ReadImpl.cpp", "jni/UnitTestingClient-InvokeSubscribeImpl.cpp", ] + + deps = [ ":data_model" ] } source_set("java-jni-sources") { From 674dc36307989b57a99774090daaaf39562bef2c Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 10:21:22 -0500 Subject: [PATCH 30/47] NRF now can use pregen folder --- build/chip/chip_codegen.cmake | 91 +++++++++++++++++++++++------------ scripts/build/builders/nrf.py | 3 ++ 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/build/chip/chip_codegen.cmake b/build/chip/chip_codegen.cmake index 17e3844fb45ba9..885d189f43f6de 100644 --- a/build/chip/chip_codegen.cmake +++ b/build/chip/chip_codegen.cmake @@ -31,40 +31,71 @@ function(chip_codegen TARGET_NAME) ${ARGN} ) - set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}") + set(CHIP_CODEGEN_PREGEN_DIR "" CACHE PATH "Pre-generated directory to use instead of compile-time code generation.") - string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}") + if ("${CHIP_CODEGEN_PREGEN_DIR}" STREQUAL "") + set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}") - file(MAKE_DIRECTORY "${GEN_FOLDER}") - file(GENERATE - OUTPUT "${GEN_FOLDER}/expected.outputs" - CONTENT "${OUTPUT_AS_NEWLINES}" - ) + string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}") + file(MAKE_DIRECTORY "${GEN_FOLDER}") + file(GENERATE + OUTPUT "${GEN_FOLDER}/expected.outputs" + CONTENT "${OUTPUT_AS_NEWLINES}" + ) + + + set(OUT_NAMES) + foreach(NAME IN LISTS ARG_OUTPUTS) + list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}") + endforeach() + + # Python is expected to be in the path + # + # find_package(Python3 REQUIRED) + add_custom_command( + OUTPUT ${OUT_NAMES} + COMMAND "${CHIP_ROOT}/scripts/codegen.py" + ARGS "--generator" "${ARG_GENERATOR}" + "--output-dir" "${GEN_FOLDER}" + "--expected-outputs" "${GEN_FOLDER}/expected.outputs" + "${ARG_INPUT}" + DEPENDS + "${ARG_INPUT}" + VERBATIM + ) + + add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}") + + # Forward outputs to the parent + set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE) + set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE) + else() + # Gets a path such as: + # examples/lock-app/lock-common/lock-app.matter + file(RELATIVE_PATH MATTER_FILE_PATH "${CHIP_ROOT}" ${ARG_INPUT}) + + # Removes the trailing file extension to get something like: + # examples/lock-app/lock-common/lock-app + string(REGEX REPLACE "\.matter$" "" CODEGEN_DIR_PATH "${MATTER_FILE_PATH}") + + + # Build the final location within the pregen directory + set(GEN_FOLDER "${CHIP_CODEGEN_PREGEN_DIR}/${CODEGEN_DIR_PATH}/codegen/${ARG_GENERATOR}") + + # TODO: build a fake target of ${TARGET_NAME} + + # Here we have ${CHIP_CODEGEN_PREGEN_DIR} + set(OUT_NAMES) + foreach(NAME IN LISTS ARG_OUTPUTS) + list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}") + endforeach() - set(OUT_NAMES) - foreach(NAME IN LISTS ARG_OUTPUTS) - list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}") - endforeach() - - # Python is expected to be in the path - # - # find_package(Python3 REQUIRED) - add_custom_command( - OUTPUT ${OUT_NAMES} - COMMAND "${CHIP_ROOT}/scripts/codegen.py" - ARGS "--generator" "${ARG_GENERATOR}" - "--output-dir" "${GEN_FOLDER}" - "--expected-outputs" "${GEN_FOLDER}/expected.outputs" - "${ARG_INPUT}" - DEPENDS - "${ARG_INPUT}" - VERBATIM - ) - add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}") + set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE) + set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE) - # Forward outputs to the parent - set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE) - set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE) + # allow adding dependencies to a phony target since no codegen is done + add_custom_target(${TARGET_NAME}) + endif() endfunction() diff --git a/scripts/build/builders/nrf.py b/scripts/build/builders/nrf.py index 39c93946b9b27d..bd831ce09ec0c1 100644 --- a/scripts/build/builders/nrf.py +++ b/scripts/build/builders/nrf.py @@ -172,6 +172,9 @@ def generate(self): if self.board == NrfBoard.NRF52840DONGLE and self.app != NrfApp.ALL_CLUSTERS and self.app != NrfApp.ALL_CLUSTERS_MINIMAL: flags.append("-DCONF_FILE=prj_no_dfu.conf") + if self.options.pregen_dir: + flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR=\"{self.options.pregen_dir}\"") + build_flags = " -- " + " ".join(flags) if len(flags) > 0 else "" cmd = ''' From 8ec31d6d5db8589b615ede342be67ef21494ed8a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 10:27:02 -0500 Subject: [PATCH 31/47] Allow telink to also use a pregen dir --- scripts/build/builders/telink.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/build/builders/telink.py b/scripts/build/builders/telink.py index a85ec7c9fe58d6..1fdaed26af03dd 100644 --- a/scripts/build/builders/telink.py +++ b/scripts/build/builders/telink.py @@ -100,15 +100,21 @@ def generate(self): if os.path.exists(self.output_dir): return + flags = [] + if self.options.pregen_dir: + flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR=\"{self.options.pregen_dir}\"") + + build_flags = " -- " + " ".join(flags) if len(flags) > 0 else "" + cmd = self.get_cmd_prefixes() cmd += ''' source "$ZEPHYR_BASE/zephyr-env.sh"; -west build --cmake-only -d {outdir} -b {board} {sourcedir} +west build --cmake-only -d {outdir} -b {board} {sourcedir}{build_flags} '''.format( - outdir=shlex.quote( - self.output_dir), board=self.board.GnArgName(), sourcedir=shlex.quote( - os.path.join( - self.root, 'examples', self.app.ExampleName(), 'telink'))).strip() + outdir=shlex.quote(self.output_dir), + board=self.board.GnArgName(), + sourcedir=shlex.quote(os.path.join(self.root, 'examples', self.app.ExampleName(), 'telink')), + build_flags=build_flags).strip() self._Execute(['bash', '-c', cmd], title='Generating ' + self.identifier) From f612450a149f2241ac00898e3816be95e5a3abe6 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 10:39:05 -0500 Subject: [PATCH 32/47] Better shell escape, make mbedos cmake flags work with pregen dir --- scripts/build/builders/mbed.py | 17 +++++++++-------- scripts/build/builders/nrf.py | 2 +- scripts/build/builders/telink.py | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/build/builders/mbed.py b/scripts/build/builders/mbed.py index 73fc802386207f..748ba488a68517 100644 --- a/scripts/build/builders/mbed.py +++ b/scripts/build/builders/mbed.py @@ -121,14 +121,15 @@ def generate(self): '--mbed-os-path', self.mbed_os_path, ], title='Generating config ' + self.identifier) - self._Execute(['cmake', '-S', shlex.quote(self.ExamplePath), '-B', shlex.quote(self.output_dir), '-GNinja', - '-DCMAKE_BUILD_TYPE={}'.format( - self.profile.ProfileName.lower()), - '-DMBED_OS_PATH={}'.format( - shlex.quote(self.mbed_os_path)), - '-DMBED_OS_POSIX_SOCKET_PATH={}'.format( - shlex.quote(self.mbed_os_posix_socket_path)), - ], title='Generating ' + self.identifier) + flags = [] + flags.append(f"-DMBED_OS_PATH={shlex.quote(self.mbed_os_path)}") + flags.append(f"-DMBED_OS_PATH={shlex.quote(self.mbed_os_path)}") + flags.append(f"-DMBED_OS_POSIX_SOCKET_PATH={shlex.quote(self.mbed_os_posix_socket_path)}") + + if self.options.pregen_dir: + flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") + + self._Execute(['cmake', '-S', shlex.quote(self.ExamplePath), '-B', shlex.quote(self.output_dir), '-GNinja'] + flags, title='Generating ' + self.identifier) def _build(self): # Remove old artifacts to force linking diff --git a/scripts/build/builders/nrf.py b/scripts/build/builders/nrf.py index bd831ce09ec0c1..6aa985f796f959 100644 --- a/scripts/build/builders/nrf.py +++ b/scripts/build/builders/nrf.py @@ -173,7 +173,7 @@ def generate(self): flags.append("-DCONF_FILE=prj_no_dfu.conf") if self.options.pregen_dir: - flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR=\"{self.options.pregen_dir}\"") + flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") build_flags = " -- " + " ".join(flags) if len(flags) > 0 else "" diff --git a/scripts/build/builders/telink.py b/scripts/build/builders/telink.py index 1fdaed26af03dd..32a9bf85c3187e 100644 --- a/scripts/build/builders/telink.py +++ b/scripts/build/builders/telink.py @@ -102,7 +102,7 @@ def generate(self): flags = [] if self.options.pregen_dir: - flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR=\"{self.options.pregen_dir}\"") + flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") build_flags = " -- " + " ".join(flags) if len(flags) > 0 else "" From 26bb108b650d1957cebef1085393dc90cba8626e Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 10:39:24 -0500 Subject: [PATCH 33/47] Restyle --- scripts/build/builders/mbed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build/builders/mbed.py b/scripts/build/builders/mbed.py index 748ba488a68517..46b3ead610bb72 100644 --- a/scripts/build/builders/mbed.py +++ b/scripts/build/builders/mbed.py @@ -129,7 +129,8 @@ def generate(self): if self.options.pregen_dir: flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") - self._Execute(['cmake', '-S', shlex.quote(self.ExamplePath), '-B', shlex.quote(self.output_dir), '-GNinja'] + flags, title='Generating ' + self.identifier) + self._Execute(['cmake', '-S', shlex.quote(self.ExamplePath), '-B', shlex.quote(self.output_dir), + '-GNinja'] + flags, title='Generating ' + self.identifier) def _build(self): # Remove old artifacts to force linking From d6c3830d733d9ed010fc4974602c39de077efa3b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 10:44:33 -0500 Subject: [PATCH 34/47] Add pregen support for esp32 as well --- scripts/build/builders/esp32.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py index d84e8f56e8edbc..c04e7978f907fe 100644 --- a/scripts/build/builders/esp32.py +++ b/scripts/build/builders/esp32.py @@ -176,10 +176,18 @@ def generate(self): self._Execute( ['bash', '-c', 'echo -e "\\nCONFIG_DISABLE_IPV4=y\\n" >>%s' % shlex.quote(defaults_out)]) - cmd = "\nexport SDKCONFIG_DEFAULTS={defaults}\nidf.py -C {example_path} -B {out} reconfigure".format( + cmake_flags = [] + + if self.options.pregen_dir: + cmake_flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") + + cmake_args = (" " + " ".join(cmake_flags)) if len(cmake_flags) > 0 else "" + + cmd = "\nexport SDKCONFIG_DEFAULTS={defaults}\nidf.py -C {example_path} -B {out}{cmake_args} reconfigure".format( defaults=shlex.quote(defaults_out), example_path=self.ExamplePath, - out=shlex.quote(self.output_dir) + out=shlex.quote(self.output_dir), + cmake_args=cmake_args, ) # This will do a 'cmake reconfigure' which will create ninja files without rebuilding From 5f1ae5c41aedb68721d6e5b0e5b5cccac52575f1 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 10:54:28 -0500 Subject: [PATCH 35/47] Add a test for esp32 pregeneration support --- .github/workflows/examples-esp32.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/examples-esp32.yaml b/.github/workflows/examples-esp32.yaml index 6755e3f64b7bbe..59cfd55df1be3c 100644 --- a/.github/workflows/examples-esp32.yaml +++ b/.github/workflows/examples-esp32.yaml @@ -71,11 +71,29 @@ jobs: "./scripts/build/build_examples.py \ --enable-flashbundle \ --target esp32-m5stack-all-clusters \ + build \ + --copy-artifacts-to out/artifacts \ + " + - name: Prepare code pregen and ensure compile time pregen not possible + run: | + ./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated" + mv scripts/codegen.py scripts/codegen.py.renamed + - name: Build some M5Stack variations with pregen + timeout-minutes: 60 + run: | + ./scripts/run_in_build_env.sh \ + "./scripts/build/build_examples.py \ + --enable-flashbundle \ --target esp32-m5stack-all-clusters-minimal \ --target esp32-m5stack-all-clusters-rpc-ipv6only \ + --pregen-dir ./zzz_pregenerated build \ --copy-artifacts-to out/artifacts \ " + - name: Undo code pregeneration changes + run: | + rm -rf ./zzz_pregenerated + mv scripts/codegen.py.renamed scripts/codegen.py - name: Prepare bloat report run: | .environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py \ From fb75a60843e5c8e797b477ba47465b47e7626eba Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 10:56:56 -0500 Subject: [PATCH 36/47] Also test pregen support in linux builds (to check gn builds) --- .github/workflows/build.yaml | 20 ++++++++++++++++++-- .github/workflows/examples-esp32.yaml | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 276b1f264e0cee..be4961700a0f4f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -229,12 +229,28 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py --no-log-timestamps \ - --target linux-x64-all-clusters-ipv6only-clang \ - --target linux-x64-chip-tool-ipv6only-clang \ --target linux-x64-minmdns-ipv6only-clang \ --target linux-x64-rpc-console \ build \ " + - name: Prepare code pregen and ensure compile time pregen not possible + run: | + ./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated" + mv scripts/codegen.py scripts/codegen.py.renamed + - name: Build using build_examples.py (pregen) + timeout-minutes: 60 + run: | + ./scripts/run_in_build_env.sh \ + "./scripts/build/build_examples.py --no-log-timestamps \ + --target linux-x64-all-clusters-ipv6only-clang \ + --target linux-x64-chip-tool-ipv6only-clang \ + --pregen-dir ./zzz_pregenerated \ + build \ + " + - name: Undo code pregeneration changes + run: | + rm -rf ./zzz_pregenerated + mv scripts/codegen.py.renamed scripts/codegen.py - name: Run fake linux tests with build_examples timeout-minutes: 15 run: | diff --git a/.github/workflows/examples-esp32.yaml b/.github/workflows/examples-esp32.yaml index 59cfd55df1be3c..9299ac3822c684 100644 --- a/.github/workflows/examples-esp32.yaml +++ b/.github/workflows/examples-esp32.yaml @@ -86,7 +86,7 @@ jobs: --enable-flashbundle \ --target esp32-m5stack-all-clusters-minimal \ --target esp32-m5stack-all-clusters-rpc-ipv6only \ - --pregen-dir ./zzz_pregenerated + --pregen-dir ./zzz_pregenerated \ build \ --copy-artifacts-to out/artifacts \ " From 4f80a04acc3723158bcb2d79a64250ad5937aa10 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 10:57:28 -0500 Subject: [PATCH 37/47] Remove unused file --- build/chip/copy_tree.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 build/chip/copy_tree.py diff --git a/build/chip/copy_tree.py b/build/chip/copy_tree.py deleted file mode 100644 index f2cfa56bb083d6..00000000000000 --- a/build/chip/copy_tree.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python - -import argparse -import shutil -import sys - - -def main(): - - parser = argparse.ArgumentParser(description="Copy a tree of files from a source directory to a destination one.") - - parser.add_argument('--src-dir', required=True) - parser.add_argument('--dest-dir', required=True) - - args = parser.parse_args() - - shutil.copytree(args.src_dir, args.dest_dir, dirs_exist_ok=True) - - -if __name__ == '__main__': - sys.exit(main()) From e2f781d0ba660e6aec4bea52097f53b91112389b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 11:05:18 -0500 Subject: [PATCH 38/47] Fix spelling --- docs/code_generation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/code_generation.md b/docs/code_generation.md index d48006d95c7a98..051a2152fb36a3 100644 --- a/docs/code_generation.md +++ b/docs/code_generation.md @@ -200,7 +200,7 @@ Code pre-generation can be used: of golden image tests in `scripts/idl/tests` The script to trigger code pre-generation is `scripts/code_pregenerate.py` and -requires the pregeneration output directory as an argument +requires the pre-generation output directory as an argument ```bash scripts/code_pregenerate.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} From c30b77aae8645186f908367aa4bea8a36cfe2eaf Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 11:29:47 -0500 Subject: [PATCH 39/47] Fix esp32 compilation - gn arguments need to be passed from cmake --- config/esp32/components/chip/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/esp32/components/chip/CMakeLists.txt b/config/esp32/components/chip/CMakeLists.txt index 44f923808e17e6..1513075ace0f28 100644 --- a/config/esp32/components/chip/CMakeLists.txt +++ b/config/esp32/components/chip/CMakeLists.txt @@ -101,6 +101,10 @@ if(CONFIG_DISABLE_IPV4) chip_gn_arg_append("chip_inet_config_enable_ipv4" "false") endif() +if(CHIP_CODEGEN_PREGEN_DIR) + chip_gn_arg_append("chip_code_pre_gegenerated_directory" "\"${CHIP_CODEGEN_PREGEN_DIR}\"") +endif() + if(CONFIG_ENABLE_PW_RPC) string(APPEND chip_gn_args "import(\"//build_overrides/pigweed.gni\")\n") chip_gn_arg_append("remove_default_configs" "[\"//third_party/connectedhomeip/third_party/pigweed/repo/pw_build:toolchain_cpp_standard\"]") From 5ee8508513da3950c3b524fd8bf4732362f9a620 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 11:47:34 -0500 Subject: [PATCH 40/47] Fix some define forwarding logic in codegen --- build/chip/chip_codegen.gni | 7 +++---- src/app/chip_data_model.gni | 11 +++++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index 188199adba7e47..9a45ef5617bab9 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -80,11 +80,10 @@ template("_chip_build_time_codegen") { forward_variables_from(invoker, [ "deps" ]) - if (defined(deps)) { - deps += [ ":${_name}_codegen" ] - } else { - deps = [ ":${_name}_codegen" ] + if (!defined(deps)) { + deps = [] } + deps += [ ":${_name}_codegen" ] } } diff --git a/src/app/chip_data_model.gni b/src/app/chip_data_model.gni index 76b8e9c3379116..ecab5a9d02a465 100644 --- a/src/app/chip_data_model.gni +++ b/src/app/chip_data_model.gni @@ -70,7 +70,11 @@ template("chip_data_model") { public_configs = [ ":${_data_model_name}_config" ] - deps = [ "${chip_root}/src/app/common:cluster-objects" ] + if (!defined(deps)) { + deps = [] + } + + deps += [ "${chip_root}/src/app/common:cluster-objects" ] } _use_default_im_dispatch = !defined(invoker.use_default_im_dispatch) || @@ -89,7 +93,10 @@ template("chip_data_model") { sources = [] } - deps = [ ":${_data_model_name}_codegen" ] + if (!defined(deps)) { + deps = [] + } + deps += [ ":${_data_model_name}_codegen" ] sources += [ "${_app_root}/clusters/barrier-control-server/barrier-control-server.h", From ad97941dcfd9fe47a831e553f4de08386c7cddd1 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 13:01:09 -0500 Subject: [PATCH 41/47] Make sure java build config (which includes header paths) is set as a config and applies to generated sources --- src/controller/data_model/BUILD.gn | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/controller/data_model/BUILD.gn b/src/controller/data_model/BUILD.gn index 99765d3c165fbc..938082f39bbd08 100644 --- a/src/controller/data_model/BUILD.gn +++ b/src/controller/data_model/BUILD.gn @@ -31,6 +31,13 @@ chip_data_model("data_model") { } if (current_os == "android" || build_java_matter_controller) { + + config("java-build-config") { + if (build_java_matter_controller) { + include_dirs = java_matter_controller_dependent_paths + } + } + chip_codegen("java-jni-generate") { input = "controller-clusters.matter" generator = "java" @@ -168,7 +175,12 @@ if (current_os == "android" || build_java_matter_controller) { "jni/UnitTestingClient-InvokeSubscribeImpl.cpp", ] - deps = [ ":data_model" ] + deps = [ + ":data_model", + "${chip_root}/src/platform:platform_buildconfig", + ] + + public_configs = [ ":java-build-config" ] } source_set("java-jni-sources") { @@ -182,7 +194,6 @@ if (current_os == "android" || build_java_matter_controller) { ] if (build_java_matter_controller) { - include_dirs = java_matter_controller_dependent_paths deps += [ "${chip_root}/src/platform/Linux" ] } else { deps += [ "${chip_root}/src/platform/android" ] From 68c4869d56056de0c9746323151c5ceb74c1e57b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 13:01:54 -0500 Subject: [PATCH 42/47] java build config should apply to all sources, not just transitively --- src/controller/data_model/BUILD.gn | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/controller/data_model/BUILD.gn b/src/controller/data_model/BUILD.gn index 938082f39bbd08..7c4f4e02bd2432 100644 --- a/src/controller/data_model/BUILD.gn +++ b/src/controller/data_model/BUILD.gn @@ -184,7 +184,11 @@ if (current_os == "android" || build_java_matter_controller) { } source_set("java-jni-sources") { - public_configs = [ "${chip_root}/src:includes" ] + public_configs = [ + ":java-build-config", + "${chip_root}/src:includes", + ] + deps = [ ":data_model", ":java-jni-generate", From cf2075c59a294f3a5c4a20793b1f5f4f44e82525 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 25 Nov 2022 13:02:11 -0500 Subject: [PATCH 43/47] Restyle --- src/controller/data_model/BUILD.gn | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/controller/data_model/BUILD.gn b/src/controller/data_model/BUILD.gn index 7c4f4e02bd2432..95b0e6cf02072a 100644 --- a/src/controller/data_model/BUILD.gn +++ b/src/controller/data_model/BUILD.gn @@ -31,7 +31,6 @@ chip_data_model("data_model") { } if (current_os == "android" || build_java_matter_controller) { - config("java-build-config") { if (build_java_matter_controller) { include_dirs = java_matter_controller_dependent_paths @@ -176,8 +175,8 @@ if (current_os == "android" || build_java_matter_controller) { ] deps = [ - ":data_model", - "${chip_root}/src/platform:platform_buildconfig", + ":data_model", + "${chip_root}/src/platform:platform_buildconfig", ] public_configs = [ ":java-build-config" ] From 0f340e988ba30e31063e44edeb592137c3b4de44 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 28 Nov 2022 11:12:54 -0500 Subject: [PATCH 44/47] Replace codege with codegen. --- build/chip/chip_codegen.gni | 2 +- scripts/codepregen.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index 9a45ef5617bab9..62905dccfac08f 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -116,7 +116,7 @@ template("_chip_build_time_codegen") { # # NOTE: content of "outputs" is verified to match the output of codegen.py # exactly. It is not inferred on purpose, to make build-rules explicit -# and verifiable (even though codege.py can at runtime report its outputs) +# and verifiable (even though codegen.py can at runtime report its outputs) # # To find the list of generated files, you can run codegen.py with the # "--name-only" argument diff --git a/scripts/codepregen.py b/scripts/codepregen.py index 3d2d7781b40312..13ca9a60d972c7 100755 --- a/scripts/codepregen.py +++ b/scripts/codepregen.py @@ -63,7 +63,7 @@ def _ParallelGenerateOne(arg): @click.option( '--parallel/--no-parallel', default=True, - help='Do parallel/multiprocessing codege.') + help='Do parallel/multiprocessing codegen.') @click.option( '--sdk-root', default=None, @@ -81,7 +81,8 @@ def main(log_level, parallel, sdk_root, output_dir): ) if not sdk_root: - sdk_root = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') + sdk_root = os.path.join(os.path.dirname( + os.path.realpath(__file__)), '..') sdk_root = os.path.abspath(sdk_root) From 33e09a7a4165c7d134bfee71af565c9a01e8496c Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 28 Nov 2022 11:13:58 -0500 Subject: [PATCH 45/47] Fix naming typo --- build/chip/chip_codegen.gni | 8 ++++---- config/esp32/components/chip/CMakeLists.txt | 2 +- scripts/build/builders/gn.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build/chip/chip_codegen.gni b/build/chip/chip_codegen.gni index 62905dccfac08f..fd5d69c40c513f 100644 --- a/build/chip/chip_codegen.gni +++ b/build/chip/chip_codegen.gni @@ -20,7 +20,7 @@ import("$dir_pw_build/python.gni") declare_args() { # Location where code has been pre-generated - chip_code_pre_gegenerated_directory = "" + chip_code_pre_generated_directory = "" } # Code generation that will happen at build time. @@ -106,7 +106,7 @@ template("_chip_build_time_codegen") { # # Command line parameters: # -# chip_code_pre_gegenerated_directory: +# chip_code_pre_generated_directory: # - If this is set, generation will NOT happen at compile time but rather # the code generation is assumed to have already happened and reside in # the given location. @@ -138,7 +138,7 @@ template("_chip_build_time_codegen") { # } # template("chip_codegen") { - if (chip_code_pre_gegenerated_directory == "") { + if (chip_code_pre_generated_directory == "") { _chip_build_time_codegen(target_name) { forward_variables_from(invoker, [ @@ -157,7 +157,7 @@ template("chip_codegen") { # USING "cpp-app" for generator: # => ${pregen_dir}/examples/all-clusters-app/all-clusters-common/all-clusters-app/codegen/cpp-app _generation_dir = - chip_code_pre_gegenerated_directory + "/" + + chip_code_pre_generated_directory + "/" + string_replace(rebase_path(invoker.input, chip_root), ".matter", "") + "/codegen/" + invoker.generator diff --git a/config/esp32/components/chip/CMakeLists.txt b/config/esp32/components/chip/CMakeLists.txt index 1513075ace0f28..d7cb69a6a8fa34 100644 --- a/config/esp32/components/chip/CMakeLists.txt +++ b/config/esp32/components/chip/CMakeLists.txt @@ -102,7 +102,7 @@ if(CONFIG_DISABLE_IPV4) endif() if(CHIP_CODEGEN_PREGEN_DIR) - chip_gn_arg_append("chip_code_pre_gegenerated_directory" "\"${CHIP_CODEGEN_PREGEN_DIR}\"") + chip_gn_arg_append("chip_code_pre_generated_directory" "\"${CHIP_CODEGEN_PREGEN_DIR}\"") endif() if(CONFIG_ENABLE_PW_RPC) diff --git a/scripts/build/builders/gn.py b/scripts/build/builders/gn.py index 396001ff582460..9e859f6025a5a7 100644 --- a/scripts/build/builders/gn.py +++ b/scripts/build/builders/gn.py @@ -66,7 +66,7 @@ def generate(self): extra_args.append('pw_command_launcher="%s"' % self.options.pw_command_launcher) if self.options.pregen_dir: - extra_args.append('chip_code_pre_gegenerated_directory="%s"' % self.options.pregen_dir) + extra_args.append('chip_code_pre_generated_directory="%s"' % self.options.pregen_dir) extra_args.extend(self.GnBuildArgs() or []) if extra_args: From 60e0f51684feb387be2b8bf8421cc37b8c9c31fa Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 28 Nov 2022 11:15:29 -0500 Subject: [PATCH 46/47] Update text for build steps --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index be4961700a0f4f..4a3ce8550c2e7e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -233,7 +233,7 @@ jobs: --target linux-x64-rpc-console \ build \ " - - name: Prepare code pregen and ensure compile time pregen not possible + - name: Create a pre-generate directory and ensure compile-time codegen would fail run: | ./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated" mv scripts/codegen.py scripts/codegen.py.renamed @@ -247,7 +247,7 @@ jobs: --pregen-dir ./zzz_pregenerated \ build \ " - - name: Undo code pregeneration changes + - name: Undo code pre-generation changes (make compile time codegen work again) run: | rm -rf ./zzz_pregenerated mv scripts/codegen.py.renamed scripts/codegen.py From cf128681edda9838921ae6b0c2721a4979aaf232 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 28 Nov 2022 11:29:30 -0500 Subject: [PATCH 47/47] Update spacing logic to make the code cleaner. The nospace args is odd --- scripts/build/builders/esp32.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py index c04e7978f907fe..54c47a48cb557f 100644 --- a/scripts/build/builders/esp32.py +++ b/scripts/build/builders/esp32.py @@ -179,16 +179,16 @@ def generate(self): cmake_flags = [] if self.options.pregen_dir: - cmake_flags.append(f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") + cmake_flags.append( + f"-DCHIP_CODEGEN_PREGEN_DIR={shlex.quote(self.options.pregen_dir)}") - cmake_args = (" " + " ".join(cmake_flags)) if len(cmake_flags) > 0 else "" + cmake_args = ['-C', self.ExamplePath, '-B', + shlex.quote(self.output_dir)] + cmake_flags - cmd = "\nexport SDKCONFIG_DEFAULTS={defaults}\nidf.py -C {example_path} -B {out}{cmake_args} reconfigure".format( - defaults=shlex.quote(defaults_out), - example_path=self.ExamplePath, - out=shlex.quote(self.output_dir), - cmake_args=cmake_args, - ) + cmake_args = " ".join(cmake_args) + defaults = shlex.quote(defaults_out) + + cmd = f"\nexport SDKCONFIG_DEFAULTS={defaults}\nidf.py {cmake_args} reconfigure" # This will do a 'cmake reconfigure' which will create ninja files without rebuilding self._IdfEnvExecute(cmd)