From 16808843b2a5abc3e2bcc29a5b91495e280b5e27 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 11 Apr 2023 13:30:08 -0400 Subject: [PATCH] Allow code pre-generation to accept files outside of chip-root (#25970) * Start supporting separate root pregeneration * Update documentation * Restyle * Fix typos * Fix pregen logic * Update path handling to support full and relative paths, codepregen of external roots now works * Restyle * Update docs/code_generation.md Co-authored-by: Boris Zbarsky * Update docs/code_generation.md Co-authored-by: Boris Zbarsky * Update docs/code_generation.md Co-authored-by: Boris Zbarsky * Slight rephrase * Another rephrase * Restyle * Normalize paths to not be off by one on generation path lenghts --------- Co-authored-by: Andrei Litvin Co-authored-by: Boris Zbarsky --- docs/code_generation.md | 41 +++++++++++++++++++++++++++- scripts/codepregen.py | 9 ++++-- scripts/pregenerate/__init__.py | 37 +++++++++++++++++-------- scripts/pregenerate/types.py | 1 + scripts/pregenerate/using_codegen.py | 4 +-- scripts/pregenerate/using_zap.py | 9 ++++-- 6 files changed, 83 insertions(+), 18 deletions(-) diff --git a/docs/code_generation.md b/docs/code_generation.md index f9b0d587f3fd9b..318cfedd359954 100644 --- a/docs/code_generation.md +++ b/docs/code_generation.md @@ -234,9 +234,48 @@ scripts/codepregen.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} # To generate a single output you can use `--input-glob`: -scripts/codepregen.py --input-glob "*all-clusters*" ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} +scripts/codepregen.py --input-glob "*all-clusters*" --input-glob "*controller*" ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} ``` +### External applications/zap files + +#### Ensure you have a `.matter` file + +Code generation generally will use both `.zap` or `.matter` files. If you only +have a `.zap` file, you can create the corresponding `.matter` file via: + +```bash +scripts/tools/zap/generate.py ${ZAP_FILE_PATH} +``` + +The above will use the template `src/app/zap-templates/matter-idl.json` to +generate a `.matter` file corresponding to the input `.zap` file. + +`.matter` files are designed to be human readable. It is recommended to take a +look at the generated file and see if it contains what is expected and also lint +it. If anything seems wrong, the `.zap` file should be fixed (`.matter` +represents the content of `.zap`). To lint use: + +```bash +scripts/idl_lint.py ${MATTER_FILE_PATH} +``` + +#### Running pre-generation + +If you have zap files outside the CHIP repository (i.e. not in `src` or +`examples`) you should provide the root of your application source. + +```bash +scripts/codepregen.py --external-root ${PATH_TO_SOURCE_ROOT} ${OUTPUT_DIRECTORY:-./zzz_pregenerated/} +``` + +NOTE: `$PATH_TO_SOURCE_ROOT` should be a top-level directory containing +zap/matter files as the code pre-generation will generate files based on the +path inside the root: + +- if files are `$PATH_TO_SOURCE_ROOT/some/path/foo.zap` this will generate + files into `$OUTPUT_DIRECTORY/some/path/foo/...` + ### Using pre-generated code Instead of generating code at compile time, the chip build system accepts usage diff --git a/scripts/codepregen.py b/scripts/codepregen.py index 7c5bfd747295cd..43de8fc21c4dc1 100755 --- a/scripts/codepregen.py +++ b/scripts/codepregen.py @@ -85,8 +85,13 @@ def _ParallelGenerateOne(arg): '--sdk-root', default=None, help='Path to the SDK root (where .zap/.matter files exist).') +@click.option( + '--external-root', + default=None, + multiple=True, + help='Path to an external app root (where .zap/.matter files exist).') @click.argument('output_dir') -def main(log_level, parallel, dry_run, generator, input_glob, sdk_root, output_dir): +def main(log_level, parallel, dry_run, generator, input_glob, sdk_root, external_root, output_dir): if _has_coloredlogs: coloredlogs.install(level=__LOG_LEVELS__[ log_level], fmt='%(asctime)s %(levelname)-7s %(message)s') @@ -122,7 +127,7 @@ def main(log_level, parallel, dry_run, generator, input_glob, sdk_root, output_d elif generator == 'codegen': filter.file_type = IdlFileType.MATTER - targets = FindPregenerationTargets(sdk_root, filter, runner) + targets = FindPregenerationTargets(sdk_root, external_root, filter, runner) runner.ensure_directory_exists(output_dir) if parallel: diff --git a/scripts/pregenerate/__init__.py b/scripts/pregenerate/__init__.py index f664f5745b81dc..8cdf56719f28a6 100644 --- a/scripts/pregenerate/__init__.py +++ b/scripts/pregenerate/__init__.py @@ -25,7 +25,20 @@ from .using_zap import ZapApplicationPregenerator -def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]: +def _IdlsInDirectory(top_directory_name: str, truncate_length: int): + for root, dirs, files in os.walk(top_directory_name): + for file in files: + if file.endswith('.zap'): + yield InputIdlFile(file_type=IdlFileType.ZAP, + full_path=os.path.join(root, file), + relative_path=os.path.join(root[truncate_length:], file)) + if file.endswith('.matter'): + yield InputIdlFile(file_type=IdlFileType.MATTER, + full_path=os.path.join(root, file), + relative_path=os.path.join(root[truncate_length:], file)) + + +def _FindAllIdls(sdk_root: str, external_roots: Optional[List[str]]) -> Iterator[InputIdlFile]: relevant_subdirs = [ 'examples', # all example apps 'src', # realistically only controller/data_model @@ -35,17 +48,19 @@ def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]: sdk_root = sdk_root[:-1] sdk_root_length = len(sdk_root) + # first go over SDK items for subdir_name in relevant_subdirs: top_directory_name = os.path.join(sdk_root, subdir_name) logging.debug(f"Searching {top_directory_name}") - for root, dirs, files in os.walk(top_directory_name): - for file in files: - if file.endswith('.zap'): - yield InputIdlFile(file_type=IdlFileType.ZAP, - relative_path=os.path.join(root[sdk_root_length+1:], file)) - if file.endswith('.matter'): - yield InputIdlFile(file_type=IdlFileType.MATTER, - relative_path=os.path.join(root[sdk_root_length+1:], file)) + for idl in _IdlsInDirectory(top_directory_name, sdk_root_length+1): + yield idl + + # next external roots + if external_roots: + for root in external_roots: + root = os.path.normpath(root) + for idl in _IdlsInDirectory(root, len(root) + 1): + yield idl @dataclass @@ -67,7 +82,7 @@ def matches(self, s: str): return fnmatch.fnmatch(s, self.pattern) -def FindPregenerationTargets(sdk_root: str, filter: TargetFilter, runner): +def FindPregenerationTargets(sdk_root: str, external_roots: Optional[List[str]], filter: TargetFilter, runner): """Finds all relevand pre-generation targets in the given SDK root. @@ -88,7 +103,7 @@ def FindPregenerationTargets(sdk_root: str, filter: TargetFilter, runner): path_matchers = [GlobMatcher(pattern) for pattern in filter.path_glob] - for idl in FindAllIdls(sdk_root): + for idl in _FindAllIdls(sdk_root, external_roots): if filter.file_type is not None: if idl.file_type != filter.file_type: logging.debug(f"Will not process file of type {idl.file_type}: {idl.relative_path}") diff --git a/scripts/pregenerate/types.py b/scripts/pregenerate/types.py index 7f7d601e09a3e8..4e4f58bc4a0df7 100644 --- a/scripts/pregenerate/types.py +++ b/scripts/pregenerate/types.py @@ -26,6 +26,7 @@ class IdlFileType(Enum): class InputIdlFile: file_type: IdlFileType relative_path: str + full_path: str @property def pregen_subdir(self): diff --git a/scripts/pregenerate/using_codegen.py b/scripts/pregenerate/using_codegen.py index 5de358a0f483b1..13e5b71b388661 100644 --- a/scripts/pregenerate/using_codegen.py +++ b/scripts/pregenerate/using_codegen.py @@ -44,14 +44,14 @@ def Generate(self, output_root: str): output_root, self.idl.pregen_subdir, self.generator) logging.info( - f"Generating: {self.generator}:{self.idl.relative_path} into {output_dir}") + f"Generating: {self.generator}:{self.idl.full_path} into {output_dir}") cmd = [ CODEGEN_PY_PATH, '--log-level', 'fatal', '--generator', self.generator, '--output-dir', output_dir, - os.path.join(self.sdk_root, self.idl.relative_path) + self.idl.full_path ] logging.debug(f"Executing {cmd}") diff --git a/scripts/pregenerate/using_zap.py b/scripts/pregenerate/using_zap.py index 4d28f213370655..57df8ec42f7890 100644 --- a/scripts/pregenerate/using_zap.py +++ b/scripts/pregenerate/using_zap.py @@ -56,16 +56,21 @@ def Generate(self, output_root: str): output_dir = os.path.join(output_root, self.idl.pregen_subdir, self.generation_type.subdir) - logging.info(f"Generating: {self.generation_type}:{self.idl.relative_path} into {output_dir}") + logging.info(f"Generating: {self.generation_type}:{self.idl.full_path} into {output_dir}") self.runner.ensure_directory_exists(output_dir) + if self.idl.full_path.startswith(self.sdk_root): + idl_path = self.idl.relative_path + else: + idl_path = self.idl.full_path + cmd = [ ZAP_GENERATE_PATH, '--templates', self.generation_type.generation_template, '--output-dir', output_dir, '--parallel', - self.idl.relative_path + idl_path ] logging.debug(f"Executing {cmd}") self.runner.run(cmd, cwd=self.sdk_root)