From 4720a18bd92755006d1a102e04f9507ffd99c56c Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 15 Nov 2023 08:20:21 -0500 Subject: [PATCH 1/4] Revert "Fix build of `esp32-m5stack-all-clusters-rpc-ipv6only` for out of tree PW_ENVIRONMENT_ROOT (#30473)" This reverts commit 27328ae26fa8d611ea3fce8de4d1a79b79f2f433. --- .../build_overrides/pigweed_environment.gni | 69 +++---------------- 1 file changed, 9 insertions(+), 60 deletions(-) diff --git a/examples/build_overrides/pigweed_environment.gni b/examples/build_overrides/pigweed_environment.gni index bcc75edcff777b..a35e7843f84885 100644 --- a/examples/build_overrides/pigweed_environment.gni +++ b/examples/build_overrides/pigweed_environment.gni @@ -20,71 +20,20 @@ _bootstrap_root = "//third_party/connectedhomeip" import("${_bootstrap_root}/build_overrides/pigweed_environment.gni") # Rebase paths to our root. -# -# If out of tree, the paths will look like: -# -# pw_env_setup_CIPD_PIGWEED = "//../home/vscode/pigweed/env/cipd/packages/pigweed" -# -# and these paths are used by things like protoc since -# https://github.com/google/pigweed/commit/ddbc9fc7f5c601ab417db37e02cbe5294f21ad6d -# -# See https://github.com/project-chip/connectedhomeip/issues/30475 -# -# Existing logic: -# - this file is imported from `examples/common/pigweed` or similar -# - we transform paths from "//../" into "//../../../" -# -# TODO: need a better expansion here. This replacement logic seems very brittle and -# it is unclear how we know exactly 3 levels of indirections are correct -# if (defined(pw_env_setup_CIPD_ARM)) { - _split_arm = string_split(pw_env_setup_CIPD_ARM, "//../") - if (_split_arm[0] == "") { - pw_env_setup_CIPD_ARM = get_path_info( - string_replace(pw_env_setup_CIPD_ARM, "//../", "//../../../"), - "abspath") - } else { - pw_env_setup_CIPD_ARM = - get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_ARM}", "abspath") - } + pw_env_setup_CIPD_ARM = + get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_ARM}", "abspath") } if (defined(pw_env_setup_CIPD_PIGWEED)) { - _split_pigweed = string_split(pw_env_setup_CIPD_PIGWEED, "//../") - if (_split_pigweed[0] == "") { - pw_env_setup_CIPD_PIGWEED = - get_path_info( - string_replace(pw_env_setup_CIPD_PIGWEED, "//../", "//../../../"), - "abspath") - } else { - pw_env_setup_CIPD_PIGWEED = - get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PIGWEED}", - "abspath") - } + pw_env_setup_CIPD_PIGWEED = + get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PIGWEED}", + "abspath") } - if (defined(pw_env_setup_CIPD_PYTHON)) { - _split_python = string_split(pw_env_setup_CIPD_PYTHON, "//../") - if (_split_python[0] == "") { - pw_env_setup_CIPD_PYTHON = - get_path_info( - string_replace(pw_env_setup_CIPD_PYTHON, "//../", "//../../../"), - "abspath") - } else { - pw_env_setup_CIPD_PYTHON = - get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PYTHON}", - "abspath") - } + pw_env_setup_CIPD_PYTHON = + get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PYTHON}", "abspath") } if (defined(pw_env_setup_VIRTUAL_ENV)) { - _split_env = string_split(pw_env_setup_VIRTUAL_ENV, "//../") - if (_split_env[0] == "") { - pw_env_setup_VIRTUAL_ENV = - get_path_info( - string_replace(pw_env_setup_VIRTUAL_ENV, "//../", "//../../../"), - "abspath") - } else { - pw_env_setup_VIRTUAL_ENV = - get_path_info("${_bootstrap_root}/${pw_env_setup_VIRTUAL_ENV}", - "abspath") - } + pw_env_setup_VIRTUAL_ENV = + get_path_info("${_bootstrap_root}/${pw_env_setup_VIRTUAL_ENV}", "abspath") } From 76e86e5f99723d988aec7b6b5a7ad61a117cd4de Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 15 Nov 2023 08:46:36 -0500 Subject: [PATCH 2/4] start adding absolute paths in pigweed gni --- .../build_overrides/pigweed_environment.gni | 34 ++++++++---- scripts/setup/bootstrap.sh | 9 ++++ scripts/setup/gni_make_paths_absolute.py | 53 +++++++++++++++++++ 3 files changed, 87 insertions(+), 9 deletions(-) create mode 100755 scripts/setup/gni_make_paths_absolute.py diff --git a/examples/build_overrides/pigweed_environment.gni b/examples/build_overrides/pigweed_environment.gni index a35e7843f84885..538e35fca20efc 100644 --- a/examples/build_overrides/pigweed_environment.gni +++ b/examples/build_overrides/pigweed_environment.gni @@ -20,20 +20,36 @@ _bootstrap_root = "//third_party/connectedhomeip" import("${_bootstrap_root}/build_overrides/pigweed_environment.gni") # Rebase paths to our root. +# This rebasing is only done if the paths are repo-relative +# (i.e. start with "//") if (defined(pw_env_setup_CIPD_ARM)) { - pw_env_setup_CIPD_ARM = - get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_ARM}", "abspath") + _split_CIPD_ARM = string_split(pw_env_setup_CIPD_ARM, "//") + if (_split_CIPD_ARM[0] == "") { + pw_env_setup_CIPD_ARM = + get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_ARM}", "abspath") + } } if (defined(pw_env_setup_CIPD_PIGWEED)) { - pw_env_setup_CIPD_PIGWEED = - get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PIGWEED}", - "abspath") + _split_CIPD_PIGWEED = string_split(pw_env_setup_CIPD_PIGWEED, "//") + if (_split_CIPD_PIGWEED[0] == "") { + pw_env_setup_CIPD_PIGWEED = + get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PIGWEED}", + "abspath") + } } if (defined(pw_env_setup_CIPD_PYTHON)) { - pw_env_setup_CIPD_PYTHON = - get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PYTHON}", "abspath") + _split_CIPD_PYTHON = string_split(pw_env_setup_CIPD_PYTHON, "//") + if (_split_CIPD_PYTHON[0] == "") { + pw_env_setup_CIPD_PYTHON = + get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PYTHON}", + "abspath") + } } if (defined(pw_env_setup_VIRTUAL_ENV)) { - pw_env_setup_VIRTUAL_ENV = - get_path_info("${_bootstrap_root}/${pw_env_setup_VIRTUAL_ENV}", "abspath") + _split_VIRTUAL_ENV = string_split(pw_env_setup_VIRTUAL_ENV, "//") + if (_split_VIRTUAL_ENV[0] == "") { + pw_env_setup_VIRTUAL_ENV = + get_path_info("${_bootstrap_root}/${pw_env_setup_VIRTUAL_ENV}", + "abspath") + } } diff --git a/scripts/setup/bootstrap.sh b/scripts/setup/bootstrap.sh index 1b813216222efe..818d987a4f864b 100644 --- a/scripts/setup/bootstrap.sh +++ b/scripts/setup/bootstrap.sh @@ -165,6 +165,15 @@ if [ -n "$BASH" ]; then . "$_CHIP_ROOT/scripts/helpers/bash-completion.sh" fi +# Update relative paths to absolute (if they exist) +# to make sure loading of paths works in build_examples +# +# See https://github.com/project-chip/connectedhomeip/issues/30475 +# for details +scripts/setup/gni_make_paths_absolute.py \ + --root "$_ORIGINAL_PW_ENVIRONMENT_ROOT" \ + build_overrides/pigweed_environment.gni + unset -f _bootstrap_or_activate unset -f _install_additional_pip_requirements diff --git a/scripts/setup/gni_make_paths_absolute.py b/scripts/setup/gni_make_paths_absolute.py new file mode 100755 index 00000000000000..5848c1322d9e55 --- /dev/null +++ b/scripts/setup/gni_make_paths_absolute.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse + +def _make_paths_absolute(gni_file: str, root: str): + with open(gni_file, "rt", encoding="utf8") as f: + data = f.read() + + # Data will contain key/value pairs like: + # pw_env_setup_CIPD_PIGWEED = "//../home/vscode/pigweed/env/cipd/packages/pigweed" + # + # Looking to replace relative paths (starting with "//../") + # to absolute paths from the root + if not root.endswith("/"): + root = root + "/" + + new_data = data.replace('"//../', f'"{root}') + + if new_data == data: + return + + with open(gni_file, "wt", encoding="utf8") as f: + f.write(new_data) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + '--root', '-r', required=True, + help="Set the root used by the build (usually PW_ENVIRONMENT_ROOT)" + ) + parser.add_argument('gni_file', type=str, help="GNI file to process") + _make_paths_absolute(**vars(parser.parse_args())) + + +if __name__ == '__main__': + main() From 7b680d4e7b9eba95e3f84d2c646bdecade11e116 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 15 Nov 2023 09:12:02 -0500 Subject: [PATCH 3/4] Fix compilation - seems to work now --- examples/build_overrides/pigweed_environment.gni | 7 +++---- scripts/setup/bootstrap.sh | 2 +- scripts/setup/gni_make_paths_absolute.py | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/examples/build_overrides/pigweed_environment.gni b/examples/build_overrides/pigweed_environment.gni index 538e35fca20efc..670978063e72f1 100644 --- a/examples/build_overrides/pigweed_environment.gni +++ b/examples/build_overrides/pigweed_environment.gni @@ -33,16 +33,15 @@ if (defined(pw_env_setup_CIPD_PIGWEED)) { _split_CIPD_PIGWEED = string_split(pw_env_setup_CIPD_PIGWEED, "//") if (_split_CIPD_PIGWEED[0] == "") { pw_env_setup_CIPD_PIGWEED = - get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PIGWEED}", - "abspath") + get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PIGWEED}", "abspath") } } if (defined(pw_env_setup_CIPD_PYTHON)) { _split_CIPD_PYTHON = string_split(pw_env_setup_CIPD_PYTHON, "//") if (_split_CIPD_PYTHON[0] == "") { pw_env_setup_CIPD_PYTHON = - get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PYTHON}", - "abspath") + get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PYTHON}", + "abspath") } } if (defined(pw_env_setup_VIRTUAL_ENV)) { diff --git a/scripts/setup/bootstrap.sh b/scripts/setup/bootstrap.sh index 818d987a4f864b..eda6e97ad9c9bd 100644 --- a/scripts/setup/bootstrap.sh +++ b/scripts/setup/bootstrap.sh @@ -171,7 +171,7 @@ fi # See https://github.com/project-chip/connectedhomeip/issues/30475 # for details scripts/setup/gni_make_paths_absolute.py \ - --root "$_ORIGINAL_PW_ENVIRONMENT_ROOT" \ + --root "$_CHIP_ROOT" \ build_overrides/pigweed_environment.gni unset -f _bootstrap_or_activate diff --git a/scripts/setup/gni_make_paths_absolute.py b/scripts/setup/gni_make_paths_absolute.py index 5848c1322d9e55..8c8699205a342d 100755 --- a/scripts/setup/gni_make_paths_absolute.py +++ b/scripts/setup/gni_make_paths_absolute.py @@ -27,13 +27,13 @@ def _make_paths_absolute(gni_file: str, root: str): if not root.endswith("/"): root = root + "/" - new_data = data.replace('"//../', f'"{root}') + new_data = data.replace('"//../', f'"{root}../') if new_data == data: return with open(gni_file, "wt", encoding="utf8") as f: - f.write(new_data) + f.write(new_data) def main(): @@ -43,7 +43,7 @@ def main(): ) parser.add_argument( '--root', '-r', required=True, - help="Set the root used by the build (usually PW_ENVIRONMENT_ROOT)" + help="Set the root used by the build (usually CHIP_ROOT)" ) parser.add_argument('gni_file', type=str, help="GNI file to process") _make_paths_absolute(**vars(parser.parse_args())) From 81f666abb86e63c0620f40131fad8e3c13cb167a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Wed, 15 Nov 2023 09:17:20 -0500 Subject: [PATCH 4/4] Restyle --- examples/build_overrides/pigweed_environment.gni | 7 ++++--- scripts/setup/gni_make_paths_absolute.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/build_overrides/pigweed_environment.gni b/examples/build_overrides/pigweed_environment.gni index 670978063e72f1..538e35fca20efc 100644 --- a/examples/build_overrides/pigweed_environment.gni +++ b/examples/build_overrides/pigweed_environment.gni @@ -33,15 +33,16 @@ if (defined(pw_env_setup_CIPD_PIGWEED)) { _split_CIPD_PIGWEED = string_split(pw_env_setup_CIPD_PIGWEED, "//") if (_split_CIPD_PIGWEED[0] == "") { pw_env_setup_CIPD_PIGWEED = - get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PIGWEED}", "abspath") + get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PIGWEED}", + "abspath") } } if (defined(pw_env_setup_CIPD_PYTHON)) { _split_CIPD_PYTHON = string_split(pw_env_setup_CIPD_PYTHON, "//") if (_split_CIPD_PYTHON[0] == "") { pw_env_setup_CIPD_PYTHON = - get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PYTHON}", - "abspath") + get_path_info("${_bootstrap_root}/${pw_env_setup_CIPD_PYTHON}", + "abspath") } } if (defined(pw_env_setup_VIRTUAL_ENV)) { diff --git a/scripts/setup/gni_make_paths_absolute.py b/scripts/setup/gni_make_paths_absolute.py index 8c8699205a342d..17f03820458bba 100755 --- a/scripts/setup/gni_make_paths_absolute.py +++ b/scripts/setup/gni_make_paths_absolute.py @@ -15,6 +15,7 @@ # limitations under the License. import argparse + def _make_paths_absolute(gni_file: str, root: str): with open(gni_file, "rt", encoding="utf8") as f: data = f.read() @@ -33,7 +34,7 @@ def _make_paths_absolute(gni_file: str, root: str): return with open(gni_file, "wt", encoding="utf8") as f: - f.write(new_data) + f.write(new_data) def main():