Use Pigweed's new Python build system (#21202)

This change updates Matter to use Pigweed's new Python build system.
This reduces the number of pip installs performed during bootstrap and
builds to just a single pigweed package along with any others
specified in //BUILD.gn.

All python_actions performed duing a GN build do not rely on any pip
install of any in-tree python packages including Pigweed ones. Instead
build a venv is created in the out directory with only 3rd party deps
installed. All Python imports of in-tree packages are accomplished by
setting PYTHONPATH for each python_action. This allows scaling up the
number of in-tree Python packages without a significant impact to
build speed.

Bootstrap time is slightly reduced however all Pigweed Python modules
are installed allong with the Python packages from:
- //examples/common/pigweed/rpc_console
- //examples/chef
- //integrations/mobly

Before:
```
  Setting up Python environment.....done (1m34.7s)
```

After:
```
  Setting up Python environment.....done (1m14.7s)
```

.github/workflows/full-android.yaml

@@ -72,7 +72,7 @@ jobs:
             - name: Run Android build rule tests
               run: |
                   ./scripts/run_in_build_env.sh \
-                    "ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test.tests"
+                    "ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test"
             - name: Clean out build output
               run: rm -rf ./out
             # - name: Build Android Studio build (arm64 only)

.github/workflows/smoketest-android.yaml

@@ -67,4 +67,4 @@ jobs:
             - name: Run Android build rule tests
               run: |
                   ./scripts/run_in_build_env.sh \
-                    "ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test.tests"
+                    "ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test"

.gn

@@ -25,5 +25,13 @@ script_executable = "python3"
 
 default_args = {
   pw_unit_test_AUTOMATIC_RUNNER = "$dir_pigweed/targets/host/run_test"
-  pw_build_PIP_CONSTRAINTS = ["//scripts/constraints.txt"]
+
+  pw_build_PIP_CONSTRAINTS = [ "//scripts/constraints.txt" ]
+  pw_build_PIP_REQUIREMENTS = [ "//scripts/requirements.txt" ]
+
+  # Use the new Python build and merged 'pigweed' Python package.
+  pw_build_USE_NEW_PYTHON_BUILD = true
+
+  # GN target to use for the default Python build venv.
+  pw_build_PYTHON_BUILD_VENV = "//:matter_build_venv"
 }

BUILD.gn

@@ -23,6 +23,8 @@ import("//build_overrides/pigweed.gni")
 import("//src/lwip/lwip.gni")
 import("//src/platform/device.gni")
 import("$dir_pw_build/python.gni")
+import("$dir_pw_build/python_dist.gni")
+import("$dir_pw_build/python_venv.gni")
 
 # This build file should not be used in superproject builds.
 assert(chip_root == "//")
@@ -53,29 +55,85 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
     }
   }
 
-  # Python packages for supporting specific targets.
+  # Pigweed Python packages expected to be used in the :matter_build_venv
+  # target. If all packages are needed this list should match
+  # _pigweed_python_deps in:
+  # https://cs.opensource.google/pigweed/pigweed/+/master:pw_env_setup/BUILD.gn?q=_pigweed_python_deps
+  _pigweed_python_packages = [
+    "$dir_pw_allocator/py",
+    "$dir_pw_arduino_build/py",
+    "$dir_pw_bloat/py",
+    "$dir_pw_build/py",
+    "$dir_pw_build_info/py",
+    "$dir_pw_build_mcuxpresso/py",
+    "$dir_pw_cli/py",
+    "$dir_pw_console/py",
+    "$dir_pw_cpu_exception_cortex_m/py",
+    "$dir_pw_docgen/py",
+    "$dir_pw_doctor/py",
+    "$dir_pw_env_setup/py",
+    "$dir_pw_hdlc/py",
+    "$dir_pw_log:protos.python",
+    "$dir_pw_log_tokenized/py",
+    "$dir_pw_metric/py",
+    "$dir_pw_module/py",
+    "$dir_pw_package/py",
+    "$dir_pw_presubmit/py",
+    "$dir_pw_protobuf/py",
+    "$dir_pw_protobuf_compiler/py",
+    "$dir_pw_rpc/py",
+    "$dir_pw_snapshot/py:pw_snapshot",
+    "$dir_pw_snapshot/py:pw_snapshot_metadata",
+    "$dir_pw_software_update/py",
+    "$dir_pw_status/py",
+    "$dir_pw_stm32cube_build/py",
+    "$dir_pw_symbolizer/py",
+    "$dir_pw_system/py",
+    "$dir_pw_thread/py",
+    "$dir_pw_tls_client/py",
+    "$dir_pw_tokenizer/py",
+    "$dir_pw_toolchain/py",
+    "$dir_pw_trace/py",
+    "$dir_pw_trace_tokenized/py",
+    "$dir_pw_transfer/py",
+    "$dir_pw_unit_test/py",
+    "$dir_pw_watch/py",
+  ]
+
+  # Matter's in-tree pw_python_package or pw_create_python_source_tree targets.
+  _matter_python_packages = [
+    "//integrations/mobly:chip_mobly",
+    "//examples/chef",
+    "//examples/common/pigweed/rpc_console/py:chip_rpc",
+  ]
+
+  pw_python_venv("matter_build_venv") {
+    path = "$root_build_dir/python-venv"
+    constraints = pw_build_PIP_CONSTRAINTS
+    requirements = pw_build_PIP_REQUIREMENTS
+
+    # Packages available to import within GN's build venv.
+    source_packages = _matter_python_packages + _pigweed_python_packages
+  }
+
+  # Python packages installed during bootstrap.
   pw_python_group("python_packages") {
     python_deps = [
-      "$dir_pw_build/py",
-      "$dir_pw_doctor/py",
-      "$dir_pw_env_setup/py",
-      "$dir_pw_hdlc/py",
-      "$dir_pw_log:protos.python",
-      "$dir_pw_module/py",
-      "$dir_pw_protobuf/py",
-      "$dir_pw_protobuf_compiler/py",
-      "$dir_pw_rpc/py",
-      "$dir_pw_status/py",
-      "$dir_pw_toolchain/py",
-      "$dir_pw_trace/py",
-      "$dir_pw_trace_tokenized/py",
-      "$dir_pw_unit_test/py",
-      "$dir_pw_watch/py",
-      "integrations/mobly:chip_mobly",
-      "scripts:requirements",
+      ":pip_install_editable_matter_packages",
+      "$dir_pw_env_setup:pip_install_pigweed_package",
     ]
   }
 
+  # These pw_python_package targets will be installed using 'pip install --editable'
+  pw_internal_pip_install("pip_install_editable_matter_packages") {
+    packages = [
+      "//integrations/mobly:chip_mobly",
+      "//examples/chef",
+      "//examples/common/pigweed/rpc_console/py:chip_rpc",
+    ]
+    editable = true
+  }
+
   # This is a real toolchain. Build CHIP.
   group("default") {
     deps = [

build/chip/chip_test.gni

@@ -44,7 +44,11 @@ if (chip_link_tests) {
     pw_python_action(_test_name + "_run") {
       deps = [ ":${_test_name}" ]
       inputs = [ pw_unit_test_AUTOMATIC_RUNNER ]
-      script = "$dir_pw_unit_test/py/pw_unit_test/test_runner.py"
+      module = "pw_unit_test.test_runner"
+      python_deps = [
+        "$dir_pw_cli/py",
+        "$dir_pw_unit_test/py",
+      ]
       args = [
         "--runner",
         rebase_path(pw_unit_test_AUTOMATIC_RUNNER, root_build_dir),

build/chip/java/tests/BUILD.gn

@@ -17,7 +17,7 @@ import("//build_overrides/pigweed.gni")
 import("$dir_pw_build/python.gni")
 import("${chip_root}/build/chip/java/rules.gni")
 
-pw_python_script("java_build_test") {
+pw_python_action("java_build_test") {
   inputs = [
     "expected_output/child_library_2_expected.json",
     "expected_output/grandchild_library_expected.json",
@@ -26,8 +26,9 @@ pw_python_script("java_build_test") {
     "expected_output/child_prebuilt_expected.json",
     "expected_output/java_prebuilt_expected.json",
   ]
-  other_deps = [ ":java_library" ]
-  tests = [ "test.py" ]
+  deps = [ ":java_library" ]
+  script = "test.py"
+  stamp = true
 }
 
 java_library("java_library") {

build/chip/java/tests/expected_output/child_library_2_expected.json

@@ -1,10 +1,8 @@
 {
     "deps_info": {
         "name": "child_library_2.json",
-        "jar_path": "python/lib/build/chip/java/tests/child_library_2.jar",
-        "deps_configs": [
-            "python/gen/build/chip/java/tests/grandchild_library.json"
-        ],
-        "deps_jars": ["python/lib/build/chip/java/tests/grandchild_library.jar"]
+        "jar_path": "lib/build/chip/java/tests/child_library_2.jar",
+        "deps_configs": ["gen/build/chip/java/tests/grandchild_library.json"],
+        "deps_jars": ["lib/build/chip/java/tests/grandchild_library.jar"]
     }
 }

build/chip/java/tests/expected_output/child_library_expected.json

@@ -1,7 +1,7 @@
 {
     "deps_info": {
         "name": "child_library.json",
-        "jar_path": "python/lib/build/chip/java/tests/child_library.jar",
+        "jar_path": "lib/build/chip/java/tests/child_library.jar",
         "deps_configs": [],
         "deps_jars": []
     }

build/chip/java/tests/expected_output/child_prebuilt_expected.json

@@ -1,7 +1,7 @@
 {
     "deps_info": {
         "name": "child_prebuilt.json",
-        "jar_path": "python/lib/build/chip/java/tests/child_jar.jar",
+        "jar_path": "lib/build/chip/java/tests/child_jar.jar",
         "deps_configs": [],
         "deps_jars": []
     }

build/chip/java/tests/expected_output/grandchild_library_expected.json

@@ -1,7 +1,7 @@
 {
     "deps_info": {
         "name": "grandchild_library.json",
-        "jar_path": "python/lib/build/chip/java/tests/grandchild_library.jar",
+        "jar_path": "lib/build/chip/java/tests/grandchild_library.jar",
         "deps_configs": [],
         "deps_jars": []
     }

build/chip/java/tests/expected_output/java_library_expected.json

@@ -1,18 +1,18 @@
 {
     "deps_info": {
         "name": "java_library.json",
-        "jar_path": "python/lib/build/chip/java/tests/java_library.jar",
+        "jar_path": "lib/build/chip/java/tests/java_library.jar",
         "deps_configs": [
-            "python/gen/build/chip/java/tests/child_library.json",
-            "python/gen/build/chip/java/tests/child_library_2.json",
-            "python/gen/build/chip/java/tests/java_prebuilt.json"
+            "gen/build/chip/java/tests/child_library.json",
+            "gen/build/chip/java/tests/child_library_2.json",
+            "gen/build/chip/java/tests/java_prebuilt.json"
         ],
         "deps_jars": [
-            "python/lib/build/chip/java/tests/child_library.jar",
-            "python/lib/build/chip/java/tests/child_library_2.jar",
-            "python/lib/build/chip/java/tests/grandchild_library.jar",
-            "python/lib/build/chip/java/tests/prebuilt_jar.jar",
-            "python/lib/build/chip/java/tests/child_jar.jar"
+            "lib/build/chip/java/tests/child_library.jar",
+            "lib/build/chip/java/tests/child_library_2.jar",
+            "lib/build/chip/java/tests/grandchild_library.jar",
+            "lib/build/chip/java/tests/prebuilt_jar.jar",
+            "lib/build/chip/java/tests/child_jar.jar"
         ]
     }
 }

build/chip/java/tests/expected_output/java_prebuilt_expected.json

@@ -1,10 +1,8 @@
 {
     "deps_info": {
         "name": "java_prebuilt.json",
-        "jar_path": "python/lib/build/chip/java/tests/prebuilt_jar.jar",
-        "deps_configs": [
-            "python/gen/build/chip/java/tests/child_prebuilt.json"
-        ],
-        "deps_jars": ["python/lib/build/chip/java/tests/child_jar.jar"]
+        "jar_path": "lib/build/chip/java/tests/prebuilt_jar.jar",
+        "deps_configs": ["gen/build/chip/java/tests/child_prebuilt.json"],
+        "deps_jars": ["lib/build/chip/java/tests/child_jar.jar"]
     }
 }

build/chip/java/tests/test.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """
 Test for GN Java build rules. This test should be executed using ninja.
 """
@@ -29,8 +28,8 @@ class JavaBuildTest(unittest.TestCase):
     local_test_dir = '/build/chip/java/tests'
     test_dir = chip_root + local_test_dir
 
-    jars_dir = 'python/lib' + local_test_dir
-    configs_dir = 'python/gen' + local_test_dir
+    jars_dir = 'lib' + local_test_dir
+    configs_dir = 'gen' + local_test_dir
 
     # Target names in the BUILD.gn
     targets_to_check = [
@@ -39,10 +38,7 @@ class JavaBuildTest(unittest.TestCase):
         'child_library_2',
         'grandchild_library',
     ]
-    prebuilt_targets_to_check = [
-        'java_prebuilt',
-        'child_prebuilt'
-    ]
+    prebuilt_targets_to_check = ['java_prebuilt', 'child_prebuilt']
 
     def testExpectedJarsCreated(self):
         jars_dir = JavaBuildTest.jars_dir
@@ -58,18 +54,22 @@ def testBuildConfigMatchesExpected(self):
         configs_dir = JavaBuildTest.configs_dir
         expected_dir = JavaBuildTest.test_dir + '/expected_output'
 
-        for target in (JavaBuildTest.targets_to_check + JavaBuildTest.prebuilt_targets_to_check):
-            with open(expected_dir + '/' + target + '_expected.json', 'r') as expected_config, open(configs_dir + '/' + target + '.json', 'r') as actual_config:
+        for target in (JavaBuildTest.targets_to_check +
+                       JavaBuildTest.prebuilt_targets_to_check):
+            with open(expected_dir + '/' + target + '_expected.json',
+                      'r') as expected_config, open(
+                          configs_dir + '/' + target + '.json',
+                          'r') as actual_config:
                 expected_json = json.load(expected_config)['deps_info']
                 actual_json = json.load(actual_config)['deps_info']
 
                 self.assertEqual(expected_json['name'], actual_json['name'])
-                self.assertEqual(
-                    expected_json['jar_path'], actual_json['jar_path'])
-                self.assertCountEqual(
-                    expected_json['deps_configs'], actual_json['deps_configs'])
-                self.assertCountEqual(
-                    expected_json['deps_jars'], actual_json['deps_jars'])
+                self.assertEqual(expected_json['jar_path'],
+                                 actual_json['jar_path'])
+                self.assertCountEqual(expected_json['deps_configs'],
+                                      actual_json['deps_configs'])
+                self.assertCountEqual(expected_json['deps_jars'],
+                                      actual_json['deps_jars'])
 
 
 if __name__ == '__main__':

examples/build_overrides/pigweed_environment.gni

@@ -26,4 +26,4 @@ dir_cipd_pigweed =
 dir_cipd_python =
     get_path_info("${_bootstrap_root}/${dir_cipd_python}", "abspath")
 dir_virtual_env =
-    get_path_info("${_bootstrap_root}/${dir_virtual_env}", "abspath")
+    get_path_info("${_bootstrap_root}/${pw_env_setup_VIRTUAL_ENV}", "abspath")

examples/chef/BUILD.gn

@@ -19,7 +19,11 @@ import("//build_overrides/pigweed.gni")
 import("$dir_pw_build/python.gni")
 
 pw_python_package("chef") {
-  setup = [ "setup.py" ]
+  setup = [
+    "pyproject.toml",
+    "setup.cfg",
+    "setup.py",
+  ]
 
   inputs = [
     "sample_app_util/test_files/sample_zap_file.zap",

examples/chef/README.md

@@ -148,7 +148,7 @@ chef_$PLATFORM:
         - name: Checkout submodules
           run: scripts/checkout_submodules.py --shallow --platform $PLATFORM
         - name: Bootstrap
-          timeout-minutes: 10
+          timeout-minutes: 25
           run: scripts/build/gn_bootstrap.sh
         - name: CI Examples $PLATFORM
           shell: bash

examples/chef/chef.py

@@ -239,7 +239,8 @@ def bundle_esp32(device_name: str) -> None:
             shutil.copy(src_item, dest_item)
 
 
-def main(argv: Sequence[str]) -> None:
+def main() -> int:
+
     check_python_version()
     config = load_config()
     cicd_config = load_cicd_config()
@@ -340,7 +341,7 @@ def main(argv: Sequence[str]) -> None:
     parser.add_option(
         "", "--cpu_type", help="CPU type to compile for. Linux only.", choices=["arm64", "arm", "x64"])
 
-    options, _ = parser.parse_args(argv)
+    options, _ = parser.parse_args(sys.argv[1:])
 
     splash()
 
@@ -795,7 +796,8 @@ def main(argv: Sequence[str]) -> None:
                 shell.run_cmd(f"python3 -m chip_rpc.console --device {config['silabs-thread']['CU']} -b 115200")
 
     flush_print("Done")
+    return 0
 
 
 if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:]))
+    sys.exit(main())

examples/chef/pyproject.toml

@@ -0,0 +1,16 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[build-system]
+requires = ['setuptools', 'wheel']
+build-backend = 'setuptools.build_meta'