pw_build: Run all gn python_actions in a venv

This CL introduces a new gn arg that switches from the old behavior of
installing every pw_* Python package to installing a single 'pigweed'
Python package. This matches the 'pigweed' package available at
https://pypi.org/project/pigweed/ but with a higher version so pip
will always treat the version created in-tree as more recent.

Additionally all python_actions are forced to be run within an isolated
Python virtualenv created in the build_dir. This has a few benefits:

1. Greatly speeds up the build process as Python packages do not
   need to be pip installed before use.
2. Enforces Python dependency correctness. If a python_deps entry is
   missing in gn the build will fail.

At this time it is disabled by default. Set this arg in the .gn file:

  pw_build_USE_NEW_PYTHON_BUILD=true

Change-Id: I65b584727c66c1e7b2371ad1f8c57cd21d7df390
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/90060
Reviewed-by: Armando Montanez <[email protected]>
Reviewed-by: Wyatt Hepler <[email protected]>
Commit-Queue: Anthony DiGirolamo <[email protected]>

.gn

@@ -34,4 +34,7 @@ default_args = {
     # Code generated by third-party tool.
     "pw_tls_client/generate_test_data",
   ]
+
+  # Use the new Python build and merged 'pigweed' Python package.
+  pw_build_USE_NEW_PYTHON_BUILD = true
 }

pw_build/py/BUILD.gn

@@ -38,6 +38,7 @@ pw_python_package("py") {
     "pw_build/host_tool.py",
     "pw_build/mirror_tree.py",
     "pw_build/nop.py",
+    "pw_build/pip_install_python_deps.py",
     "pw_build/python_package.py",
     "pw_build/python_runner.py",
     "pw_build/python_wheels.py",

pw_build/py/pw_build/create_python_tree.py

@@ -24,11 +24,15 @@
 import tempfile
 from typing import Iterable
 
+import setuptools  # type: ignore
+
 try:
-    from pw_build.python_package import PythonPackage, load_packages
+    from pw_build.python_package import (PythonPackage, load_packages,
+                                         change_working_dir)
 except ImportError:
     # Load from python_package from this directory if pw_build is not available.
-    from python_package import PythonPackage, load_packages  # type: ignore
+    from python_package import (  # type: ignore
+        PythonPackage, load_packages, change_working_dir)
 
 
 def _parse_args():
@@ -203,6 +207,45 @@ def write_config(
     setup_cfg_file.write_text(comment_block_text + setup_cfg_text.getvalue())
 
 
+def setuptools_build_with_base(pkg: PythonPackage,
+                               build_base: Path,
+                               include_tests: bool = False) -> Path:
+    """Run setuptools build for this package."""
+
+    # If there is no setup_dir or setup_sources, just copy this packages
+    # source files.
+    if not pkg.setup_dir:
+        pkg.copy_sources_to(build_base)
+        return build_base
+    # Create the lib install dir in case it doesn't exist.
+    lib_dir_path = build_base / 'lib'
+    lib_dir_path.mkdir(parents=True, exist_ok=True)
+
+    starting_directory = Path.cwd()
+    # cd to the location of setup.py
+    with change_working_dir(pkg.setup_dir):
+        # Run build with temp build-base location
+        # Note: New files will be placed inside lib_dir_path
+        setuptools.setup(script_args=[
+            'build',
+            '--force',
+            '--build-base',
+            str(build_base),
+        ])
+
+        new_pkg_dir = lib_dir_path / pkg.package_name
+        # If tests should be included, copy them to the tests dir
+        if include_tests and pkg.tests:
+            test_dir_path = new_pkg_dir / 'tests'
+            test_dir_path.mkdir(parents=True, exist_ok=True)
+
+            for test_source_path in pkg.tests:
+                shutil.copy(starting_directory / test_source_path,
+                            test_dir_path)
+
+    return lib_dir_path
+
+
 def build_python_tree(python_packages: Iterable[PythonPackage],
                       tree_destination_dir: Path,
                       include_tests: bool = False) -> None:
@@ -219,8 +262,8 @@ def build_python_tree(python_packages: Iterable[PythonPackage],
         build_base = Path(build_base_name)
 
         for pkg in python_packages:
-            lib_dir_path = pkg.setuptools_build_with_base(
-                build_base, include_tests=include_tests)
+            lib_dir_path = setuptools_build_with_base(
+                pkg, build_base, include_tests=include_tests)
 
             # Move installed files from the temp build-base into
             # destination_path.

pw_build/py/pw_build/generate_python_package.py

@@ -107,10 +107,21 @@ def _collect_all_files(
 
     # Make sure there are __init__.py and py.typed files for each subpackage.
     for pkg in subpackages:
-        for file in (pkg / name for name in ['__init__.py', 'py.typed']):
-            if not file.exists():
-                file.touch()
-            files.append(file)
+        pytyped = pkg / 'py.typed'
+        if not pytyped.exists():
+            pytyped.touch()
+        files.append(pytyped)
+
+        # Create an __init__.py file if it doesn't already exist.
+        initpy = pkg / '__init__.py'
+        if not initpy.exists():
+            # Use pkgutil.extend_path to treat this as a namespaced package.
+            # This allows imports with the same name to live in multiple
+            # separate PYTHONPATH locations.
+            initpy.write_text(
+                'from pkgutil import extend_path\n'
+                '__path__ = extend_path(__path__, __name__)  # type: ignore\n')
+        files.append(initpy)
 
     pkg_data: Dict[str, Set[str]] = defaultdict(set)
 

pw_build/py/pw_build/pip_install_python_deps.py

@@ -0,0 +1,105 @@
+# Copyright 2022 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Pip install Pigweed Python packages."""
+
+import argparse
+from pathlib import Path
+import subprocess
+import sys
+from typing import List, Tuple
+
+try:
+    from pw_build.python_package import load_packages
+except ImportError:
+    # Load from python_package from this directory if pw_build is not available.
+    from python_package import load_packages  # type: ignore
+
+
+def _parse_args() -> Tuple[argparse.Namespace, List[str]]:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        '--python-dep-list-files',
+        type=Path,
+        required=True,
+        help=
+        'Path to a text file containing the list of Python package metadata '
+        'json files.',
+    )
+    parser.add_argument('--gn-packages',
+                        required=True,
+                        help=('Comma separated list of GN python package '
+                              'targets to install.'))
+    parser.add_argument('--editable-pip-install',
+                        action='store_true',
+                        help=('If true run the pip install command with the '
+                              '\'--editable\' option.'))
+    return parser.parse_known_args()
+
+
+class NoMatchingGnPythonDependency(Exception):
+    """An error occurred while processing a Python dependency."""
+
+
+def main(python_dep_list_files: Path, editable_pip_install: bool,
+         gn_targets: List[str], pip_args: List[str]) -> int:
+    """Find matching python packages to pip install."""
+    pip_target_dirs: List[str] = []
+
+    py_packages = load_packages([python_dep_list_files], ignore_missing=True)
+    for pkg in py_packages:
+        valid_target = [target in pkg.gn_target_name for target in gn_targets]
+        if not any(valid_target):
+            continue
+        top_level_source_dir = pkg.package_dir
+        pip_target_dirs.append(str(top_level_source_dir.parent.resolve()))
+
+    if not pip_target_dirs:
+        raise NoMatchingGnPythonDependency(
+            'No matching GN Python dependency found to install.\n'
+            'GN Targets to pip install:\n' + '\n'.join(gn_targets) + '\n\n'
+            'Declared Python Dependencies:\n' +
+            '\n'.join(pkg.gn_target_name for pkg in py_packages) + '\n\n')
+
+    for target in pip_target_dirs:
+        command_args = [sys.executable, "-m", "pip"]
+        command_args += pip_args
+        if editable_pip_install:
+            command_args.append('--editable')
+        command_args.append(target)
+
+        process = subprocess.run(command_args,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.STDOUT)
+        pip_output = process.stdout.decode()
+        if process.returncode != 0:
+            print(pip_output)
+            return process.returncode
+    return 0
+
+
+if __name__ == '__main__':
+    # Parse this script's args and pass any remaining args to pip.
+    argparse_args, remaining_args_for_pip = _parse_args()
+
+    # Split the comma separated string and remove leading slashes.
+    gn_target_names = [
+        target.lstrip('/') for target in argparse_args.gn_packages.split(',')
+        if target  # The last target may be an empty string.
+    ]
+
+    result = main(python_dep_list_files=argparse_args.python_dep_list_files,
+                  editable_pip_install=argparse_args.editable_pip_install,
+                  gn_targets=gn_target_names,
+                  pip_args=remaining_args_for_pip)
+    sys.exit(result)

pw_build/py/pw_build/python_package.py

@@ -24,8 +24,6 @@
 import shutil
 from typing import Dict, List, Optional, Iterable
 
-import setuptools  # type: ignore
-
 # List of known environment markers supported by pip.
 # https://peps.python.org/pep-0508/#environment-markers
 _PY_REQUIRE_ENVIRONMENT_MARKER_NAMES = [
@@ -176,64 +174,6 @@ def copy_sources_to(self, destination: Path) -> None:
         new_destination.mkdir(parents=True, exist_ok=True)
         shutil.copytree(self.package_dir, new_destination, dirs_exist_ok=True)
 
-    def setuptools_build_with_base(self,
-                                   build_base: Path,
-                                   include_tests: bool = False) -> Path:
-        """Run setuptools build for this package."""
-        # If there is no setup_dir or setup_sources, just copy this packages
-        # source files.
-        if not self.setup_dir:
-            self.copy_sources_to(build_base)
-            return build_base
-        # Create the lib install dir in case it doesn't exist.
-        lib_dir_path = build_base / 'lib'
-        lib_dir_path.mkdir(parents=True, exist_ok=True)
-
-        starting_directory = Path.cwd()
-        # cd to the location of setup.py
-        with change_working_dir(self.setup_dir):
-            # Run build with temp build-base location
-            # Note: New files will be placed inside lib_dir_path
-            setuptools.setup(script_args=[
-                'build',
-                '--force',
-                '--build-base',
-                str(build_base),
-            ])
-
-            new_pkg_dir = lib_dir_path / self.package_name
-            # If tests should be included, copy them to the tests dir
-            if include_tests and self.tests:
-                test_dir_path = new_pkg_dir / 'tests'
-                test_dir_path.mkdir(parents=True, exist_ok=True)
-
-                for test_source_path in self.tests:
-                    shutil.copy(starting_directory / test_source_path,
-                                test_dir_path)
-
-        return lib_dir_path
-
-    def setuptools_develop(self, no_deps=False) -> None:
-        if not self.setup_dir:
-            raise MissingSetupSources(
-                'Cannot find setup source file root folder (the location of '
-                f'setup.cfg) for the Python library/package: {self}')
-
-        with change_working_dir(self.setup_dir):
-            develop_args = ['develop']
-            if no_deps:
-                develop_args.append('--no-deps')
-            setuptools.setup(script_args=develop_args)
-
-    def setuptools_install(self) -> None:
-        if not self.setup_dir:
-            raise MissingSetupSources(
-                'Cannot find setup source file root folder (the location of '
-                f'setup.cfg) for the Python library/package: {self}')
-
-        with change_working_dir(self.setup_dir):
-            setuptools.setup(script_args=['install'])
-
     def install_requires_entries(self) -> List[str]:
         """Convert the install_requires entry into a list of strings."""
         this_requires: List[str] = []