From dec2b24cd1207dc2e05fe20855cfdc3cd575ebf8 Mon Sep 17 00:00:00 2001 From: Anthony DiGirolamo Date: Fri, 22 Apr 2022 16:39:37 -0700 Subject: [PATCH] pw_build: Run all gn python_actions in a venv This CL introduces a new gn arg that switches from the old behavior of installing every pw_* Python package to installing a single 'pigweed' Python package. This matches the 'pigweed' package available at https://pypi.org/project/pigweed/ but with a higher version so pip will always treat the version created in-tree as more recent. Additionally all python_actions are forced to be run within an isolated Python virtualenv created in the build_dir. This has a few benefits: 1. Greatly speeds up the build process as Python packages do not need to be pip installed before use. 2. Enforces Python dependency correctness. If a python_deps entry is missing in gn the build will fail. At this time it is disabled by default. Set this arg in the .gn file: pw_build_USE_NEW_PYTHON_BUILD=true Change-Id: I65b584727c66c1e7b2371ad1f8c57cd21d7df390 Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/90060 Reviewed-by: Armando Montanez Reviewed-by: Wyatt Hepler Commit-Queue: Anthony DiGirolamo --- .gn | 3 + pw_build/py/BUILD.gn | 1 + pw_build/py/pw_build/create_python_tree.py | 51 ++++- .../py/pw_build/generate_python_package.py | 19 +- .../py/pw_build/pip_install_python_deps.py | 105 +++++++++ pw_build/py/pw_build/python_package.py | 60 ------ pw_build/py/pw_build/python_runner.py | 113 +++++++++- pw_build/python.gni | 59 ++++- pw_build/python.rst | 8 +- pw_build/python_action.gni | 137 ++++++++++-- pw_build/python_dist.gni | 204 +++++++++++++++--- pw_build/python_gn_args.gni | 24 +++ pw_build/update_bundle.gni | 1 + pw_console/py/command_runner_test.py | 10 +- pw_console/py/console_app_test.py | 19 +- pw_console/py/pw_console/console_prefs.py | 3 + pw_console/py/repl_pane_test.py | 8 +- pw_console/py/setup.cfg | 6 +- pw_console/py/window_manager_test.py | 9 +- pw_env_setup/BUILD.gn | 93 +++++++- pw_env_setup/create_gn_venv.py | 38 ++++ .../virtualenv_setup/constraint.list | 5 +- pw_rpc/py/BUILD.gn | 1 + pw_rpc/py/pw_rpc/__init__.py | 5 + targets/lm3s6965evb_qemu/py/setup.cfg | 3 +- targets/stm32f429i_disc1/py/setup.cfg | 4 +- 26 files changed, 821 insertions(+), 168 deletions(-) create mode 100644 pw_build/py/pw_build/pip_install_python_deps.py create mode 100644 pw_build/python_gn_args.gni create mode 100644 pw_env_setup/create_gn_venv.py diff --git a/.gn b/.gn index fc3984a3dd..5916cfb1a3 100644 --- a/.gn +++ b/.gn @@ -34,4 +34,7 @@ default_args = { # Code generated by third-party tool. "pw_tls_client/generate_test_data", ] + + # Use the new Python build and merged 'pigweed' Python package. + pw_build_USE_NEW_PYTHON_BUILD = true } diff --git a/pw_build/py/BUILD.gn b/pw_build/py/BUILD.gn index 5341a92b92..7af6e2bbed 100644 --- a/pw_build/py/BUILD.gn +++ b/pw_build/py/BUILD.gn @@ -38,6 +38,7 @@ pw_python_package("py") { "pw_build/host_tool.py", "pw_build/mirror_tree.py", "pw_build/nop.py", + "pw_build/pip_install_python_deps.py", "pw_build/python_package.py", "pw_build/python_runner.py", "pw_build/python_wheels.py", diff --git a/pw_build/py/pw_build/create_python_tree.py b/pw_build/py/pw_build/create_python_tree.py index ae1dab6ee0..ca641d5c23 100644 --- a/pw_build/py/pw_build/create_python_tree.py +++ b/pw_build/py/pw_build/create_python_tree.py @@ -24,11 +24,15 @@ import tempfile from typing import Iterable +import setuptools # type: ignore + try: - from pw_build.python_package import PythonPackage, load_packages + from pw_build.python_package import (PythonPackage, load_packages, + change_working_dir) except ImportError: # Load from python_package from this directory if pw_build is not available. - from python_package import PythonPackage, load_packages # type: ignore + from python_package import ( # type: ignore + PythonPackage, load_packages, change_working_dir) def _parse_args(): @@ -203,6 +207,45 @@ def write_config( setup_cfg_file.write_text(comment_block_text + setup_cfg_text.getvalue()) +def setuptools_build_with_base(pkg: PythonPackage, + build_base: Path, + include_tests: bool = False) -> Path: + """Run setuptools build for this package.""" + + # If there is no setup_dir or setup_sources, just copy this packages + # source files. + if not pkg.setup_dir: + pkg.copy_sources_to(build_base) + return build_base + # Create the lib install dir in case it doesn't exist. + lib_dir_path = build_base / 'lib' + lib_dir_path.mkdir(parents=True, exist_ok=True) + + starting_directory = Path.cwd() + # cd to the location of setup.py + with change_working_dir(pkg.setup_dir): + # Run build with temp build-base location + # Note: New files will be placed inside lib_dir_path + setuptools.setup(script_args=[ + 'build', + '--force', + '--build-base', + str(build_base), + ]) + + new_pkg_dir = lib_dir_path / pkg.package_name + # If tests should be included, copy them to the tests dir + if include_tests and pkg.tests: + test_dir_path = new_pkg_dir / 'tests' + test_dir_path.mkdir(parents=True, exist_ok=True) + + for test_source_path in pkg.tests: + shutil.copy(starting_directory / test_source_path, + test_dir_path) + + return lib_dir_path + + def build_python_tree(python_packages: Iterable[PythonPackage], tree_destination_dir: Path, include_tests: bool = False) -> None: @@ -219,8 +262,8 @@ def build_python_tree(python_packages: Iterable[PythonPackage], build_base = Path(build_base_name) for pkg in python_packages: - lib_dir_path = pkg.setuptools_build_with_base( - build_base, include_tests=include_tests) + lib_dir_path = setuptools_build_with_base( + pkg, build_base, include_tests=include_tests) # Move installed files from the temp build-base into # destination_path. diff --git a/pw_build/py/pw_build/generate_python_package.py b/pw_build/py/pw_build/generate_python_package.py index d93cf8cc44..60fb98aeb8 100644 --- a/pw_build/py/pw_build/generate_python_package.py +++ b/pw_build/py/pw_build/generate_python_package.py @@ -107,10 +107,21 @@ def _collect_all_files( # Make sure there are __init__.py and py.typed files for each subpackage. for pkg in subpackages: - for file in (pkg / name for name in ['__init__.py', 'py.typed']): - if not file.exists(): - file.touch() - files.append(file) + pytyped = pkg / 'py.typed' + if not pytyped.exists(): + pytyped.touch() + files.append(pytyped) + + # Create an __init__.py file if it doesn't already exist. + initpy = pkg / '__init__.py' + if not initpy.exists(): + # Use pkgutil.extend_path to treat this as a namespaced package. + # This allows imports with the same name to live in multiple + # separate PYTHONPATH locations. + initpy.write_text( + 'from pkgutil import extend_path\n' + '__path__ = extend_path(__path__, __name__) # type: ignore\n') + files.append(initpy) pkg_data: Dict[str, Set[str]] = defaultdict(set) diff --git a/pw_build/py/pw_build/pip_install_python_deps.py b/pw_build/py/pw_build/pip_install_python_deps.py new file mode 100644 index 0000000000..d35154cd7c --- /dev/null +++ b/pw_build/py/pw_build/pip_install_python_deps.py @@ -0,0 +1,105 @@ +# Copyright 2022 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Pip install Pigweed Python packages.""" + +import argparse +from pathlib import Path +import subprocess +import sys +from typing import List, Tuple + +try: + from pw_build.python_package import load_packages +except ImportError: + # Load from python_package from this directory if pw_build is not available. + from python_package import load_packages # type: ignore + + +def _parse_args() -> Tuple[argparse.Namespace, List[str]]: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--python-dep-list-files', + type=Path, + required=True, + help= + 'Path to a text file containing the list of Python package metadata ' + 'json files.', + ) + parser.add_argument('--gn-packages', + required=True, + help=('Comma separated list of GN python package ' + 'targets to install.')) + parser.add_argument('--editable-pip-install', + action='store_true', + help=('If true run the pip install command with the ' + '\'--editable\' option.')) + return parser.parse_known_args() + + +class NoMatchingGnPythonDependency(Exception): + """An error occurred while processing a Python dependency.""" + + +def main(python_dep_list_files: Path, editable_pip_install: bool, + gn_targets: List[str], pip_args: List[str]) -> int: + """Find matching python packages to pip install.""" + pip_target_dirs: List[str] = [] + + py_packages = load_packages([python_dep_list_files], ignore_missing=True) + for pkg in py_packages: + valid_target = [target in pkg.gn_target_name for target in gn_targets] + if not any(valid_target): + continue + top_level_source_dir = pkg.package_dir + pip_target_dirs.append(str(top_level_source_dir.parent.resolve())) + + if not pip_target_dirs: + raise NoMatchingGnPythonDependency( + 'No matching GN Python dependency found to install.\n' + 'GN Targets to pip install:\n' + '\n'.join(gn_targets) + '\n\n' + 'Declared Python Dependencies:\n' + + '\n'.join(pkg.gn_target_name for pkg in py_packages) + '\n\n') + + for target in pip_target_dirs: + command_args = [sys.executable, "-m", "pip"] + command_args += pip_args + if editable_pip_install: + command_args.append('--editable') + command_args.append(target) + + process = subprocess.run(command_args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + pip_output = process.stdout.decode() + if process.returncode != 0: + print(pip_output) + return process.returncode + return 0 + + +if __name__ == '__main__': + # Parse this script's args and pass any remaining args to pip. + argparse_args, remaining_args_for_pip = _parse_args() + + # Split the comma separated string and remove leading slashes. + gn_target_names = [ + target.lstrip('/') for target in argparse_args.gn_packages.split(',') + if target # The last target may be an empty string. + ] + + result = main(python_dep_list_files=argparse_args.python_dep_list_files, + editable_pip_install=argparse_args.editable_pip_install, + gn_targets=gn_target_names, + pip_args=remaining_args_for_pip) + sys.exit(result) diff --git a/pw_build/py/pw_build/python_package.py b/pw_build/py/pw_build/python_package.py index ccca1a34cf..226d3f3062 100644 --- a/pw_build/py/pw_build/python_package.py +++ b/pw_build/py/pw_build/python_package.py @@ -24,8 +24,6 @@ import shutil from typing import Dict, List, Optional, Iterable -import setuptools # type: ignore - # List of known environment markers supported by pip. # https://peps.python.org/pep-0508/#environment-markers _PY_REQUIRE_ENVIRONMENT_MARKER_NAMES = [ @@ -176,64 +174,6 @@ def copy_sources_to(self, destination: Path) -> None: new_destination.mkdir(parents=True, exist_ok=True) shutil.copytree(self.package_dir, new_destination, dirs_exist_ok=True) - def setuptools_build_with_base(self, - build_base: Path, - include_tests: bool = False) -> Path: - """Run setuptools build for this package.""" - # If there is no setup_dir or setup_sources, just copy this packages - # source files. - if not self.setup_dir: - self.copy_sources_to(build_base) - return build_base - # Create the lib install dir in case it doesn't exist. - lib_dir_path = build_base / 'lib' - lib_dir_path.mkdir(parents=True, exist_ok=True) - - starting_directory = Path.cwd() - # cd to the location of setup.py - with change_working_dir(self.setup_dir): - # Run build with temp build-base location - # Note: New files will be placed inside lib_dir_path - setuptools.setup(script_args=[ - 'build', - '--force', - '--build-base', - str(build_base), - ]) - - new_pkg_dir = lib_dir_path / self.package_name - # If tests should be included, copy them to the tests dir - if include_tests and self.tests: - test_dir_path = new_pkg_dir / 'tests' - test_dir_path.mkdir(parents=True, exist_ok=True) - - for test_source_path in self.tests: - shutil.copy(starting_directory / test_source_path, - test_dir_path) - - return lib_dir_path - - def setuptools_develop(self, no_deps=False) -> None: - if not self.setup_dir: - raise MissingSetupSources( - 'Cannot find setup source file root folder (the location of ' - f'setup.cfg) for the Python library/package: {self}') - - with change_working_dir(self.setup_dir): - develop_args = ['develop'] - if no_deps: - develop_args.append('--no-deps') - setuptools.setup(script_args=develop_args) - - def setuptools_install(self) -> None: - if not self.setup_dir: - raise MissingSetupSources( - 'Cannot find setup source file root folder (the location of ' - f'setup.cfg) for the Python library/package: {self}') - - with change_working_dir(self.setup_dir): - setuptools.setup(script_args=['install']) - def install_requires_entries(self) -> List[str]: """Convert the install_requires entry into a list of strings.""" this_requires: List[str] = [] diff --git a/pw_build/py/pw_build/python_runner.py b/pw_build/py/pw_build/python_runner.py index 41c15904af..451b82291d 100755 --- a/pw_build/py/pw_build/python_runner.py +++ b/pw_build/py/pw_build/python_runner.py @@ -32,6 +32,12 @@ from typing import Callable, Dict, Iterable, Iterator, List, NamedTuple from typing import Optional, Tuple +try: + from pw_build.python_package import load_packages +except ImportError: + # Load from python_package from this directory if pw_build is not available. + from python_package import load_packages # type: ignore + if sys.platform != 'win32': import fcntl # pylint: disable=import-error # TODO(b/227670947): Support Windows. @@ -78,6 +84,23 @@ def _parse_args() -> argparse.Namespace: type=Path, help='Change to this working directory before running the subcommand', ) + parser.add_argument( + '--python-dep-list-files', + nargs='+', + type=Path, + help='Paths to text files containing lists of Python package metadata ' + 'json files.', + ) + parser.add_argument( + '--python-interpreter', + type=Path, + help='Python interpreter to use for this action.', + ) + parser.add_argument( + '--python-virtualenv', + type=Path, + help='Path to a virtualenv to use for this action.', + ) parser.add_argument( 'original_cmd', nargs=argparse.REMAINDER, @@ -86,7 +109,6 @@ def _parse_args() -> argparse.Namespace: parser.add_argument( '--lockfile', type=Path, - required=True, help=('Path to a pip lockfile. Any pip execution will acquire an ' 'exclusive lock on it, any other module a shared lock.')) return parser.parse_args() @@ -507,7 +529,11 @@ def cleanup(): f"Failed to acquire lock {lockfile} in {_LOCK_ACQUISITION_TIMEOUT}") -def main( # pylint: disable=too-many-arguments +class MissingPythonDependency(Exception): + """An error occurred while processing a Python dependency.""" + + +def main( # pylint: disable=too-many-arguments,too-many-branches,too-many-locals gn_root: Path, current_path: Path, original_cmd: List[str], @@ -515,13 +541,34 @@ def main( # pylint: disable=too-many-arguments current_toolchain: str, module: Optional[str], env: Optional[List[str]], + python_dep_list_files: List[Path], + python_interpreter: Optional[Path], + python_virtualenv: Optional[Path], capture_output: bool, touch: Optional[Path], working_directory: Optional[Path], - lockfile: Path, + lockfile: Optional[Path], ) -> int: """Script entry point.""" + python_paths_list = [] + if python_dep_list_files: + py_packages = load_packages( + python_dep_list_files, + # If this python_action has no gn python_deps this file will be + # empty. + ignore_missing=True) + + for pkg in py_packages: + top_level_source_dir = pkg.package_dir + if not top_level_source_dir: + raise MissingPythonDependency( + 'Unable to find top level source dir for the Python ' + f'package "{pkg}"') + python_paths_list.append(top_level_source_dir.parent.resolve()) + # Sort the PYTHONPATH list, it will be in a different order each build. + python_paths_list = sorted(python_paths_list) + if not original_cmd or original_cmd[0] != '--': _LOG.error('%s requires a command to run', sys.argv[0]) return 1 @@ -536,23 +583,70 @@ def main( # pylint: disable=too-many-arguments toolchain=tool) command = [sys.executable] + if python_interpreter is not None: + command = [str(root_build_dir / python_interpreter)] if module is not None: command += ['-m', module] run_args: dict = dict() + # Always inherit the environtment by default. If PYTHONPATH or VIRTUALENV is + # set below then the environment vars must be copied in or subprocess.run + # will run with only the new updated variables. + run_args['env'] = os.environ.copy() if env is not None: environment = os.environ.copy() environment.update((k, v) for k, v in (a.split('=', 1) for a in env)) run_args['env'] = environment + script_command = original_cmd[0] + if script_command == '--': + script_command = original_cmd[1] + + is_pip_command = (module == 'pip' + or 'pip_install_python_deps.py' in script_command) + + if python_paths_list and not is_pip_command: + existing_env = (run_args['env'] + if 'env' in run_args else os.environ.copy()) + + python_path_prepend = os.pathsep.join( + str(p) for p in set(python_paths_list)) + + # Append the existing PYTHONPATH to the new one. + new_python_path = os.pathsep.join( + path_str for path_str in + [python_path_prepend, + existing_env.get('PYTHONPATH', '')] if path_str) + new_env = { + 'PYTHONPATH': new_python_path, + # mypy doesn't use PYTHONPATH for analyzing imports so module + # directories must be added to the MYPYPATH environment variable. + 'MYPYPATH': new_python_path, + } + # print('PYTHONPATH') + # for ppath in python_paths_list: + # print(str(ppath)) + + if python_virtualenv: + new_env['VIRTUAL_ENV'] = str(root_build_dir / python_virtualenv) + new_env['PATH'] = os.pathsep.join([ + str(root_build_dir / python_virtualenv / 'bin'), + existing_env.get('PATH', '') + ]) + + if 'env' not in run_args: + run_args['env'] = {} + run_args['env'].update(new_env) + if capture_output: # Combine stdout and stderr so that error messages are correctly # interleaved with the rest of the output. run_args['stdout'] = subprocess.PIPE run_args['stderr'] = subprocess.STDOUT + # Build the command to run. try: for arg in original_cmd[1:]: command += expand_expressions(paths, arg) @@ -563,11 +657,14 @@ def main( # pylint: disable=too-many-arguments if working_directory: run_args['cwd'] = working_directory - try: - acquire_lock(lockfile, module == 'pip') - except LockAcquisitionTimeoutError as exception: - _LOG.error('%s', exception) - return 1 + # TODO(pwbug/666): Deprecate the --lockfile option as part of the Python GN + # template refactor. + if lockfile: + try: + acquire_lock(lockfile, is_pip_command) + except LockAcquisitionTimeoutError as exception: + _LOG.error('%s', exception) + return 1 _LOG.debug('RUN %s', ' '.join(shlex.quote(arg) for arg in command)) diff --git a/pw_build/python.gni b/pw_build/python.gni index 3b069fd1a4..0b944638c5 100644 --- a/pw_build/python.gni +++ b/pw_build/python.gni @@ -17,6 +17,7 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_build/input_group.gni") import("$dir_pw_build/mirror_tree.gni") import("$dir_pw_build/python_action.gni") +import("$dir_pw_build/python_gn_args.gni") import("$dir_pw_protobuf_compiler/toolchain.gni") declare_args() { @@ -89,8 +90,15 @@ template("_pw_python_static_analysis_mypy") { deps = invoker.deps - foreach(dep, invoker.python_deps) { - deps += [ string_replace(dep, "(", ".lint.mypy(") ] + if (defined(invoker.python_deps)) { + python_deps = [] + foreach(dep, invoker.python_deps) { + deps += [ string_replace(dep, "(", ".lint.mypy(") ] + python_deps += [ dep ] + } + } + if (defined(invoker.python_metadata_deps)) { + python_metadata_deps = invoker.python_metadata_deps } } } @@ -123,8 +131,15 @@ template("_pw_python_static_analysis_pylint") { public_deps = invoker.deps - foreach(dep, invoker.python_deps) { - public_deps += [ string_replace(dep, "(", ".lint.pylint(") ] + if (defined(invoker.python_deps)) { + python_deps = [] + foreach(dep, invoker.python_deps) { + public_deps += [ string_replace(dep, "(", ".lint.pylint(") ] + python_deps += [ dep ] + } + } + if (defined(invoker.python_metadata_deps)) { + python_metadata_deps = invoker.python_metadata_deps } } } @@ -191,6 +206,20 @@ template("pw_python_package") { _is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone) _generate_package = false + if (_is_package) { + _pip_install_package = true + + # Always run pip installs if not opting into the new Pigweed Python Build. + if (pw_build_USE_NEW_PYTHON_BUILD) { + _pip_install_package = false + } + } + + _pydeplabel = get_label_info(":$target_name", "label_with_toolchain") + + # If a package does not run static analysis or if it does but doesn't have + # any tests then this variable is not used. + not_needed([ "_pydeplabel" ]) # Check the generate_setup and import_protos args to determine if this package # is generated. @@ -461,7 +490,7 @@ template("pw_python_package") { } } - if (_is_package) { + if (_is_package && _pip_install_package) { # Install this Python package and its dependencies in the current Python # environment using pip. pw_python_action("$target_name._run_pip_install") { @@ -589,8 +618,10 @@ template("pw_python_package") { if (_static_analysis != [] || _test_sources != []) { # All packages to install for either general use or test running. _test_install_deps = [ ":$target_name.install" ] + foreach(dep, _python_test_deps + [ "$dir_pw_build:python_lint" ]) { _test_install_deps += [ string_replace(dep, "(", ".install(") ] + _test_install_deps += [ dep ] } } @@ -602,7 +633,11 @@ template("pw_python_package") { target("_pw_python_static_analysis_$_tool", "$target_name.lint.$_tool") { sources = _all_py_files deps = _test_install_deps - python_deps = _python_deps + python_deps = _python_deps + _python_test_deps + + if (_is_package) { + python_metadata_deps = [ _pydeplabel ] + } _optional_variables = [ "mypy_ini", @@ -694,6 +729,16 @@ template("pw_python_package") { stamp = true + # Make sure the python test deps are added to the PYTHONPATH. + python_metadata_deps = _test_install_deps + + # If this is a test for a package, add it to PYTHONPATH as well. This is + # required if the test source file isn't in the same directory as the + # folder containing the package sources to allow local Python imports. + if (_is_package) { + python_metadata_deps += [ _pydeplabel ] + } + deps = _test_install_deps foreach(dep, _python_test_deps) { @@ -775,6 +820,8 @@ template("pw_python_script") { "sources", "tests", "python_deps", + "python_test_deps", + "python_metadata_deps", "other_deps", "inputs", "pylintrc", diff --git a/pw_build/python.rst b/pw_build/python.rst index 07f7a1fd4e..741353af7e 100644 --- a/pw_build/python.rst +++ b/pw_build/python.rst @@ -289,16 +289,14 @@ Arguments append_date_to_version = true } -Using this template will create additional targets for installing and building a -Python wheel. For example if you define ``pw_create_python_source_tree("awesome")`` -the 3 resulting targets that get created will be: +Using this template will create an additional target for and building a Python +wheel. For example if you define ``pw_create_python_source_tree("awesome")`` the +resulting targets that get created will be: - ``awesome`` - This will create the merged package with all source files in place in the out directory under ``out/obj/awesome/``. - ``awesome.wheel`` - This builds a Python wheel from the above source files under ``out/obj/awesome._build_wheel/awesome*.whl``. -- ``awesome.install`` - This pip installs the merged package into the user's - development environment. Example ------- diff --git a/pw_build/python_action.gni b/pw_build/python_action.gni index 7fc160f77d..4029dafe15 100644 --- a/pw_build/python_action.gni +++ b/pw_build/python_action.gni @@ -14,6 +14,8 @@ import("//build_overrides/pigweed.gni") +import("$dir_pw_build/python_gn_args.gni") + # Defines an action that runs a Python script. # # This wraps a regular Python script GN action with an invocation of a script- @@ -65,16 +67,12 @@ template("pw_python_action") { "--current-path", rebase_path(".", root_build_dir), - # pip lockfile, prevents pip from running in parallel with other Python - # actions. - "--lockfile", - - # Use a single lockfile for the entire out directory. - "pip.lock", "--default-toolchain=$default_toolchain", "--current-toolchain=$current_toolchain", ] + _use_build_dir_virtualenv = true + if (defined(invoker.environment)) { foreach(variable, invoker.environment) { _script_args += [ "--env=$variable" ] @@ -124,6 +122,17 @@ template("pw_python_action") { "--module", invoker.module, ] + + # Pip installs should only ever need to occur in the Pigweed + # environment. For these actions do not use the build_dir virtualenv. + if (invoker.module == "pip") { + _use_build_dir_virtualenv = false + } + } + + # Override to force using or not using the venv. + if (defined(invoker._pw_internal_run_in_venv)) { + _use_build_dir_virtualenv = invoker._pw_internal_run_in_venv } if (defined(invoker.working_directory)) { @@ -133,19 +142,6 @@ template("pw_python_action") { ] } - # "--" indicates the end of arguments to the runner script. - # Everything beyond this point is interpreted as the command and arguments - # of the Python script to run. - _script_args += [ "--" ] - - if (defined(invoker.script)) { - _script_args += [ rebase_path(invoker.script, root_build_dir) ] - } - - if (defined(invoker.args)) { - _script_args += invoker.args - } - if (defined(invoker._pw_action_type)) { _action_type = invoker._pw_action_type } else { @@ -158,15 +154,115 @@ template("pw_python_action") { _deps = [] } + _py_metadata_deps = [] + if (defined(invoker.python_deps)) { foreach(dep, invoker.python_deps) { _deps += [ get_label_info(dep, "label_no_toolchain") + ".install(" + get_label_info(dep, "toolchain") + ")" ] + _py_metadata_deps += [ get_label_info(dep, "label_no_toolchain") + + "($dir_pw_build/python_toolchain:python)" ] } # Add the base target as a dep so the action reruns when any source files # change, even if the package does not have to be reinstalled. _deps += invoker.python_deps + _deps += _py_metadata_deps + } + + _extra_python_metadata_deps = [] + if (defined(invoker.python_metadata_deps)) { + foreach(dep, invoker.python_metadata_deps) { + _extra_python_metadata_deps += + [ get_label_info(dep, "label_no_toolchain") + + "($dir_pw_build/python_toolchain:python)" ] + } + } + + _metadata_path_list_file = + "${target_gen_dir}/${target_name}_metadata_path_list.txt" + + # Build a list of relative paths containing all the python + # package_metadata.json files we depend on. + _metadata_path_list_target = "${target_name}._metadata_path_list.txt" + generated_file(_metadata_path_list_target) { + data_keys = [ "pw_python_package_metadata_json" ] + rebase = root_build_dir + deps = _py_metadata_deps + _extra_python_metadata_deps + outputs = [ _metadata_path_list_file ] + } + _deps += [ ":${_metadata_path_list_target}" ] + + if (pw_build_USE_NEW_PYTHON_BUILD) { + _script_args += [ + "--python-dep-list-files", + rebase_path(_metadata_path_list_file, root_build_dir), + ] + + # Set venv options if needed. + if (_use_build_dir_virtualenv) { + _script_args += [ "--python-interpreter" ] + + if (host_os == "win") { + _script_args += [ "python-venv/Scripts/python.exe" ] + } else { + _script_args += [ "python-venv/bin/python" ] + } + + _script_args += [ + "--python-virtualenv", + "python-venv", + ] + } + } + + if (!pw_build_USE_NEW_PYTHON_BUILD) { + _script_args += [ + # pip lockfile, prevents pip from running in parallel with other Python + # actions. + "--lockfile", + + # Use a single lockfile for the entire out directory. + "pip.lock", + ] + } + + # "--" indicates the end of arguments to the runner script. + # Everything beyond this point is interpreted as the command and arguments + # of the Python script to run. + _script_args += [ "--" ] + + if (defined(invoker.script)) { + _script_args += [ rebase_path(invoker.script, root_build_dir) ] + } + + _forward_python_metadata_deps = false + if (defined(invoker._forward_python_metadata_deps)) { + _forward_python_metadata_deps = true + } + if (_forward_python_metadata_deps) { + _script_args += [ + "--python-dep-list-files", + rebase_path(_metadata_path_list_file, root_build_dir), + ] + } + + if (defined(invoker.args)) { + _script_args += invoker.args + } + + # Assume third party PyPI deps should be available in the build_dir virtualenv. + _install_venv_3p_deps = true + if (!_use_build_dir_virtualenv || + (defined(invoker.skip_installing_external_python_deps) && + invoker.skip_installing_external_python_deps)) { + _install_venv_3p_deps = false + } + + # Check that script or module is a present and not a no-op. + _run_script_or_module = false + if (defined(invoker.script) || defined(invoker.module)) { + _run_script_or_module = true } target(_action_type, target_name) { @@ -184,6 +280,9 @@ template("pw_python_action") { inputs = _inputs outputs = _outputs deps = _deps + if (_install_venv_3p_deps && _run_script_or_module) { + deps += [ "$dir_pw_env_setup:install_3p_deps($default_toolchain)" ] + } } } diff --git a/pw_build/python_dist.gni b/pw_build/python_dist.gni index a40e310eeb..9e8dfeba04 100644 --- a/pw_build/python_dist.gni +++ b/pw_build/python_dist.gni @@ -14,8 +14,10 @@ import("//build_overrides/pigweed.gni") +import("$dir_pw_build/error.gni") import("$dir_pw_build/python.gni") import("$dir_pw_build/python_action.gni") +import("$dir_pw_build/python_gn_args.gni") import("$dir_pw_build/zip.gni") # Builds a directory containing a collection of Python wheels. @@ -65,6 +67,7 @@ template("pw_python_wheels") { forward_variables_from(invoker, [ "public_deps" ]) deps = _deps + [ ":$target_name._wheel_paths" ] module = "pw_build.collect_wheels" + python_deps = [ "$dir_pw_build/py" ] args = [ "--prefix", @@ -207,6 +210,44 @@ template("pw_create_python_source_tree") { _public_deps += invoker.public_deps } + # Set source files for the Python package metadata json file. + _sources = [] + _setup_sources = [ + "$_output_dir/pyproject.toml", + "$_output_dir/setup.cfg", + ] + _test_sources = [] + + # Create the Python package_metadata.json file so this can be used as a + # Python dependency. + _package_metadata_json_file = + "$target_gen_dir/$target_name/package_metadata.json" + + # Get Python package metadata and write to disk as JSON. + _package_metadata = { + gn_target_name = + get_label_info(":${invoker.target_name}", "label_no_toolchain") + + # Get package source files + sources = rebase_path(_sources, root_build_dir) + + # Get setup.cfg, pyproject.toml, or setup.py file + setup_sources = rebase_path(_setup_sources, root_build_dir) + + # Get test source files + tests = rebase_path(_test_sources, root_build_dir) + + # Get package input files (package data) + inputs = [] + if (defined(invoker.inputs)) { + inputs = rebase_path(invoker.inputs, root_build_dir) + } + inputs += rebase_path(_extra_file_inputs, root_build_dir) + } + + # Finally, write out the json + write_file(_package_metadata_json_file, _package_metadata, "json") + # Build a list of relative paths containing all the python # package_metadata.json files we depend on. generated_file("${target_name}.${_metadata_path_list_suffix}") { @@ -218,12 +259,19 @@ template("pw_create_python_source_tree") { # Run the python action on the metadata_path_list.txt file pw_python_action(target_name) { + # Save the Python package metadata so this can be installed using + # pw_internal_pip_install. + metadata = { + pw_python_package_metadata_json = [ _package_metadata_json_file ] + } + deps = invoker.packages + [ ":${invoker.target_name}.${_metadata_path_list_suffix}" ] script = "$dir_pw_build/py/pw_build/create_python_tree.py" inputs = _extra_file_inputs public_deps = _public_deps + _pw_internal_run_in_venv = false args = [ "--tree-destination-dir", @@ -264,42 +312,13 @@ template("pw_create_python_source_tree") { } } - # Template to install a bundled Python package. - pw_python_action("$target_name.install") { - module = "pip" - public_deps = [] - if (defined(invoker.public_deps)) { - public_deps += invoker.public_deps - } - - args = [ - "install", - - # This speeds up pip installs. At this point in the gn build the - # virtualenv is already activated so build isolation isn't required. - # This requires that pip, setuptools, and wheel packages are - # installed. - "--no-build-isolation", - ] - public_deps += [ ":${invoker.target_name}" ] - - inputs = pw_build_PIP_CONSTRAINTS - foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) { - args += [ - "--constraint", - rebase_path(_constraints_file, root_build_dir), - ] - } - stamp = true - args += [ rebase_path(_output_dir, root_build_dir) ] - } - # Template to build a bundled Python package wheel. pw_python_action("$target_name._build_wheel") { metadata = { pw_python_package_wheels = [ "$target_out_dir/$target_name" ] } module = "build" + _pw_internal_run_in_venv = false args = [ rebase_path(_output_dir, root_build_dir), "--wheel", @@ -321,6 +340,8 @@ template("pw_create_python_source_tree") { # Stub target groups to match a pw_python_package. This lets $target_name be # used as a python_dep in pw_python_group. + group("$target_name.install") { + } group("$target_name._run_pip_install") { } group("$target_name.lint") { @@ -332,3 +353,126 @@ template("pw_create_python_source_tree") { group("$target_name.tests") { } } + +# Runs pip install on a set of pw_python_packages. This will install +# pw_python_packages into the user's developer environment. +# +# This is an experimental template. +# +# Args: +# packages: A list of pw_python_package targets to be pip installed. +# These will be installed one at a time. +# +# editable: If true, --editable is passed to the pip install command. +# +# force_reinstall: If true, --force-reinstall is passed to the pip install +# command. +template("pw_internal_pip_install") { + if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { + if (!pw_build_USE_NEW_PYTHON_BUILD) { + pw_error("$target_name") { + message_lines = [ "pw_internal_pip_install targets are not supported unless this gn arg is set pw_build_USE_NEW_PYTHON_BUILD=true." ] + } + } else { + pw_python_action("$target_name") { + script = "$dir_pw_build/py/pw_build/pip_install_python_deps.py" + + assert( + defined(invoker.packages), + "packages = [ 'python_package' ] is required by pw_internal_pip_install") + + public_deps = [] + if (defined(invoker.public_deps)) { + public_deps += invoker.public_deps + } + + python_deps = [] + python_metadata_deps = [] + if (defined(invoker.packages)) { + public_deps += invoker.packages + python_deps += invoker.packages + python_metadata_deps += invoker.packages + } + + python_deps = [] + if (defined(invoker.python_deps)) { + python_deps += invoker.python_deps + } + + _pw_internal_run_in_venv = false + _forward_python_metadata_deps = true + + _editable_install = false + if (defined(invoker.editable)) { + _editable_install = invoker.editable + } + + _pkg_gn_labels = [] + foreach(pkg, invoker.packages) { + _pkg_gn_labels += [ get_label_info(pkg, "label_no_toolchain") ] + } + + args = [ + "--gn-packages", + string_join(",", _pkg_gn_labels), + ] + + if (_editable_install) { + args += [ "--editable-pip-install" ] + } + + args += [ + "install", + "--no-build-isolation", + ] + + _force_reinstall = false + if (defined(invoker.force_reinstall)) { + _force_reinstall = true + } + if (_force_reinstall) { + args += [ "--force-reinstall" ] + } + + inputs = pw_build_PIP_CONSTRAINTS + foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) { + args += [ + "--constraint", + rebase_path(_constraints_file, root_build_dir), + ] + } + + stamp = true + + # Parallel pip installations don't work, so serialize pip invocations. + pool = "$dir_pw_build/pool:pip($default_toolchain)" + } + } + } else { + group("$target_name") { + deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ] + } + not_needed("*") + not_needed(invoker, "*") + } + + # Stub target groups to match a pw_python_package. This lets $target_name be + # used as a python_dep in pw_python_group. + group("$target_name.install") { + public_deps = [ ":${invoker.target_name}" ] + } + group("$target_name._run_pip_install") { + } + group("$target_name.wheel") { + } + group("$target_name._build_wheel") { + } + group("$target_name.lint") { + } + group("$target_name.lint.mypy") { + } + group("$target_name.lint.pylint") { + } + group("$target_name.tests") { + } +} diff --git a/pw_build/python_gn_args.gni b/pw_build/python_gn_args.gni new file mode 100644 index 0000000000..24395a2733 --- /dev/null +++ b/pw_build/python_gn_args.gni @@ -0,0 +1,24 @@ +# Copyright 2022 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import("//build_overrides/pigweed.gni") + +declare_args() { + # If true, individual Pigweed Python packages will not be installed into the + # user's development environment. Instead they will be merged into a Python + # package titled 'pigweed' with a version matching the latest version + # available at https://pypi.org/project/pigweed/ with the current date + # appended. + pw_build_USE_NEW_PYTHON_BUILD = false +} diff --git a/pw_build/update_bundle.gni b/pw_build/update_bundle.gni index 1b0ff7ba0c..8e4b766de3 100644 --- a/pw_build/update_bundle.gni +++ b/pw_build/update_bundle.gni @@ -56,6 +56,7 @@ template("pw_update_bundle") { _persist_path = "${target_out_dir}/${target_name}/tuf_repo" } module = "pw_software_update.update_bundle" + python_deps = [ "$dir_pw_software_update/py" ] args = [ "--out", rebase_path(_out_path), diff --git a/pw_console/py/command_runner_test.py b/pw_console/py/command_runner_test.py index 45362fe90e..a4012c6fe8 100644 --- a/pw_console/py/command_runner_test.py +++ b/pw_console/py/command_runner_test.py @@ -35,10 +35,12 @@ def _create_console_app(log_pane_count=2): - console_app = ConsoleApp(color_depth=ColorDepth.DEPTH_8_BIT, - prefs=ConsolePrefs(project_file=False, - project_user_file=False, - user_file=False)) + prefs = ConsolePrefs(project_file=False, + project_user_file=False, + user_file=False) + prefs.set_code_theme('default') + console_app = ConsoleApp(color_depth=ColorDepth.DEPTH_8_BIT, prefs=prefs) + console_app.prefs.reset_config() # Setup log panes diff --git a/pw_console/py/console_app_test.py b/pw_console/py/console_app_test.py index 4dec65f796..f5ef20f58d 100644 --- a/pw_console/py/console_app_test.py +++ b/pw_console/py/console_app_test.py @@ -30,22 +30,25 @@ class TestConsoleApp(unittest.TestCase): def test_instantiate(self) -> None: """Test init.""" with create_app_session(output=FakeOutput()): + prefs = ConsolePrefs(project_file=False, + project_user_file=False, + user_file=False) + prefs.set_code_theme('default') console_app = ConsoleApp(color_depth=ColorDepth.DEPTH_8_BIT, - prefs=ConsolePrefs( - project_file=False, - project_user_file=False, - user_file=False)) + prefs=prefs) + self.assertIsNotNone(console_app) def test_multiple_loggers_in_one_pane(self) -> None: """Test window resizing.""" # pylint: disable=protected-access with create_app_session(output=FakeOutput()): + prefs = ConsolePrefs(project_file=False, + project_user_file=False, + user_file=False) + prefs.set_code_theme('default') console_app = ConsoleApp(color_depth=ColorDepth.DEPTH_8_BIT, - prefs=ConsolePrefs( - project_file=False, - project_user_file=False, - user_file=False)) + prefs=prefs) loggers = { 'Logs': [ diff --git a/pw_console/py/pw_console/console_prefs.py b/pw_console/py/pw_console/console_prefs.py index 1fda908f5c..00e748bd77 100644 --- a/pw_console/py/pw_console/console_prefs.py +++ b/pw_console/py/pw_console/console_prefs.py @@ -139,6 +139,9 @@ def theme_colors(self): def code_theme(self) -> str: return self._config.get('code_theme', '') + def set_code_theme(self, theme_name: str): + self._config['code_theme'] = theme_name + @property def swap_light_and_dark(self) -> bool: return self._config.get('swap_light_and_dark', False) diff --git a/pw_console/py/repl_pane_test.py b/pw_console/py/repl_pane_test.py index 86f41cea70..551cd509f6 100644 --- a/pw_console/py/repl_pane_test.py +++ b/pw_console/py/repl_pane_test.py @@ -114,10 +114,12 @@ async def test_user_thread(self) -> None: with create_app_session(output=FakeOutput()): # Setup Mocks + prefs = ConsolePrefs(project_file=False, + project_user_file=False, + user_file=False) + prefs.set_code_theme('default') app = ConsoleApp(color_depth=ColorDepth.DEPTH_8_BIT, - prefs=ConsolePrefs(project_file=False, - project_user_file=False, - user_file=False)) + prefs=prefs) app.start_user_code_thread() diff --git a/pw_console/py/setup.cfg b/pw_console/py/setup.cfg index 413d7142b0..d16c4734c4 100644 --- a/pw_console/py/setup.cfg +++ b/pw_console/py/setup.cfg @@ -23,16 +23,16 @@ packages = find: zip_safe = False install_requires = ipython - jinja2 + Jinja2 prompt_toolkit>=3.0.26 ptpython>=3.0.20 pw_cli pw_log_tokenized - pygments + Pygments pygments-style-dracula pygments-style-tomorrow pyperclip - pyyaml + PyYAML types-pygments types-PyYAML diff --git a/pw_console/py/window_manager_test.py b/pw_console/py/window_manager_test.py index 676d07cb4a..f9e7095b6b 100644 --- a/pw_console/py/window_manager_test.py +++ b/pw_console/py/window_manager_test.py @@ -29,10 +29,11 @@ def _create_console_app(logger_count=2): - console_app = ConsoleApp(color_depth=ColorDepth.DEPTH_8_BIT, - prefs=ConsolePrefs(project_file=False, - project_user_file=False, - user_file=False)) + prefs = ConsolePrefs(project_file=False, + project_user_file=False, + user_file=False) + prefs.set_code_theme('default') + console_app = ConsoleApp(color_depth=ColorDepth.DEPTH_8_BIT, prefs=prefs) console_app.focus_on_container = MagicMock() loggers = {} diff --git a/pw_env_setup/BUILD.gn b/pw_env_setup/BUILD.gn index 5af2d8ee71..af86dd92a5 100644 --- a/pw_env_setup/BUILD.gn +++ b/pw_env_setup/BUILD.gn @@ -65,7 +65,14 @@ _pigweed_python_deps = [ ] pw_python_group("python") { - python_deps = _pigweed_python_deps + python_deps = [] + if (pw_build_USE_NEW_PYTHON_BUILD) { + # This target creates and installs Python package named 'pigweed' including + # every package listed in _pigweed_python_deps. + python_deps += [ ":pip_install_pigweed_package" ] + } else { + python_deps += _pigweed_python_deps + } python_deps += [ # Standalone scripts "$dir_pw_hdlc/rpc_example:example_script", @@ -78,12 +85,60 @@ pw_python_group("python") { ] } +if (pw_build_USE_NEW_PYTHON_BUILD) { + group("create_gn_venv") { + if (current_toolchain == default_toolchain) { + exec_script("create_gn_venv.py", + [ + "--destination-dir", + rebase_path("$root_build_dir/python-venv", root_build_dir), + ]) + } + } + + pw_python_action("install_3p_deps") { + module = "pip" + _pw_internal_run_in_venv = true + skip_installing_external_python_deps = true + args = [ + "install", + "--no-build-isolation", + ] + inputs = pw_build_PIP_CONSTRAINTS + foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) { + args += [ + "--requirement", + rebase_path(_constraints_file, root_build_dir), + ] + } + outputs = [ "$root_build_dir/python-venv" ] + deps = [ ":create_gn_venv" ] + stamp = true + pool = "$dir_pw_build/pool:pip($default_toolchain)" + } +} else { + group("create_gn_venv") { + } + group("install_3p_deps") { + } +} + # Python packages for supporting specific targets. -pw_python_group("target_support_packages") { - python_deps = [ - "$dir_pigweed/targets/lm3s6965evb_qemu/py", - "$dir_pigweed/targets/stm32f429i_disc1/py", - ] +if (pw_build_USE_NEW_PYTHON_BUILD) { + pw_internal_pip_install("target_support_packages") { + packages = [ + "$dir_pigweed/targets/lm3s6965evb_qemu/py", + "$dir_pigweed/targets/stm32f429i_disc1/py", + ] + editable = true + } +} else { + pw_python_group("target_support_packages") { + python_deps = [ + "$dir_pigweed/targets/lm3s6965evb_qemu/py", + "$dir_pigweed/targets/stm32f429i_disc1/py", + ] + } } pw_python_requirements("renode_requirements") { @@ -122,3 +177,29 @@ pw_create_python_source_tree("pypi_pigweed_python_source_tree") { "pypi_pyproject.toml > pyproject.toml", ] } + +if (pw_build_USE_NEW_PYTHON_BUILD) { + # This target is creates the 'pigweed' Python package installed to the user's + # dev environment. It's similar to the source tree for PyPI but it appends the + # current date to the version so pip will consider it more up to date than the + # one in PyPI. + pw_create_python_source_tree("generate_pigweed_python_package") { + packages = _pigweed_python_deps + public_deps = _pigweed_python_deps + + generate_setup_cfg = { + common_config_file = "pypi_common_setup.cfg" + append_date_to_version = true + } + extra_files = [ + "$dir_pigweed/LICENSE > LICENSE", + "$dir_pigweed/README.md > README.md", + "pypi_pyproject.toml > pyproject.toml", + ] + } + + # This pip installs the generate_pigweed_python_package + pw_internal_pip_install("pip_install_pigweed_package") { + packages = [ ":generate_pigweed_python_package" ] + } +} diff --git a/pw_env_setup/create_gn_venv.py b/pw_env_setup/create_gn_venv.py new file mode 100644 index 0000000000..10bb09408c --- /dev/null +++ b/pw_env_setup/create_gn_venv.py @@ -0,0 +1,38 @@ +# Copyright 2022 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Crate a venv.""" + +import argparse +import os +import venv +from pathlib import Path + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--destination-dir', + type=Path, + required=True, + help='Path to venv directory.') + return parser.parse_args() + + +def main(destination_dir: Path) -> None: + if not destination_dir.is_dir(): + use_symlinks = not os.name == 'nt' + venv.create(destination_dir, symlinks=use_symlinks, with_pip=True) + + +if __name__ == '__main__': + main(**vars(_parse_args())) diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list b/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list index 84f30a24e7..a6ff76ca13 100644 --- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list +++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list @@ -69,6 +69,7 @@ requests==2.27.1 robotframework==3.1 rsa==4.8 scan-build==2.0.19 +setuptools==62.1.0 six==1.16.0 snowballstemmer==2.2.0 soupsieve==2.3.1 @@ -90,8 +91,8 @@ types-docutils==0.17.4 types-futures==3.3.2 types-protobuf==3.18.4 types-Pygments==2.9.13 -types-PyYAML==6.0.3 -types-setuptools==57.4.7 +types-PyYAML==6.0.7 +types-setuptools==57.4.14 types-six==1.16.9 typing_extensions==4.0.1 urllib3==1.26.8 diff --git a/pw_rpc/py/BUILD.gn b/pw_rpc/py/BUILD.gn index 3968fe281a..cd5ac975a5 100644 --- a/pw_rpc/py/BUILD.gn +++ b/pw_rpc/py/BUILD.gn @@ -93,6 +93,7 @@ pw_python_script("python_client_cpp_server_test") { "$dir_pw_status/py", "$dir_pw_tokenizer/py:test_proto.python", ] + pylintrc = "$dir_pigweed/.pylintrc" action = { diff --git a/pw_rpc/py/pw_rpc/__init__.py b/pw_rpc/py/pw_rpc/__init__.py index ff1f8713bc..338122d80d 100644 --- a/pw_rpc/py/pw_rpc/__init__.py +++ b/pw_rpc/py/pw_rpc/__init__.py @@ -13,5 +13,10 @@ # the License. """Package for calling Pigweed RPCs from Python.""" +# TODO(667): Remove this pkgutil snippet +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) # type: ignore + from pw_rpc.client import Client from pw_rpc.descriptors import Channel, ChannelManipulator diff --git a/targets/lm3s6965evb_qemu/py/setup.cfg b/targets/lm3s6965evb_qemu/py/setup.cfg index f380275010..0d325189e6 100644 --- a/targets/lm3s6965evb_qemu/py/setup.cfg +++ b/targets/lm3s6965evb_qemu/py/setup.cfg @@ -21,7 +21,8 @@ description = Target-specific python scripts for the lm3s6965evb-qemu target [options] packages = find: zip_safe = False -install_requires = coloredlogs +install_requires = + coloredlogs [options.entry_points] console_scripts = diff --git a/targets/stm32f429i_disc1/py/setup.cfg b/targets/stm32f429i_disc1/py/setup.cfg index 42a74b8311..26ec850a41 100644 --- a/targets/stm32f429i_disc1/py/setup.cfg +++ b/targets/stm32f429i_disc1/py/setup.cfg @@ -21,7 +21,9 @@ description = Target-specific python scripts for the stm32f429i-disc1 target [options] packages = find: zip_safe = False -install_requires = pyserial>=3.5,<4.0; coloredlogs; pw_cli +install_requires = + pyserial>=3.5,<4.0 + coloredlogs [options.entry_points] console_scripts =