From 7d420a1dc59bf0319a5032ec21b1d0fd3224bfa3 Mon Sep 17 00:00:00 2001 From: Anthony DiGirolamo Date: Thu, 21 Apr 2022 11:05:15 -0700 Subject: [PATCH] python: Fix missing GN deps I discovered a few missing Python GN dependencies while working on the feature to run all python_actions within an isolated virtualenv. CL for that feature is: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/90060 This CL breaks out those new deps into their own change that can be landed ahead of the gn venv feature release. This may help stop some flaky build failures due to Python packages not being installed. Change-Id: I9e22e6e3a7f55c0dde57b54880f9a55ccc048876 Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/92142 Reviewed-by: Armando Montanez Reviewed-by: Wyatt Hepler Commit-Queue: Anthony DiGirolamo --- pw_build_info/py/BUILD.gn | 15 ++++------ pw_build_info/py/build_id_test.py | 4 +-- pw_build_info/py/pyproject.toml | 16 ++++++++++ pw_build_info/py/setup.cfg | 30 +++++++++++++++++++ pw_build_info/py/setup.py | 18 +++++++++++ pw_console/py/BUILD.gn | 1 - pw_console/py/setup.cfg | 2 +- pw_doctor/py/BUILD.gn | 5 +++- .../pw_env_setup/virtualenv_setup/install.py | 1 + pw_hdlc/py/BUILD.gn | 8 ++++- pw_hdlc/rpc_example/BUILD.gn | 7 ++++- pw_presubmit/py/BUILD.gn | 1 + pw_rpc/py/BUILD.gn | 9 +++++- pw_system/py/BUILD.gn | 1 + pw_tokenizer/py/BUILD.gn | 5 ++++ pw_trace_tokenized/py/BUILD.gn | 1 + pw_transfer/py/BUILD.gn | 3 ++ pw_unit_test/test.gni | 5 +++- pw_watch/py/BUILD.gn | 5 +++- 19 files changed, 118 insertions(+), 19 deletions(-) create mode 100644 pw_build_info/py/pyproject.toml create mode 100644 pw_build_info/py/setup.cfg create mode 100644 pw_build_info/py/setup.py diff --git a/pw_build_info/py/BUILD.gn b/pw_build_info/py/BUILD.gn index 95240b9f7e..72098cb7fd 100644 --- a/pw_build_info/py/BUILD.gn +++ b/pw_build_info/py/BUILD.gn @@ -17,20 +17,17 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_build/python.gni") pw_python_package("py") { - generate_setup = { - metadata = { - name = "pw_build_info" - version = "0.0.1" - } - options = { - install_requires = [ "pyelftools" ] - } - } inputs = [ "print_build_id.cc" ] + setup = [ + "pyproject.toml", + "setup.cfg", + "setup.py", + ] sources = [ "pw_build_info/__init__.py", "pw_build_info/build_id.py", ] + python_deps = [ "$dir_pw_cli/py" ] # This test will only ever work on Linux as it requires the ability to compile # AND run an ELF file. diff --git a/pw_build_info/py/build_id_test.py b/pw_build_info/py/build_id_test.py index 6d420a3706..7ba1565419 100644 --- a/pw_build_info/py/build_id_test.py +++ b/pw_build_info/py/build_id_test.py @@ -17,11 +17,11 @@ import tempfile import unittest from pathlib import Path -from pw_cli import env + from pw_build_info import build_id # Since build_id.cc depends on pw_preprocessor, we have to use the in-tree path. -_MODULE_DIR = Path(env.pigweed_environment().PW_ROOT) / 'pw_build_info' +_MODULE_DIR = Path(__file__).parent.parent.resolve() _MODULE_PY_DIR = Path(__file__).parent.resolve() _SHA1_BUILD_ID_LENGTH = 20 diff --git a/pw_build_info/py/pyproject.toml b/pw_build_info/py/pyproject.toml new file mode 100644 index 0000000000..798b747eca --- /dev/null +++ b/pw_build_info/py/pyproject.toml @@ -0,0 +1,16 @@ +# Copyright 2021 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +[build-system] +requires = ['setuptools', 'wheel'] +build-backend = 'setuptools.build_meta' diff --git a/pw_build_info/py/setup.cfg b/pw_build_info/py/setup.cfg new file mode 100644 index 0000000000..50f16cbf78 --- /dev/null +++ b/pw_build_info/py/setup.cfg @@ -0,0 +1,30 @@ +# Copyright 2021 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +[metadata] +name = pw_build_info +version = 0.0.1 +author = Pigweed Authors +author_email = pigweed-developers@googlegroups.com +description = Python scripts that support the GN build + +[options] +packages = find: +zip_safe = False +install_requires = + pyelftools + +[options.package_data] +pw_build_info = + py.typed + print_build_id.cc diff --git a/pw_build_info/py/setup.py b/pw_build_info/py/setup.py new file mode 100644 index 0000000000..c1606beaa8 --- /dev/null +++ b/pw_build_info/py/setup.py @@ -0,0 +1,18 @@ +# Copyright 2021 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""pw_build_info""" + +import setuptools # type: ignore + +setuptools.setup() # Package definition in setup.cfg diff --git a/pw_console/py/BUILD.gn b/pw_console/py/BUILD.gn index 72329540a8..088d770214 100644 --- a/pw_console/py/BUILD.gn +++ b/pw_console/py/BUILD.gn @@ -90,7 +90,6 @@ pw_python_package("py") { python_deps = [ "$dir_pw_cli/py", "$dir_pw_log_tokenized/py", - "$dir_pw_tokenizer/py", ] inputs = [ "pw_console/templates/keybind_list.jinja", diff --git a/pw_console/py/setup.cfg b/pw_console/py/setup.cfg index 8d3ce30ab8..413d7142b0 100644 --- a/pw_console/py/setup.cfg +++ b/pw_console/py/setup.cfg @@ -27,7 +27,7 @@ install_requires = prompt_toolkit>=3.0.26 ptpython>=3.0.20 pw_cli - pw_tokenizer + pw_log_tokenized pygments pygments-style-dracula pygments-style-tomorrow diff --git a/pw_doctor/py/BUILD.gn b/pw_doctor/py/BUILD.gn index 1750e5dab5..43cb2c990d 100644 --- a/pw_doctor/py/BUILD.gn +++ b/pw_doctor/py/BUILD.gn @@ -26,6 +26,9 @@ pw_python_package("py") { "pw_doctor/__init__.py", "pw_doctor/doctor.py", ] - python_deps = [ "$dir_pw_cli/py" ] + python_deps = [ + "$dir_pw_cli/py", + "$dir_pw_env_setup/py", + ] pylintrc = "$dir_pigweed/.pylintrc" } diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py index 067b244b8f..b8841b9994 100644 --- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py +++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py @@ -223,6 +223,7 @@ def pip_install(*args): '--upgrade', 'pip', 'setuptools', + 'toml', # Needed for pyproject.toml package installs. # Include wheel so pip installs can be done without build # isolation. 'wheel') diff --git a/pw_hdlc/py/BUILD.gn b/pw_hdlc/py/BUILD.gn index 8e9c157ff2..ca1e6bbd1a 100644 --- a/pw_hdlc/py/BUILD.gn +++ b/pw_hdlc/py/BUILD.gn @@ -35,10 +35,16 @@ pw_python_package("py") { "encode_test.py", ] python_deps = [ + "$dir_pw_cli/py", "$dir_pw_console/py", "$dir_pw_protobuf_compiler/py", "$dir_pw_rpc/py", + "$dir_pw_tokenizer/py", + ] + python_test_deps = [ + "$dir_pw_build/py", + "$dir_pw_log:protos.python", + "$dir_pw_tokenizer/py:test_proto.python", ] - python_test_deps = [ "$dir_pw_build/py" ] pylintrc = "$dir_pigweed/.pylintrc" } diff --git a/pw_hdlc/rpc_example/BUILD.gn b/pw_hdlc/rpc_example/BUILD.gn index 06a6407762..13f069c38a 100644 --- a/pw_hdlc/rpc_example/BUILD.gn +++ b/pw_hdlc/rpc_example/BUILD.gn @@ -40,6 +40,11 @@ if (dir_pw_third_party_nanopb == "") { pw_python_script("example_script") { sources = [ "example_script.py" ] - python_deps = [ "$dir_pw_hdlc/py" ] + python_deps = [ + "$dir_pw_build/py", + "$dir_pw_hdlc/py", + "$dir_pw_log:protos.python", + "$dir_pw_tokenizer/py:test_proto.python", + ] pylintrc = "$dir_pigweed/.pylintrc" } diff --git a/pw_presubmit/py/BUILD.gn b/pw_presubmit/py/BUILD.gn index 0fa08fcd60..235e506dfe 100644 --- a/pw_presubmit/py/BUILD.gn +++ b/pw_presubmit/py/BUILD.gn @@ -43,6 +43,7 @@ pw_python_package("py") { python_deps = [ "$dir_pw_build:python_lint", "$dir_pw_cli/py", + "$dir_pw_env_setup/py", "$dir_pw_package/py", ] pylintrc = "$dir_pigweed/.pylintrc" diff --git a/pw_rpc/py/BUILD.gn b/pw_rpc/py/BUILD.gn index ac67284e08..03c6ddf03d 100644 --- a/pw_rpc/py/BUILD.gn +++ b/pw_rpc/py/BUILD.gn @@ -67,7 +67,11 @@ pw_python_package("py") { "$dir_pw_protobuf_compiler/py", "$dir_pw_status/py", ] - python_test_deps = [ "$dir_pw_build/py" ] + python_test_deps = [ + "$dir_pw_build/py", + "$dir_pw_log:protos.python", + "$dir_pw_tokenizer/py:test_proto.python", + ] pylintrc = "$dir_pigweed/.pylintrc" proto_library = "..:protos" } @@ -81,8 +85,11 @@ pw_python_script("python_client_cpp_server_test") { sources = [ "tests/python_client_cpp_server_test.py" ] python_deps = [ ":py", + "$dir_pw_build/py", "$dir_pw_hdlc/py", + "$dir_pw_log:protos.python", "$dir_pw_status/py", + "$dir_pw_tokenizer/py:test_proto.python", ] pylintrc = "$dir_pigweed/.pylintrc" diff --git a/pw_system/py/BUILD.gn b/pw_system/py/BUILD.gn index ec9e08d59a..53ee33652f 100644 --- a/pw_system/py/BUILD.gn +++ b/pw_system/py/BUILD.gn @@ -31,6 +31,7 @@ pw_python_package("py") { "$dir_pw_cli/py", "$dir_pw_console/py", "$dir_pw_hdlc/py", + "$dir_pw_log:protos.python", "$dir_pw_protobuf_compiler/py", "$dir_pw_rpc/py", "$dir_pw_tokenizer/py", diff --git a/pw_tokenizer/py/BUILD.gn b/pw_tokenizer/py/BUILD.gn index 2ac30448c1..4c70a28ea4 100644 --- a/pw_tokenizer/py/BUILD.gn +++ b/pw_tokenizer/py/BUILD.gn @@ -64,6 +64,11 @@ pw_python_package("py") { ] proto_library = "..:proto" pylintrc = "$dir_pigweed/.pylintrc" + + python_deps = [ + "$dir_pw_cli/py", + "$dir_pw_protobuf_compiler:test_protos.python", + ] } # This setup.py may be used to install pw_tokenizer without GN. It does not diff --git a/pw_trace_tokenized/py/BUILD.gn b/pw_trace_tokenized/py/BUILD.gn index 4efae83ed0..158baee668 100644 --- a/pw_trace_tokenized/py/BUILD.gn +++ b/pw_trace_tokenized/py/BUILD.gn @@ -29,6 +29,7 @@ pw_python_package("py") { ] python_deps = [ "$dir_pw_hdlc/py", + "$dir_pw_log:protos.python", "$dir_pw_tokenizer/py", "$dir_pw_trace/py", ] diff --git a/pw_transfer/py/BUILD.gn b/pw_transfer/py/BUILD.gn index f21ec38fa5..95ed4db3cb 100644 --- a/pw_transfer/py/BUILD.gn +++ b/pw_transfer/py/BUILD.gn @@ -44,9 +44,12 @@ pw_python_script("python_cpp_transfer_test") { sources = [ "tests/python_cpp_transfer_test.py" ] python_deps = [ ":py", + "$dir_pw_build/py", "$dir_pw_hdlc/py", + "$dir_pw_log:protos.python", "$dir_pw_rpc/py", "$dir_pw_status/py", + "$dir_pw_tokenizer/py:test_proto.python", "..:test_server_proto.python", ] pylintrc = "$dir_pigweed/.pylintrc" diff --git a/pw_unit_test/test.gni b/pw_unit_test/test.gni index 0169194149..868a8052a1 100644 --- a/pw_unit_test/test.gni +++ b/pw_unit_test/test.gni @@ -235,7 +235,10 @@ template("pw_test") { deps = [ ":$_test_target_name" ] inputs = [ pw_unit_test_AUTOMATIC_RUNNER ] module = "pw_unit_test.test_runner" - python_deps = [ "$dir_pw_unit_test/py" ] + python_deps = [ + "$dir_pw_cli/py", + "$dir_pw_unit_test/py", + ] args = [ "--runner", rebase_path(pw_unit_test_AUTOMATIC_RUNNER, root_build_dir), diff --git a/pw_watch/py/BUILD.gn b/pw_watch/py/BUILD.gn index 2197983ab7..2bd1fe05eb 100644 --- a/pw_watch/py/BUILD.gn +++ b/pw_watch/py/BUILD.gn @@ -30,5 +30,8 @@ pw_python_package("py") { ] tests = [ "watch_test.py" ] pylintrc = "$dir_pigweed/.pylintrc" - python_deps = [ "$dir_pw_cli/py" ] + python_deps = [ + "$dir_pw_cli/py", + "$dir_pw_console/py", + ] }