String resources

pants.toml

@@ -120,6 +120,9 @@ experimental_lockfile = "3rdparty/python/lockfiles/user_reqs.txt"
 interpreter_constraints = [">=3.7,<3.10"]
 macos_big_sur_compatibility = true
 
+[python-infer]
+string_resources = true
+
 [docformatter]
 args = ["--wrap-summaries=100", "--wrap-descriptions=100"]
 

src/python/pants/backend/python/dependency_inference/BUILD

@@ -1,13 +1,6 @@
 # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-python_sources(
-    overrides={
-        "parse_python_imports.py": {
-            # This Python script is loaded as a resource, see parse_python_imports.py for more info.
-            "dependencies": ["./scripts:import_parser"]
-        }
-    }
-)
+python_sources()
 
 python_tests(name="tests")

src/python/pants/backend/python/dependency_inference/parse_python_imports.py → src/python/pants/backend/python/dependency_inference/parse_python_deps.py

@@ -2,14 +2,17 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import json
+import pathlib
 import pkgutil
 from dataclasses import dataclass
+from typing import Tuple
 
 from pants.backend.python.target_types import PythonSourceField
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.pex_environment import PythonExecutable
 from pants.core.util_rules.source_files import SourceFilesRequest
 from pants.core.util_rules.stripped_source_files import StrippedSourceFiles
+from pants.engine.collection import DeduplicatedCollection
 from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests
 from pants.engine.process import Process, ProcessResult
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
@@ -30,22 +33,48 @@ class ParsedPythonImportInfo:
 class ParsedPythonImports(FrozenDict[str, ParsedPythonImportInfo]):
     """All the discovered imports from a Python source file mapped to the relevant info."""
 
+    # N.B Don't set `sort_input`, as the input is already sorted
+
+
+class ParsedPythonResources(DeduplicatedCollection[Tuple[str, str]]):
+    """All the discovered possible resources from a Python source file.
+
+    The tuple is of (containing module, relative filename), similar to the arguments of
+    `pkgutil.get_data`.
+    """
+
+    # N.B. Don't set `sort_input`, as the input is already sorted
+
 
 @dataclass(frozen=True)
-class ParsePythonImportsRequest:
+class ParsedPythonDependencies:
+    imports: ParsedPythonImports
+    resources: ParsedPythonResources
+
+
+@dataclass(frozen=True)
+class ParsePythonDependenciesRequest:
     source: PythonSourceField
     interpreter_constraints: InterpreterConstraints
     string_imports: bool
     string_imports_min_dots: int
+    string_resources: bool
+    string_resources_min_slashes: int
+
+
+def _filepath_to_modname(filepath: str) -> str:
+    return str(pathlib.Path(filepath).with_suffix("")).replace("/", ".")
 
 
 @rule
-async def parse_python_imports(request: ParsePythonImportsRequest) -> ParsedPythonImports:
-    script = pkgutil.get_data(__name__, "scripts/import_parser.py")
+async def parse_python_dependencies(
+    request: ParsePythonDependenciesRequest,
+) -> ParsedPythonDependencies:
+    script = pkgutil.get_data(__name__, "scripts/dependency_parser.py")
     assert script is not None
     python_interpreter, script_digest, stripped_sources = await MultiGet(
         Get(PythonExecutable, InterpreterConstraints, request.interpreter_constraints),
-        Get(Digest, CreateDigest([FileContent("__parse_python_imports.py", script)])),
+        Get(Digest, CreateDigest([FileContent("__parse_python_dependencies.py", script)])),
         Get(StrippedSourceFiles, SourceFilesRequest([request.source])),
     )
 
@@ -61,23 +90,38 @@ async def parse_python_imports(request: ParsePythonImportsRequest) -> ParsedPyth
         Process(
             argv=[
                 python_interpreter.path,
-                "./__parse_python_imports.py",
+                "./__parse_python_dependencies.py",
                 file,
             ],
             input_digest=input_digest,
-            description=f"Determine Python imports for {request.source.address}",
+            description=f"Determine Python dependencies for {request.source.address}",
             env={
                 "STRING_IMPORTS": "y" if request.string_imports else "n",
                 "MIN_DOTS": str(request.string_imports_min_dots),
+                "STRING_RESOURCES": "y" if request.string_resources else "n",
+                "MIN_SLASHES": str(request.string_resources_min_slashes),
             },
             level=LogLevel.DEBUG,
         ),
     )
     # See above for where we explicitly encoded as utf8. Even though utf8 is the
     # default for decode(), we make that explicit here for emphasis.
     process_output = process_result.stdout.decode("utf8") or "{}"
-    return ParsedPythonImports(
-        (imp, ParsedPythonImportInfo(**info)) for imp, info in json.loads(process_output).items()
+    output = json.loads(process_output)
+
+    return ParsedPythonDependencies(
+        imports=ParsedPythonImports(
+            (key, ParsedPythonImportInfo(**val)) for key, val in output.get("imports", {}).items()
+        ),
+        resources=ParsedPythonResources(
+            [
+                (
+                    _filepath_to_modname(request.source.file_path) if pkgname is None else pkgname,
+                    filepath,
+                )
+                for pkgname, filepath in output.get("resources", [])
+            ]
+        ),
     )
 
 

src/python/pants/backend/python/dependency_inference/parse_python_imports_test.py → src/python/pants/backend/python/dependency_inference/parse_python_deps_test.py

@@ -7,13 +7,13 @@
 
 import pytest
 
-from pants.backend.python.dependency_inference import parse_python_imports
-from pants.backend.python.dependency_inference.parse_python_imports import (
+from pants.backend.python.dependency_inference import parse_python_deps
+from pants.backend.python.dependency_inference.parse_python_deps import ParsedPythonDependencies
+from pants.backend.python.dependency_inference.parse_python_deps import (
     ParsedPythonImportInfo as ImpInfo,
 )
-from pants.backend.python.dependency_inference.parse_python_imports import (
-    ParsedPythonImports,
-    ParsePythonImportsRequest,
+from pants.backend.python.dependency_inference.parse_python_deps import (
+    ParsePythonDependenciesRequest,
 )
 from pants.backend.python.target_types import PythonSourceField, PythonSourceTarget
 from pants.backend.python.util_rules import pex
@@ -32,24 +32,27 @@
 def rule_runner() -> RuleRunner:
     return RuleRunner(
         rules=[
-            *parse_python_imports.rules(),
+            *parse_python_deps.rules(),
             *stripped_source_files.rules(),
             *pex.rules(),
-            QueryRule(ParsedPythonImports, [ParsePythonImportsRequest]),
+            QueryRule(ParsedPythonDependencies, [ParsePythonDependenciesRequest]),
         ],
         target_types=[PythonSourceTarget],
     )
 
 
-def assert_imports_parsed(
+def assert_deps_parsed(
     rule_runner: RuleRunner,
     content: str,
     *,
-    expected: dict[str, ImpInfo],
+    expected_imports: dict[str, ImpInfo] = {},
+    expected_resources: list[str] = [],
     filename: str = "project/foo.py",
     constraints: str = ">=3.6",
     string_imports: bool = True,
     string_imports_min_dots: int = 2,
+    string_resources: bool = True,
+    string_resources_min_slashes: int = 1,
 ) -> None:
     rule_runner.set_options([], env_inherit={"PATH", "PYENV_ROOT", "HOME"})
     rule_runner.write_files(
@@ -59,18 +62,23 @@ def assert_imports_parsed(
         }
     )
     tgt = rule_runner.get_target(Address("", target_name="t"))
-    imports = rule_runner.request(
-        ParsedPythonImports,
+    result = rule_runner.request(
+        ParsedPythonDependencies,
         [
-            ParsePythonImportsRequest(
+            ParsePythonDependenciesRequest(
                 tgt[PythonSourceField],
                 InterpreterConstraints([constraints]),
                 string_imports=string_imports,
                 string_imports_min_dots=string_imports_min_dots,
+                string_resources=string_resources,
+                string_resources_min_slashes=string_resources_min_slashes,
             )
         ],
     )
-    assert dict(imports) == expected
+    assert dict(result.imports) == expected_imports
+    assert list(result.resources) == sorted(
+        ("project.foo", resource) for resource in expected_resources
+    )
 
 
 def test_normal_imports(rule_runner: RuleRunner) -> None:
@@ -107,10 +115,10 @@ def test_normal_imports(rule_runner: RuleRunner) -> None:
             from project.circular_dep import CircularDep
         """
     )
-    assert_imports_parsed(
+    assert_deps_parsed(
         rule_runner,
         content,
-        expected={
+        expected_imports={
             "__future__.print_function": ImpInfo(lineno=1, weak=False),
             "os": ImpInfo(lineno=3, weak=False),
             "os.path": ImpInfo(lineno=5, weak=False),
@@ -143,10 +151,10 @@ def test_dunder_import_call(rule_runner: RuleRunner) -> None:
         )  # pants: no-infer-dep
         """
     )
-    assert_imports_parsed(
+    assert_deps_parsed(
         rule_runner,
         content,
-        expected={
+        expected_imports={
             "pkg_resources": ImpInfo(lineno=1, weak=False),
             "not_ignored_but_looks_like_it_could_be": ImpInfo(lineno=4, weak=False),
             "also_not_ignored_but_looks_like_it_could_be": ImpInfo(lineno=10, weak=False),
@@ -204,10 +212,10 @@ def test_try_except(rule_runner: RuleRunner) -> None:
             import strong9
         """
     )
-    assert_imports_parsed(
+    assert_deps_parsed(
         rule_runner,
         content,
-        expected={
+        expected_imports={
             "strong1": ImpInfo(lineno=1, weak=False),
             "weak1": ImpInfo(lineno=4, weak=True),
             "weak2": ImpInfo(lineno=7, weak=True),
@@ -243,11 +251,11 @@ def test_relative_imports(rule_runner: RuleRunner, basename: str) -> None:
         )
         """
     )
-    assert_imports_parsed(
+    assert_deps_parsed(
         rule_runner,
         content,
         filename=f"project/util/{basename}",
-        expected={
+        expected_imports={
             "project.util.sibling": ImpInfo(lineno=1, weak=False),
             "project.util.sibling.Nibling": ImpInfo(lineno=2, weak=False),
             "project.util.subdir.child.Child": ImpInfo(lineno=3, weak=False),
@@ -307,20 +315,22 @@ def test_imports_from_strings(rule_runner: RuleRunner, min_dots: int) -> None:
     }
     expected = {sym: info for sym, info in potentially_valid.items() if sym.count(".") >= min_dots}
 
-    assert_imports_parsed(rule_runner, content, expected=expected, string_imports_min_dots=min_dots)
-    assert_imports_parsed(rule_runner, content, string_imports=False, expected={})
+    assert_deps_parsed(
+        rule_runner, content, expected_imports=expected, string_imports_min_dots=min_dots
+    )
+    assert_deps_parsed(rule_runner, content, string_imports=False, expected_imports={})
 
 
 def test_real_import_beats_string_import(rule_runner: RuleRunner) -> None:
-    assert_imports_parsed(
+    assert_deps_parsed(
         rule_runner,
         "import one.two.three; 'one.two.three'",
-        expected={"one.two.three": ImpInfo(lineno=1, weak=False)},
+        expected_imports={"one.two.three": ImpInfo(lineno=1, weak=False)},
     )
 
 
 def test_real_import_beats_tryexcept_import(rule_runner: RuleRunner) -> None:
-    assert_imports_parsed(
+    assert_deps_parsed(
         rule_runner,
         dedent(
             """\
@@ -329,16 +339,16 @@ def test_real_import_beats_tryexcept_import(rule_runner: RuleRunner) -> None:
                 except ImportError: pass
             """
         ),
-        expected={"one.two.three": ImpInfo(lineno=1, weak=False)},
+        expected_imports={"one.two.three": ImpInfo(lineno=1, weak=False)},
     )
 
 
 def test_gracefully_handle_syntax_errors(rule_runner: RuleRunner) -> None:
-    assert_imports_parsed(rule_runner, "x =", expected={})
+    assert_deps_parsed(rule_runner, "x =", expected_imports={})
 
 
 def test_handle_unicode(rule_runner: RuleRunner) -> None:
-    assert_imports_parsed(rule_runner, "x = 'äbç'", expected={})
+    assert_deps_parsed(rule_runner, "x = 'äbç'", expected_imports={})
 
 
 @skip_unless_python27_present
@@ -367,11 +377,11 @@ def test_works_with_python2(rule_runner: RuleRunner) -> None:
         finally: import strong3
         """
     )
-    assert_imports_parsed(
+    assert_deps_parsed(
         rule_runner,
         content,
         constraints="==2.7.*",
-        expected={
+        expected_imports={
             "demo": ImpInfo(lineno=4, weak=False),
             "project.demo.Demo": ImpInfo(lineno=5, weak=False),
             "pkg_resources": ImpInfo(lineno=7, weak=False),
@@ -404,11 +414,11 @@ def test_works_with_python38(rule_runner: RuleRunner) -> None:
         importlib.import_module("dep.from.str")
         """
     )
-    assert_imports_parsed(
+    assert_deps_parsed(
         rule_runner,
         content,
         constraints=">=3.8",
-        expected={
+        expected_imports={
             "demo": ImpInfo(lineno=5, weak=False),
             "project.demo.Demo": ImpInfo(lineno=6, weak=False),
             "pkg_resources": ImpInfo(lineno=8, weak=False),
@@ -437,11 +447,11 @@ def test_works_with_python39(rule_runner: RuleRunner) -> None:
         importlib.import_module("dep.from.str")
         """
     )
-    assert_imports_parsed(
+    assert_deps_parsed(
         rule_runner,
         content,
         constraints=">=3.9",
-        expected={
+        expected_imports={
             "demo": ImpInfo(lineno=7, weak=False),
             "project.demo.Demo": ImpInfo(lineno=8, weak=False),
             "pkg_resources": ImpInfo(lineno=10, weak=False),