From 64913137ae47cfb7c5a3fcf2e840f32c7e7db09e Mon Sep 17 00:00:00 2001 From: Jannick Kremer Date: Tue, 27 Aug 2024 13:46:55 +0200 Subject: [PATCH 1/7] Implement flag to allow typechecking of untyped modules Fixes #8545 --- mypy/main.py | 5 +++++ mypy/modulefinder.py | 15 ++++++++++++++- mypy/options.py | 3 +++ test-data/unit/pep561.test | 7 +++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/mypy/main.py b/mypy/main.py index f177bb1c2062..79ac530dcd94 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -572,6 +572,11 @@ def add_invertible_flag( action="store_true", help="Silently ignore imports of missing modules", ) + imports_group.add_argument( + "--enable-installed-packages", + action="store_true", + help="Typecheck modules without stubs or py.typed marker", + ) imports_group.add_argument( "--follow-imports", choices=["normal", "silent", "skip", "error"], diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 452cfef20f4c..4d37f26f00bc 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -13,7 +13,7 @@ import subprocess import sys from enum import Enum, unique -from typing import Dict, Final, List, NamedTuple, Optional, Tuple, Union +from typing import cast, Dict, Final, List, NamedTuple, Optional, Tuple, Union from typing_extensions import TypeAlias as _TypeAlias from mypy import pyinfo @@ -334,6 +334,19 @@ def _find_module_non_stub_helper( if approved_stub_package_exists(".".join(components[:i])): return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED if plausible_match: + if self.options: + enable_installed_packages = self.options.enable_installed_packages + try: + enable_installed_packages = cast( + bool, + self.options.per_module_options[components[0]][ + "enable_installed_packages" + ], + ) + except KeyError: + pass + if enable_installed_packages: + return os.path.join(pkg_dir, *components[:-1]), False return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS else: return ModuleNotFoundReason.NOT_FOUND diff --git a/mypy/options.py b/mypy/options.py index 5e64d5e40035..0a1676ccc7ae 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -39,6 +39,7 @@ class BuildType: "disallow_untyped_defs", "enable_error_code", "enabled_error_codes", + "enable_installed_packages", "extra_checks", "follow_imports_for_stubs", "follow_imports", @@ -113,6 +114,8 @@ def __init__(self) -> None: self.ignore_missing_imports = False # Is ignore_missing_imports set in a per-module section self.ignore_missing_imports_per_module = False + # Typecheck modules without stubs or py.typed marker + self.enable_installed_packages = False self.follow_imports = "normal" # normal|silent|skip|error # Whether to respect the follow_imports setting even for stub files. # Intended to be used for disabling specific stubs. diff --git a/test-data/unit/pep561.test b/test-data/unit/pep561.test index 9969c2894c36..acde01d0e891 100644 --- a/test-data/unit/pep561.test +++ b/test-data/unit/pep561.test @@ -187,6 +187,13 @@ b.bf(1) testNamespacePkgWStubsWithNamespacePackagesFlag.py:7: error: Argument 1 to "bf" has incompatible type "int"; expected "bool" testNamespacePkgWStubsWithNamespacePackagesFlag.py:8: error: Argument 1 to "bf" has incompatible type "int"; expected "bool" +[case testMissingPytypedFlag] +# pkgs: typedpkg_ns_b +# flags: --namespace-packages --enable-installed-packages +import typedpkg_ns.b.bbb as b +b.bf("foo", "bar") +[out] +testMissingPytypedFlag.py:4: error: Too many arguments for "bf" [case testTypedPkgNamespaceRegFromImportTwiceMissing] # pkgs: typedpkg_ns_a From 05466a3269caf02d23102228d6f70aed217537de Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:14:33 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/modulefinder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 4d37f26f00bc..f3ed955dcc5a 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -13,7 +13,7 @@ import subprocess import sys from enum import Enum, unique -from typing import cast, Dict, Final, List, NamedTuple, Optional, Tuple, Union +from typing import Dict, Final, List, NamedTuple, Optional, Tuple, Union, cast from typing_extensions import TypeAlias as _TypeAlias from mypy import pyinfo From ccd9cf739b665dbef6b7512bb446c8d6533951e6 Mon Sep 17 00:00:00 2001 From: Jannick Kremer Date: Sat, 23 Nov 2024 09:43:47 +0100 Subject: [PATCH 3/7] Properly handle glob options and simplify detection --- mypy/modulefinder.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 9f2aee8d1012..efc3f0043a1d 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -334,10 +334,11 @@ def _typeshed_has_version(self, module: str) -> bool: return version >= min_version and (max_version is None or version <= max_version) def _find_module_non_stub_helper( - self, components: list[str], pkg_dir: str + self, id: str, pkg_dir: str ) -> OnePackageDir | ModuleNotFoundReason: plausible_match = False dir_path = pkg_dir + components = id.split(".") for index, component in enumerate(components): dir_path = os_path_join(dir_path, component) if self.fscache.isfile(os_path_join(dir_path, "py.typed")): @@ -351,17 +352,9 @@ def _find_module_non_stub_helper( break if plausible_match: if self.options: - enable_installed_packages = self.options.enable_installed_packages - try: - enable_installed_packages = cast( - bool, - self.options.per_module_options[components[0]][ - "enable_installed_packages" - ], - ) - except KeyError: - pass - if enable_installed_packages: + module_specific_options = self.options.clone_for_module(id) + if module_specific_options.enable_installed_packages: + # print("Returning ", id) return os.path.join(pkg_dir, *components[:-1]), False return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS else: @@ -476,7 +469,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: third_party_stubs_dirs.append((path, True)) else: third_party_stubs_dirs.append((path, True)) - non_stub_match = self._find_module_non_stub_helper(components, pkg_dir) + non_stub_match = self._find_module_non_stub_helper(id, pkg_dir) if isinstance(non_stub_match, ModuleNotFoundReason): if non_stub_match is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS: found_possible_third_party_missing_type_hints = True From 0ab6608bdaec738ffce6466d8cd9c4a65ea2c805 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 23 Nov 2024 08:44:31 +0000 Subject: [PATCH 4/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/modulefinder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index efc3f0043a1d..288a8aa606d4 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -13,7 +13,7 @@ import subprocess import sys from enum import Enum, unique -from typing import Dict, Final, List, Optional, Tuple, Union, cast +from typing import Dict, Final, List, Optional, Tuple, Union from typing_extensions import TypeAlias as _TypeAlias from mypy import pyinfo From 7333b60f7824be123a8d053209ec4993e01999f9 Mon Sep 17 00:00:00 2001 From: Jannick Kremer Date: Sat, 23 Nov 2024 09:53:23 +0100 Subject: [PATCH 5/7] Change config option name to follow-untyped-imports --- mypy/main.py | 2 +- mypy/modulefinder.py | 2 +- mypy/options.py | 4 ++-- test-data/unit/pep561.test | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index dcdc7be93570..fb4a1f61a01d 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -581,7 +581,7 @@ def add_invertible_flag( help="Silently ignore imports of missing modules", ) imports_group.add_argument( - "--enable-installed-packages", + "--follow-untyped-imports", action="store_true", help="Typecheck modules without stubs or py.typed marker", ) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 288a8aa606d4..a36f50c334c7 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -353,7 +353,7 @@ def _find_module_non_stub_helper( if plausible_match: if self.options: module_specific_options = self.options.clone_for_module(id) - if module_specific_options.enable_installed_packages: + if module_specific_options.follow_untyped_imports: # print("Returning ", id) return os.path.join(pkg_dir, *components[:-1]), False return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS diff --git a/mypy/options.py b/mypy/options.py index 209bd282988f..561b23fec7d0 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -39,10 +39,10 @@ class BuildType: "disallow_untyped_defs", "enable_error_code", "enabled_error_codes", - "enable_installed_packages", "extra_checks", "follow_imports_for_stubs", "follow_imports", + "follow_untyped_imports", "ignore_errors", "ignore_missing_imports", "implicit_optional", @@ -115,7 +115,7 @@ def __init__(self) -> None: # Is ignore_missing_imports set in a per-module section self.ignore_missing_imports_per_module = False # Typecheck modules without stubs or py.typed marker - self.enable_installed_packages = False + self.follow_untyped_imports = False self.follow_imports = "normal" # normal|silent|skip|error # Whether to respect the follow_imports setting even for stub files. # Intended to be used for disabling specific stubs. diff --git a/test-data/unit/pep561.test b/test-data/unit/pep561.test index acde01d0e891..fb303a8fb5ec 100644 --- a/test-data/unit/pep561.test +++ b/test-data/unit/pep561.test @@ -189,7 +189,7 @@ testNamespacePkgWStubsWithNamespacePackagesFlag.py:8: error: Argument 1 to "bf" [case testMissingPytypedFlag] # pkgs: typedpkg_ns_b -# flags: --namespace-packages --enable-installed-packages +# flags: --namespace-packages --follow-untyped-imports import typedpkg_ns.b.bbb as b b.bf("foo", "bar") [out] From 0a0e413cf7278d5d609aac949917e670d37b2cba Mon Sep 17 00:00:00 2001 From: Jannick Kremer Date: Sat, 23 Nov 2024 22:16:59 +0100 Subject: [PATCH 6/7] Remove commented-out code --- mypy/modulefinder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index a36f50c334c7..fdd89837002f 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -354,7 +354,6 @@ def _find_module_non_stub_helper( if self.options: module_specific_options = self.options.clone_for_module(id) if module_specific_options.follow_untyped_imports: - # print("Returning ", id) return os.path.join(pkg_dir, *components[:-1]), False return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS else: From 095b4b66c492e8ab6270173a52486e48c1f831af Mon Sep 17 00:00:00 2001 From: Jannick Kremer Date: Mon, 2 Dec 2024 18:50:39 +0100 Subject: [PATCH 7/7] Document follow_untyped_imports config option & flag --- docs/source/command_line.rst | 4 ++++ docs/source/config_file.rst | 12 ++++++++++++ docs/source/running_mypy.rst | 6 ++++++ 3 files changed, 22 insertions(+) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 92db5d59d0ee..e65317331d55 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -166,6 +166,10 @@ imports. For more details, see :ref:`ignore-missing-imports`. +.. option:: --follow-untyped-imports + + This flag makes mypy analyze imports without stubs or a py.typed marker. + .. option:: --follow-imports {normal,silent,skip,error} This flag adjusts how mypy follows imported modules that were not diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index 310d0c3dbcb1..e970c23a9ecb 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -315,6 +315,18 @@ section of the command line docs. match the name of the *imported* module, not the module containing the import statement. +.. confval:: follow_untyped_imports + + :type: boolean + :default: False + + Typechecks imports from modules that do not have stubs or a py.typed marker. + + If this option is used in a per-module section, the module name should + match the name of the *imported* module, not the module containing the + import statement. Note that scanning all unannotated modules might + significantly increase the runtime of your mypy calls. + .. confval:: follow_imports :type: string diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst index a8ebc61d4774..91fe525c46e0 100644 --- a/docs/source/running_mypy.rst +++ b/docs/source/running_mypy.rst @@ -321,6 +321,12 @@ not catch errors in its use. recommend avoiding ``--ignore-missing-imports`` if possible: it's equivalent to adding a ``# type: ignore`` to all unresolved imports in your codebase. +4. To make mypy typecheck imports from modules without stubs or a py.typed + marker, you can set the :option:`--follow-untyped-imports ` + command line flag or set the :confval:`follow_untyped_imports` config file option to True, + either in the global section of your mypy config file, or individually on a + per-module basis. + Library stubs not installed ---------------------------