From bee24cd55af4f1231e787aed5a1cc072492adee6 Mon Sep 17 00:00:00 2001 From: Bruce Dawson Date: Sat, 25 Jun 2022 00:52:13 -0700 Subject: [PATCH] Avoid hangs on many-core Windows machines (#7035) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creating a multiprocessing Pool with too many processes can hit ValueError exceptions or hangs or both. The number that counts as "too many" depends on the Python version so this change uses 56 as a guaranteed safe limit. Details in #6965 Note that this only avoids the issue if an explicit jobs count is not passed in. Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- doc/whatsnew/2/2.14/full.rst | 5 +++++ pylint/lint/base_options.py | 3 ++- pylint/lint/run.py | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/whatsnew/2/2.14/full.rst b/doc/whatsnew/2/2.14/full.rst index d306a7f825..92d9b42286 100644 --- a/doc/whatsnew/2/2.14/full.rst +++ b/doc/whatsnew/2/2.14/full.rst @@ -13,6 +13,11 @@ Release date: TBA Closes #7006 +* Fixed an issue where many-core Windows machines (>~60 logical processors) would hang when + using the default jobs count. + + Closes #6965 + What's New in Pylint 2.14.3? ---------------------------- Release date: 2022-06-18 diff --git a/pylint/lint/base_options.py b/pylint/lint/base_options.py index 25327e635e..d909e39c59 100644 --- a/pylint/lint/base_options.py +++ b/pylint/lint/base_options.py @@ -244,7 +244,8 @@ def _make_linter_options(linter: PyLinter) -> Options: "short": "j", "default": 1, "help": "Use multiple processes to speed up Pylint. Specifying 0 will " - "auto-detect the number of processors available to use.", + "auto-detect the number of processors available to use, and will cap " + "the count on Windows to avoid hangs.", }, ), ( diff --git a/pylint/lint/run.py b/pylint/lint/run.py index 27a4d9d84d..b8923aac5f 100644 --- a/pylint/lint/run.py +++ b/pylint/lint/run.py @@ -82,6 +82,9 @@ def _cpu_count() -> int: cpu_count = multiprocessing.cpu_count() else: cpu_count = 1 + if sys.platform == "win32": + # See also https://github.com/python/cpython/issues/94242 + cpu_count = min(cpu_count, 56) # pragma: no cover if cpu_share is not None: return min(cpu_share, cpu_count) return cpu_count