Skip to content

Commit

Permalink
Provide CLI option -J|--jobs to control number of jobs
Browse files Browse the repository at this point in the history
  • Loading branch information
yarikoptic committed Dec 23, 2023
1 parent 045231e commit abebc4f
Showing 1 changed file with 36 additions and 4 deletions.
40 changes: 36 additions & 4 deletions codespell_lib/_codespell.py
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,20 @@ def parse_options(
"should match the to-be-excluded lines exactly",
)

parser.add_argument(
"-J",
"--jobs",
action="store",
type=int,
default=0,
help="set number of jobs to parallelize processing - one "
"subprocess per file:\n"
"- 0: no parallelization (default)"
"- positive integer: number of sub-processes to use\n"
"- -1: use all available CPUs\n"
"Interactive mode is not compatible with parallel processing",
)

parser.add_argument(
"-i",
"--interactive",
Expand Down Expand Up @@ -1084,7 +1098,7 @@ def _script_main() -> int:
return main(*sys.argv[1:])


def main(*args: str) -> int: # noqa: C901,PLR0915
def main(*args: str) -> int: # noqa: C901,PLR0915,PLR0911
"""Contains flow control"""
try:
options, parser, used_cfg_files = parse_options(args)
Expand Down Expand Up @@ -1196,6 +1210,25 @@ def main(*args: str) -> int: # noqa: C901,PLR0915
else:
summary = None

if options.jobs and options.interactive:
print(
"ERROR: do not enable parallelization in interactive mode",
file=sys.stderr,
)
# no point to parser.print_help() - just hides ERROR away here
return EX_USAGE

jobs = options.jobs
if jobs == -1:
jobs = os.cpu_count()
elif jobs < -1:
print(
f"ERROR: invalid number of jobs: {jobs}",
file=sys.stderr,
)
parser.print_help()
return EX_USAGE

context = None
if options.context is not None:
if (options.before_context is not None) or (options.after_context is not None):
Expand Down Expand Up @@ -1285,10 +1318,9 @@ def _find_files() -> Generator[str, None, None]:
options,
)

njobs = os.cpu_count() or 1
if njobs:
if jobs:
# parse_file would be in subprocess(es)
with Pool(njobs) as pool:
with Pool(jobs) as pool:
results = pool.map(file_parser, _find_files())
for result in results:
if isinstance(result, Exception):
Expand Down

0 comments on commit abebc4f

Please sign in to comment.