From 6ea4eddf936e88c24a6757c0c858812d5ca1a9c6 Mon Sep 17 00:00:00 2001 From: "Yilei \"Dolee\" Yang" Date: Tue, 19 Jul 2022 14:26:11 -0700 Subject: [PATCH 01/20] Fix the handling of `# fmt: skip` when it's at a colon line (#3148) When the Leaf node with `# fmt: skip` is a NEWLINE inside a `suite` Node, the nodes to ignore should be from the siblings of the parent `suite` Node. There is a also a special case for the ASYNC token, where it expands to the grandparent Node where the ASYNC token is. This fixes GH-2646, GH-3126, GH-2680, GH-2421, GH-2339, and GH-2138. --- CHANGES.md | 1 + src/black/comments.py | 74 +++++++++++++++++++++-------- src/black/linegen.py | 4 +- tests/data/simple_cases/fmtskip8.py | 62 ++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 21 deletions(-) create mode 100644 tests/data/simple_cases/fmtskip8.py diff --git a/CHANGES.md b/CHANGES.md index 8543a8dbfe0..90c62de6b98 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,7 @@ +- Fix incorrect handling of `# fmt: skip` on colon `:` lines. (#3148) - Comments are no longer deleted when a line had spaces removed around power operators (#2874) diff --git a/src/black/comments.py b/src/black/comments.py index 23bf87fca7c..522c1a7b88c 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -9,7 +9,7 @@ else: from typing_extensions import Final -from blib2to3.pytree import Node, Leaf +from blib2to3.pytree import Node, Leaf, type_repr from blib2to3.pgen2 import token from black.nodes import first_leaf_column, preceding_leaf, container_of @@ -174,6 +174,11 @@ def convert_one_fmt_off_pair(node: Node, *, preview: bool) -> bool: first.prefix = prefix[comment.consumed :] if comment.value in FMT_SKIP: first.prefix = "" + standalone_comment_prefix = prefix + else: + standalone_comment_prefix = ( + prefix[:previous_consumed] + "\n" * comment.newlines + ) hidden_value = "".join(str(n) for n in ignored_nodes) if comment.value in FMT_OFF: hidden_value = comment.value + "\n" + hidden_value @@ -195,7 +200,7 @@ def convert_one_fmt_off_pair(node: Node, *, preview: bool) -> bool: Leaf( STANDALONE_COMMENT, hidden_value, - prefix=prefix[:previous_consumed] + "\n" * comment.newlines, + prefix=standalone_comment_prefix, ), ) return True @@ -211,26 +216,10 @@ def generate_ignored_nodes( If comment is skip, returns leaf only. Stops at the end of the block. """ - container: Optional[LN] = container_of(leaf) if comment.value in FMT_SKIP: - prev_sibling = leaf.prev_sibling - # Need to properly format the leaf prefix to compare it to comment.value, - # which is also formatted - comments = list_comments(leaf.prefix, is_endmarker=False, preview=preview) - if comments and comment.value == comments[0].value and prev_sibling is not None: - leaf.prefix = "" - siblings = [prev_sibling] - while ( - "\n" not in prev_sibling.prefix - and prev_sibling.prev_sibling is not None - ): - prev_sibling = prev_sibling.prev_sibling - siblings.insert(0, prev_sibling) - for sibling in siblings: - yield sibling - elif leaf.parent is not None: - yield leaf.parent + yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment, preview=preview) return + container: Optional[LN] = container_of(leaf) while container is not None and container.type != token.ENDMARKER: if is_fmt_on(container, preview=preview): return @@ -246,6 +235,51 @@ def generate_ignored_nodes( container = container.next_sibling +def _generate_ignored_nodes_from_fmt_skip( + leaf: Leaf, comment: ProtoComment, *, preview: bool +) -> Iterator[LN]: + """Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`.""" + prev_sibling = leaf.prev_sibling + parent = leaf.parent + # Need to properly format the leaf prefix to compare it to comment.value, + # which is also formatted + comments = list_comments(leaf.prefix, is_endmarker=False, preview=preview) + if not comments or comment.value != comments[0].value: + return + if prev_sibling is not None: + leaf.prefix = "" + siblings = [prev_sibling] + while "\n" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None: + prev_sibling = prev_sibling.prev_sibling + siblings.insert(0, prev_sibling) + for sibling in siblings: + yield sibling + elif ( + parent is not None + and type_repr(parent.type) == "suite" + and leaf.type == token.NEWLINE + ): + # The `# fmt: skip` is on the colon line of the if/while/def/class/... + # statements. The ignored nodes should be previous siblings of the + # parent suite node. + leaf.prefix = "" + ignored_nodes: List[LN] = [] + parent_sibling = parent.prev_sibling + while parent_sibling is not None and type_repr(parent_sibling.type) != "suite": + ignored_nodes.insert(0, parent_sibling) + parent_sibling = parent_sibling.prev_sibling + # Special case for `async_stmt` where the ASYNC token is on the + # grandparent node. + grandparent = parent.parent + if ( + grandparent is not None + and grandparent.prev_sibling is not None + and grandparent.prev_sibling.type == token.ASYNC + ): + ignored_nodes.insert(0, grandparent.prev_sibling) + yield from iter(ignored_nodes) + + def is_fmt_on(container: LN, preview: bool) -> bool: """Determine whether formatting is switched on within a container. Determined by whether the last `# fmt:` comment is `on` or `off`. diff --git a/src/black/linegen.py b/src/black/linegen.py index 1f132b7888f..8e8d41e239a 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -220,7 +220,9 @@ def visit_async_stmt(self, node: Node) -> Iterator[Line]: for child in children: yield from self.visit(child) - if child.type == token.ASYNC: + if child.type == token.ASYNC or child.type == STANDALONE_COMMENT: + # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async + # line. break internal_stmt = next(children) diff --git a/tests/data/simple_cases/fmtskip8.py b/tests/data/simple_cases/fmtskip8.py new file mode 100644 index 00000000000..38e9c2a9f47 --- /dev/null +++ b/tests/data/simple_cases/fmtskip8.py @@ -0,0 +1,62 @@ +# Make sure a leading comment is not removed. +def some_func( unformatted, args ): # fmt: skip + print("I am some_func") + return 0 + # Make sure this comment is not removed. + + +# Make sure a leading comment is not removed. +async def some_async_func( unformatted, args): # fmt: skip + print("I am some_async_func") + await asyncio.sleep(1) + + +# Make sure a leading comment is not removed. +class SomeClass( Unformatted, SuperClasses ): # fmt: skip + def some_method( self, unformatted, args ): # fmt: skip + print("I am some_method") + return 0 + + async def some_async_method( self, unformatted, args ): # fmt: skip + print("I am some_async_method") + await asyncio.sleep(1) + + +# Make sure a leading comment is not removed. +if unformatted_call( args ): # fmt: skip + print("First branch") + # Make sure this is not removed. +elif another_unformatted_call( args ): # fmt: skip + print("Second branch") +else : # fmt: skip + print("Last branch") + + +while some_condition( unformatted, args ): # fmt: skip + print("Do something") + + +for i in some_iter( unformatted, args ): # fmt: skip + print("Do something") + + +async def test_async_for(): + async for i in some_async_iter( unformatted, args ): # fmt: skip + print("Do something") + + +try : # fmt: skip + some_call() +except UnformattedError as ex: # fmt: skip + handle_exception() +finally : # fmt: skip + finally_call() + + +with give_me_context( unformatted, args ): # fmt: skip + print("Do something") + + +async def test_async_with(): + async with give_me_async_context( unformatted, args ): # fmt: skip + print("Do something") From 249c6536c4dff50773f30f222d1f81f0afe41f4c Mon Sep 17 00:00:00 2001 From: "Yilei \"Dolee\" Yang" Date: Tue, 19 Jul 2022 17:57:23 -0700 Subject: [PATCH 02/20] Fix an infinite loop when using `# fmt: on/off` ... (#3158) ... in the middle of an expression or code block by adding a missing return. Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 ++ src/black/comments.py | 10 +++++++- tests/data/simple_cases/fmtonoff5.py | 36 ++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 tests/data/simple_cases/fmtonoff5.py diff --git a/CHANGES.md b/CHANGES.md index 90c62de6b98..94c3bdda68e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,8 @@ +- Fix an infinite loop when using `# fmt: on/off` in the middle of an expression or code + block (#3158) - Fix incorrect handling of `# fmt: skip` on colon `:` lines. (#3148) - Comments are no longer deleted when a line had spaces removed around power operators (#2874) diff --git a/src/black/comments.py b/src/black/comments.py index 522c1a7b88c..7069da16528 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -13,7 +13,7 @@ from blib2to3.pgen2 import token from black.nodes import first_leaf_column, preceding_leaf, container_of -from black.nodes import STANDALONE_COMMENT, WHITESPACE +from black.nodes import CLOSING_BRACKETS, STANDALONE_COMMENT, WHITESPACE # types LN = Union[Leaf, Node] @@ -227,6 +227,14 @@ def generate_ignored_nodes( # fix for fmt: on in children if contains_fmt_on_at_column(container, leaf.column, preview=preview): for child in container.children: + if isinstance(child, Leaf) and is_fmt_on(child, preview=preview): + if child.type in CLOSING_BRACKETS: + # This means `# fmt: on` is placed at a different bracket level + # than `# fmt: off`. This is an invalid use, but as a courtesy, + # we include this closing bracket in the ignored nodes. + # The alternative is to fail the formatting. + yield child + return if contains_fmt_on_at_column(child, leaf.column, preview=preview): return yield child diff --git a/tests/data/simple_cases/fmtonoff5.py b/tests/data/simple_cases/fmtonoff5.py new file mode 100644 index 00000000000..746aa41f4e4 --- /dev/null +++ b/tests/data/simple_cases/fmtonoff5.py @@ -0,0 +1,36 @@ +# Regression test for https://github.com/psf/black/issues/3129. +setup( + entry_points={ + # fmt: off + "console_scripts": [ + "foo-bar" + "=foo.bar.:main", + # fmt: on + ] # Includes an formatted indentation. + }, +) + + +# Regression test for https://github.com/psf/black/issues/2015. +run( + # fmt: off + [ + "ls", + "-la", + ] + # fmt: on + + path, + check=True, +) + + +# Regression test for https://github.com/psf/black/issues/3026. +def test_func(): + # yapf: disable + if unformatted( args ): + return True + # yapf: enable + elif b: + return True + + return False From b4dc40bf7aab4cd8914c3a2046c8ffc71bba2ff9 Mon Sep 17 00:00:00 2001 From: "Yilei \"Dolee\" Yang" Date: Tue, 19 Jul 2022 18:33:00 -0700 Subject: [PATCH 03/20] Use underscores instead of a space in a test file's name (#3180) ... for *consistency* --- ...emove_newline_after match.py => remove_newline_after_match.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/data/preview_310/{remove_newline_after match.py => remove_newline_after_match.py} (100%) diff --git a/tests/data/preview_310/remove_newline_after match.py b/tests/data/preview_310/remove_newline_after_match.py similarity index 100% rename from tests/data/preview_310/remove_newline_after match.py rename to tests/data/preview_310/remove_newline_after_match.py From e9e756da7a68890fe36b79f711f93630758fd99d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 20:51:32 -0400 Subject: [PATCH 04/20] Bump sphinx from 5.0.2 to 5.1.0 in /docs (#3183) Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 5.0.2 to 5.1.0. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/5.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v5.0.2...v5.1.0) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 65387e05e7e..e843a68566a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ # Used by ReadTheDocs; pinned requirements for stability. myst-parser==0.18.0 -Sphinx==5.0.2 +Sphinx==5.1.0 # Older versions break Sphinx even though they're declared to be supported. docutils==0.18.1 sphinxcontrib-programoutput==0.17 From e0a780a5056f1039edcf12c7a44198be902afbbc Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Tue, 26 Jul 2022 21:32:31 -0400 Subject: [PATCH 05/20] Add isort to linting toolchain Co-authored-by: Shivansh-007 --- .pre-commit-config.yaml | 5 +++++ pyproject.toml | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6dedc44968..3a9c0ceda85 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,6 +23,11 @@ repos: files: '(CHANGES\.md|the_basics\.md)$' additional_dependencies: *version_check_dependencies + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + - repo: https://github.com/pycqa/flake8 rev: 4.0.1 hooks: diff --git a/pyproject.toml b/pyproject.toml index 6df037c8a39..36765072056 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,13 +22,21 @@ extend-exclude = ''' # this off. preview = true -# Build system information below. +# Build system information and other project-specific configuration below. # NOTE: You don't need this in your own Black configuration. [build-system] requires = ["setuptools>=45.0", "setuptools_scm[toml]>=6.3.1", "wheel"] build-backend = "setuptools.build_meta" +[tool.isort] +atomic = true +profile = "black" +line_length = 88 +skip_gitignore = true +skip_glob = ["src/blib2to3", "tests/data", "profiling"] +known_first_party = ["black", "blib2to3", "blackd", "_black_version"] + [tool.pytest.ini_options] # Option below requires `tests/optional.py` addopts = "--strict-config --strict-markers" From 44d5da00b520a05cd56e58b3998660f64ea59ebd Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Tue, 26 Jul 2022 21:33:08 -0400 Subject: [PATCH 06/20] Reformat codebase with isort --- action/main.py | 2 +- fuzz.py | 4 +- gallery/gallery.py | 10 +--- scripts/migrate-black.py | 2 +- setup.py | 5 +- src/black/__init__.py | 89 ++++++++++++++++++-------------- src/black/brackets.py | 20 ++++--- src/black/cache.py | 6 +-- src/black/comments.py | 15 ++++-- src/black/debug.py | 5 +- src/black/files.py | 8 +-- src/black/handle_ipynb_magics.py | 20 +++---- src/black/linegen.py | 73 ++++++++++++++++++-------- src/black/lines.py | 29 +++++++---- src/black/mode.py | 3 +- src/black/nodes.py | 20 ++----- src/black/output.py | 4 +- src/black/parsing.py | 8 ++- src/black/report.py | 2 +- src/black/rusty.py | 1 - src/black/trans.py | 38 +++++++++----- src/blackd/__init__.py | 5 +- src/blackd/middlewares.py | 7 +-- tests/optional.py | 6 +-- tests/test_black.py | 4 +- tests/test_blackd.py | 9 ++-- tests/test_format.py | 2 +- tests/test_ipynb.py | 15 +++--- tests/test_no_ipynb.py | 7 +-- tests/test_trans.py | 1 + 30 files changed, 235 insertions(+), 185 deletions(-) diff --git a/action/main.py b/action/main.py index d14b10f421d..cd920f5fe0d 100644 --- a/action/main.py +++ b/action/main.py @@ -2,7 +2,7 @@ import shlex import sys from pathlib import Path -from subprocess import run, PIPE, STDOUT +from subprocess import PIPE, STDOUT, run ACTION_PATH = Path(os.environ["GITHUB_ACTION_PATH"]) ENV_PATH = ACTION_PATH / ".black-env" diff --git a/fuzz.py b/fuzz.py index f5f655ea279..83e02f45152 100644 --- a/fuzz.py +++ b/fuzz.py @@ -8,7 +8,8 @@ import re import hypothesmith -from hypothesis import HealthCheck, given, settings, strategies as st +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st import black from blib2to3.pgen2.tokenize import TokenError @@ -78,6 +79,7 @@ def test_idempotent_any_syntatically_valid_python( # (if you want only bounded fuzzing, just use `pytest fuzz.py`) try: import sys + import atheris except ImportError: pass diff --git a/gallery/gallery.py b/gallery/gallery.py index be4d81dc427..38e52e34795 100755 --- a/gallery/gallery.py +++ b/gallery/gallery.py @@ -10,15 +10,7 @@ from concurrent.futures import ThreadPoolExecutor from functools import lru_cache, partial from pathlib import Path -from typing import ( - Generator, - List, - NamedTuple, - Optional, - Tuple, - Union, - cast, -) +from typing import Generator, List, NamedTuple, Optional, Tuple, Union, cast from urllib.request import urlopen, urlretrieve PYPI_INSTANCE = "https://pypi.org/pypi" diff --git a/scripts/migrate-black.py b/scripts/migrate-black.py index 1183cb8a104..63cc5096a93 100755 --- a/scripts/migrate-black.py +++ b/scripts/migrate-black.py @@ -5,7 +5,7 @@ import logging import os import sys -from subprocess import check_output, run, Popen, PIPE +from subprocess import PIPE, Popen, check_output, run def git(*args: str) -> str: diff --git a/setup.py b/setup.py index 522a42a7ce2..3accdf433bc 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ # Copyright (C) 2020 Łukasz Langa -from setuptools import setup, find_packages -import sys import os +import sys + +from setuptools import find_packages, setup assert sys.version_info >= (3, 6, 2), "black requires Python 3.6.2+" from pathlib import Path # noqa E402 diff --git a/src/black/__init__.py b/src/black/__init__.py index d2df1cbee7c..2a5c750a583 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1,20 +1,20 @@ import asyncio -from json.decoder import JSONDecodeError -import json -from contextlib import contextmanager -from datetime import datetime -from enum import Enum import io -from multiprocessing import Manager, freeze_support +import json import os -from pathlib import Path -from pathspec.patterns.gitwildmatch import GitWildMatchPatternError import platform import re import signal import sys import tokenize import traceback +from contextlib import contextmanager +from dataclasses import replace +from datetime import datetime +from enum import Enum +from json.decoder import JSONDecodeError +from multiprocessing import Manager, freeze_support +from pathlib import Path from typing import ( TYPE_CHECKING, Any, @@ -34,48 +34,61 @@ import click from click.core import ParameterSource -from dataclasses import replace from mypy_extensions import mypyc_attr +from pathspec.patterns.gitwildmatch import GitWildMatchPatternError -from black.const import DEFAULT_LINE_LENGTH, DEFAULT_INCLUDES, DEFAULT_EXCLUDES -from black.const import STDIN_PLACEHOLDER -from black.nodes import STARS, syms, is_simple_decorator_expression -from black.nodes import is_string_token, is_number_token -from black.lines import Line, EmptyLineTracker -from black.linegen import transform_line, LineGenerator, LN +from _black_version import version as __version__ +from black.cache import Cache, filter_cached, get_cache_info, read_cache, write_cache from black.comments import normalize_fmt_off -from black.mode import FUTURE_FLAG_TO_FEATURE, Mode, TargetVersion -from black.mode import Feature, supports_feature, VERSION_TO_FEATURES -from black.cache import read_cache, write_cache, get_cache_info, filter_cached, Cache -from black.concurrency import cancel, shutdown, maybe_install_uvloop -from black.output import dump_to_file, ipynb_diff, diff, color_diff, out, err -from black.report import Report, Changed, NothingChanged +from black.concurrency import cancel, maybe_install_uvloop, shutdown +from black.const import ( + DEFAULT_EXCLUDES, + DEFAULT_INCLUDES, + DEFAULT_LINE_LENGTH, + STDIN_PLACEHOLDER, +) from black.files import ( find_project_root, find_pyproject_toml, - parse_pyproject_toml, find_user_pyproject_toml, + gen_python_files, + get_gitignore, + normalize_path_maybe_ignore, + parse_pyproject_toml, + wrap_stream_for_windows, ) -from black.files import gen_python_files, get_gitignore, normalize_path_maybe_ignore -from black.files import wrap_stream_for_windows -from black.parsing import InvalidInput # noqa F401 -from black.parsing import lib2to3_parse, parse_ast, stringify_ast from black.handle_ipynb_magics import ( - mask_cell, - unmask_cell, - remove_trailing_semicolon, - put_trailing_semicolon_back, - TRANSFORMED_MAGICS, PYTHON_CELL_MAGICS, + TRANSFORMED_MAGICS, jupyter_dependencies_are_installed, + mask_cell, + put_trailing_semicolon_back, + remove_trailing_semicolon, + unmask_cell, ) - - -# lib2to3 fork -from blib2to3.pytree import Node, Leaf +from black.linegen import LN, LineGenerator, transform_line +from black.lines import EmptyLineTracker, Line +from black.mode import ( + FUTURE_FLAG_TO_FEATURE, + VERSION_TO_FEATURES, + Feature, + Mode, + TargetVersion, + supports_feature, +) +from black.nodes import ( + STARS, + is_number_token, + is_simple_decorator_expression, + is_string_token, + syms, +) +from black.output import color_diff, diff, dump_to_file, err, ipynb_diff, out +from black.parsing import InvalidInput # noqa F401 +from black.parsing import lib2to3_parse, parse_ast, stringify_ast +from black.report import Changed, NothingChanged, Report from blib2to3.pgen2 import token - -from _black_version import version as __version__ +from blib2to3.pytree import Leaf, Node if TYPE_CHECKING: from concurrent.futures import Executor @@ -770,7 +783,7 @@ def reformat_many( workers: Optional[int], ) -> None: """Reformat multiple files using a ProcessPoolExecutor.""" - from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor + from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor executor: Executor worker_count = workers if workers is not None else DEFAULT_WORKERS diff --git a/src/black/brackets.py b/src/black/brackets.py index c5ed4bf5b9f..3566f5b6c37 100644 --- a/src/black/brackets.py +++ b/src/black/brackets.py @@ -1,7 +1,7 @@ """Builds on top of nodes.py to track brackets.""" -from dataclasses import dataclass, field import sys +from dataclasses import dataclass, field from typing import Dict, Iterable, List, Optional, Tuple, Union if sys.version_info < (3, 8): @@ -9,12 +9,20 @@ else: from typing import Final -from blib2to3.pytree import Leaf, Node +from black.nodes import ( + BRACKET, + CLOSING_BRACKETS, + COMPARATORS, + LOGIC_OPERATORS, + MATH_OPERATORS, + OPENING_BRACKETS, + UNPACKING_PARENTS, + VARARGS_PARENTS, + is_vararg, + syms, +) from blib2to3.pgen2 import token - -from black.nodes import syms, is_vararg, VARARGS_PARENTS, UNPACKING_PARENTS -from black.nodes import BRACKET, OPENING_BRACKETS, CLOSING_BRACKETS -from black.nodes import MATH_OPERATORS, COMPARATORS, LOGIC_OPERATORS +from blib2to3.pytree import Leaf, Node # types LN = Union[Leaf, Node] diff --git a/src/black/cache.py b/src/black/cache.py index 552c248d2ad..9455ff44772 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -2,16 +2,14 @@ import os import pickle -from pathlib import Path import tempfile +from pathlib import Path from typing import Dict, Iterable, Set, Tuple from platformdirs import user_cache_dir -from black.mode import Mode - from _black_version import version as __version__ - +from black.mode import Mode # types Timestamp = float diff --git a/src/black/comments.py b/src/black/comments.py index 7069da16528..dc58934f9d3 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -1,7 +1,7 @@ +import re import sys from dataclasses import dataclass from functools import lru_cache -import re from typing import Iterator, List, Optional, Union if sys.version_info >= (3, 8): @@ -9,11 +9,16 @@ else: from typing_extensions import Final -from blib2to3.pytree import Node, Leaf, type_repr +from black.nodes import ( + CLOSING_BRACKETS, + STANDALONE_COMMENT, + WHITESPACE, + container_of, + first_leaf_column, + preceding_leaf, +) from blib2to3.pgen2 import token - -from black.nodes import first_leaf_column, preceding_leaf, container_of -from black.nodes import CLOSING_BRACKETS, STANDALONE_COMMENT, WHITESPACE +from blib2to3.pytree import Leaf, Node, type_repr # types LN = Union[Leaf, Node] diff --git a/src/black/debug.py b/src/black/debug.py index 5143076ab35..150b44842dd 100644 --- a/src/black/debug.py +++ b/src/black/debug.py @@ -1,12 +1,11 @@ from dataclasses import dataclass from typing import Iterator, TypeVar, Union -from blib2to3.pytree import Node, Leaf, type_repr -from blib2to3.pgen2 import token - from black.nodes import Visitor from black.output import out from black.parsing import lib2to3_parse +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node, type_repr LN = Union[Leaf, Node] T = TypeVar("T") diff --git a/src/black/files.py b/src/black/files.py index 0382397e8a2..17515d52b57 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -1,9 +1,10 @@ -from functools import lru_cache import io import os -from pathlib import Path import sys +from functools import lru_cache +from pathlib import Path from typing import ( + TYPE_CHECKING, Any, Dict, Iterable, @@ -14,7 +15,6 @@ Sequence, Tuple, Union, - TYPE_CHECKING, ) from mypy_extensions import mypyc_attr @@ -30,9 +30,9 @@ else: import tomli as tomllib +from black.handle_ipynb_magics import jupyter_dependencies_are_installed from black.output import err from black.report import Report -from black.handle_ipynb_magics import jupyter_dependencies_are_installed if TYPE_CHECKING: import colorama # noqa: F401 diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index a0ed56baafc..693f1a68bd4 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -1,22 +1,20 @@ """Functions to process IPython magics with.""" -from functools import lru_cache -import dataclasses import ast -from typing import Dict, List, Tuple, Optional - +import collections +import dataclasses import secrets import sys -import collections +from functools import lru_cache +from typing import Dict, List, Optional, Tuple if sys.version_info >= (3, 10): from typing import TypeGuard else: from typing_extensions import TypeGuard -from black.report import NothingChanged from black.output import out - +from black.report import NothingChanged TRANSFORMED_MAGICS = frozenset( ( @@ -90,11 +88,7 @@ def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses ``tokenize_rt`` so that round-tripping works fine. """ - from tokenize_rt import ( - src_to_tokens, - tokens_to_src, - reversed_enumerate, - ) + from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src tokens = src_to_tokens(src) trailing_semicolon = False @@ -118,7 +112,7 @@ def put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str: """ if not has_trailing_semicolon: return src - from tokenize_rt import src_to_tokens, tokens_to_src, reversed_enumerate + from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src tokens = src_to_tokens(src) for idx, token in reversed_enumerate(tokens): diff --git a/src/black/linegen.py b/src/black/linegen.py index 8e8d41e239a..a2e41bf5912 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1,38 +1,67 @@ """ Generating lines of code. """ -from functools import partial, wraps import sys +from functools import partial, wraps from typing import Collection, Iterator, List, Optional, Set, Union, cast -from black.nodes import WHITESPACE, RARROW, STATEMENT, STANDALONE_COMMENT -from black.nodes import ASSIGNMENTS, OPENING_BRACKETS, CLOSING_BRACKETS -from black.nodes import Visitor, syms, is_arith_like, ensure_visible +from black.brackets import COMMA_PRIORITY, DOT_PRIORITY, max_delimiter_priority_in_atom +from black.comments import FMT_OFF, generate_comments, list_comments +from black.lines import ( + Line, + append_leaves, + can_be_split, + can_omit_invisible_parens, + is_line_short_enough, + line_to_string, +) +from black.mode import Feature, Mode, Preview from black.nodes import ( + ASSIGNMENTS, + CLOSING_BRACKETS, + OPENING_BRACKETS, + RARROW, + STANDALONE_COMMENT, + STATEMENT, + WHITESPACE, + Visitor, + ensure_visible, + is_arith_like, + is_atom_with_invisible_parens, is_docstring, is_empty_tuple, - is_one_tuple, + is_lpar_token, + is_multiline_string, + is_name_token, is_one_sequence_between, + is_one_tuple, + is_rpar_token, + is_stub_body, + is_stub_suite, + is_vararg, + is_walrus_assignment, + is_yield, + syms, + wrap_in_parentheses, ) -from black.nodes import is_name_token, is_lpar_token, is_rpar_token -from black.nodes import is_walrus_assignment, is_yield, is_vararg, is_multiline_string -from black.nodes import is_stub_suite, is_stub_body, is_atom_with_invisible_parens -from black.nodes import wrap_in_parentheses -from black.brackets import max_delimiter_priority_in_atom -from black.brackets import DOT_PRIORITY, COMMA_PRIORITY -from black.lines import Line, line_to_string, is_line_short_enough -from black.lines import can_omit_invisible_parens, can_be_split, append_leaves -from black.comments import generate_comments, list_comments, FMT_OFF from black.numerics import normalize_numeric_literal -from black.strings import get_string_prefix, fix_docstring -from black.strings import normalize_string_prefix, normalize_string_quotes -from black.trans import Transformer, CannotTransform, StringMerger, StringSplitter -from black.trans import StringParenWrapper, StringParenStripper, hug_power_op -from black.mode import Mode, Feature, Preview - -from blib2to3.pytree import Node, Leaf +from black.strings import ( + fix_docstring, + get_string_prefix, + normalize_string_prefix, + normalize_string_quotes, +) +from black.trans import ( + CannotTransform, + StringMerger, + StringParenStripper, + StringParenWrapper, + StringSplitter, + Transformer, + hug_power_op, +) from blib2to3.pgen2 import token - +from blib2to3.pytree import Leaf, Node # types LeafID = int diff --git a/src/black/lines.py b/src/black/lines.py index 8b591c324a5..1ebc7808901 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -1,6 +1,6 @@ -from dataclasses import dataclass, field import itertools import sys +from dataclasses import dataclass, field from typing import ( Callable, Dict, @@ -13,16 +13,25 @@ cast, ) -from blib2to3.pytree import Node, Leaf -from blib2to3.pgen2 import token - -from black.brackets import BracketTracker, DOT_PRIORITY +from black.brackets import DOT_PRIORITY, BracketTracker from black.mode import Mode, Preview -from black.nodes import STANDALONE_COMMENT, TEST_DESCENDANTS -from black.nodes import BRACKETS, OPENING_BRACKETS, CLOSING_BRACKETS -from black.nodes import syms, whitespace, replace_child, child_towards -from black.nodes import is_multiline_string, is_import, is_type_comment -from black.nodes import is_one_sequence_between +from black.nodes import ( + BRACKETS, + CLOSING_BRACKETS, + OPENING_BRACKETS, + STANDALONE_COMMENT, + TEST_DESCENDANTS, + child_towards, + is_import, + is_multiline_string, + is_one_sequence_between, + is_type_comment, + replace_child, + syms, + whitespace, +) +from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node # types T = TypeVar("T") diff --git a/src/black/mode.py b/src/black/mode.py index b7359fab213..32b65d16ca5 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -4,11 +4,10 @@ chosen by the user. """ -from hashlib import sha256 import sys - from dataclasses import dataclass, field from enum import Enum, auto +from hashlib import sha256 from operator import attrgetter from typing import Dict, Set from warnings import warn diff --git a/src/black/nodes.py b/src/black/nodes.py index 12f24b96687..8f341ab35d6 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -3,16 +3,7 @@ """ import sys -from typing import ( - Generic, - Iterator, - List, - Optional, - Set, - Tuple, - TypeVar, - Union, -) +from typing import Generic, Iterator, List, Optional, Set, Tuple, TypeVar, Union if sys.version_info >= (3, 8): from typing import Final @@ -25,14 +16,11 @@ from mypy_extensions import mypyc_attr -# lib2to3 fork -from blib2to3.pytree import Node, Leaf, type_repr, NL -from blib2to3 import pygram -from blib2to3.pgen2 import token - from black.cache import CACHE_DIR from black.strings import has_triple_quotes - +from blib2to3 import pygram +from blib2to3.pgen2 import token +from blib2to3.pytree import NL, Leaf, Node, type_repr pygram.initialize(CACHE_DIR) syms: Final = pygram.python_symbols diff --git a/src/black/output.py b/src/black/output.py index 9561d4b57d2..f4c17f28ea4 100644 --- a/src/black/output.py +++ b/src/black/output.py @@ -4,11 +4,11 @@ """ import json -from typing import Any, Optional -from mypy_extensions import mypyc_attr import tempfile +from typing import Any, Optional from click import echo, style +from mypy_extensions import mypyc_attr @mypyc_attr(patchable=True) diff --git a/src/black/parsing.py b/src/black/parsing.py index d1ad7d2c671..64c0b1e3018 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -11,16 +11,14 @@ else: from typing import Final -# lib2to3 fork -from blib2to3.pytree import Node, Leaf +from black.mode import Feature, TargetVersion, supports_feature +from black.nodes import syms from blib2to3 import pygram from blib2to3.pgen2 import driver from blib2to3.pgen2.grammar import Grammar from blib2to3.pgen2.parse import ParseError from blib2to3.pgen2.tokenize import TokenError - -from black.mode import TargetVersion, Feature, supports_feature -from black.nodes import syms +from blib2to3.pytree import Leaf, Node ast3: Any diff --git a/src/black/report.py b/src/black/report.py index 43b942c9e3c..a507671e4c0 100644 --- a/src/black/report.py +++ b/src/black/report.py @@ -7,7 +7,7 @@ from click import style -from black.output import out, err +from black.output import err, out class Changed(Enum): diff --git a/src/black/rusty.py b/src/black/rusty.py index 822e3d7858a..84a80b5a2c2 100644 --- a/src/black/rusty.py +++ b/src/black/rusty.py @@ -4,7 +4,6 @@ """ from typing import Generic, TypeVar, Union - T = TypeVar("T") E = TypeVar("E", bound=Exception) diff --git a/src/black/trans.py b/src/black/trans.py index 28d9250adc1..dc9c5520d5b 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -1,10 +1,11 @@ """ String transformers that can split and merge strings. """ +import re +import sys from abc import ABC, abstractmethod from collections import defaultdict from dataclasses import dataclass -import re from typing import ( Any, Callable, @@ -21,29 +22,38 @@ TypeVar, Union, ) -import sys if sys.version_info < (3, 8): - from typing_extensions import Literal, Final + from typing_extensions import Final, Literal else: from typing import Literal, Final from mypy_extensions import trait -from black.rusty import Result, Ok, Err - -from black.mode import Feature -from black.nodes import syms, replace_child, parent_type -from black.nodes import is_empty_par, is_empty_lpar, is_empty_rpar -from black.nodes import OPENING_BRACKETS, CLOSING_BRACKETS, STANDALONE_COMMENT -from black.lines import Line, append_leaves from black.brackets import BracketMatchError from black.comments import contains_pragma_comment -from black.strings import has_triple_quotes, get_string_prefix, assert_is_leaf_string -from black.strings import normalize_string_quotes - -from blib2to3.pytree import Leaf, Node +from black.lines import Line, append_leaves +from black.mode import Feature +from black.nodes import ( + CLOSING_BRACKETS, + OPENING_BRACKETS, + STANDALONE_COMMENT, + is_empty_lpar, + is_empty_par, + is_empty_rpar, + parent_type, + replace_child, + syms, +) +from black.rusty import Err, Ok, Result +from black.strings import ( + assert_is_leaf_string, + get_string_prefix, + has_triple_quotes, + normalize_string_quotes, +) from blib2to3.pgen2 import token +from blib2to3.pytree import Leaf, Node class CannotTransform(Exception): diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index 0463f169e19..a6de79fbeaa 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -8,6 +8,7 @@ try: from aiohttp import web + from .middlewares import cors except ImportError as ie: raise ImportError( @@ -16,11 +17,11 @@ + "to obtain aiohttp_cors: `pip install black[d]`" ) from None -import black -from black.concurrency import maybe_install_uvloop import click +import black from _black_version import version as __version__ +from black.concurrency import maybe_install_uvloop # This is used internally by tests to shut down the server prematurely _stop_signal = asyncio.Event() diff --git a/src/blackd/middlewares.py b/src/blackd/middlewares.py index 97994ecc1df..7abde525bfd 100644 --- a/src/blackd/middlewares.py +++ b/src/blackd/middlewares.py @@ -1,7 +1,8 @@ -from typing import Iterable, Awaitable, Callable -from aiohttp.web_response import StreamResponse -from aiohttp.web_request import Request +from typing import Awaitable, Callable, Iterable + from aiohttp.web_middlewares import middleware +from aiohttp.web_request import Request +from aiohttp.web_response import StreamResponse Handler = Callable[[Request], Awaitable[StreamResponse]] Middleware = Callable[[Request, Handler], Awaitable[StreamResponse]] diff --git a/tests/optional.py b/tests/optional.py index a4e9441ef1c..853ecaa2a43 100644 --- a/tests/optional.py +++ b/tests/optional.py @@ -14,11 +14,11 @@ Adapted from https://pypi.org/project/pytest-optional-tests/, (c) 2019 Reece Hart """ -from functools import lru_cache import itertools import logging import re -from typing import FrozenSet, List, Set, TYPE_CHECKING +from functools import lru_cache +from typing import TYPE_CHECKING, FrozenSet, List, Set import pytest @@ -32,8 +32,8 @@ if TYPE_CHECKING: - from _pytest.config.argparsing import Parser from _pytest.config import Config + from _pytest.config.argparsing import Parser from _pytest.mark.structures import MarkDecorator from _pytest.nodes import Node diff --git a/tests/test_black.py b/tests/test_black.py index 8adcaed5ef8..bb7784d5478 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -6,6 +6,7 @@ import logging import multiprocessing import os +import re import sys import types import unittest @@ -31,7 +32,6 @@ import click import pytest -import re from click import unstyle from click.testing import CliRunner from pathspec import PathSpec @@ -59,8 +59,8 @@ dump_to_stderr, ff, fs, - read_data, get_case_path, + read_data, read_data_from_file, ) diff --git a/tests/test_blackd.py b/tests/test_blackd.py index 75d756705be..18b2c98ac1f 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -2,15 +2,16 @@ from typing import Any from unittest.mock import patch -from click.testing import CliRunner import pytest +from click.testing import CliRunner -from tests.util import read_data, DETERMINISTIC_HEADER +from tests.util import DETERMINISTIC_HEADER, read_data try: - import blackd - from aiohttp.test_utils import AioHTTPTestCase from aiohttp import web + from aiohttp.test_utils import AioHTTPTestCase + + import blackd except ImportError as e: raise RuntimeError("Please install Black with the 'd' extra") from e diff --git a/tests/test_format.py b/tests/test_format.py index 7a099fb9f33..3645934721f 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -8,10 +8,10 @@ from tests.util import ( DEFAULT_MODE, PY36_VERSIONS, + all_data_cases, assert_format, dump_to_stderr, read_data, - all_data_cases, ) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index e1d7dd88dcb..7aa2e91dd00 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -1,23 +1,24 @@ import contextlib -from dataclasses import replace import pathlib import re from contextlib import ExitStack as does_not_raise +from dataclasses import replace from typing import ContextManager +import pytest +from _pytest.monkeypatch import MonkeyPatch from click.testing import CliRunner -from black.handle_ipynb_magics import jupyter_dependencies_are_installed + from black import ( - main, + Mode, NothingChanged, format_cell, format_file_contents, format_file_in_place, + main, ) -import pytest -from black import Mode -from _pytest.monkeypatch import MonkeyPatch -from tests.util import DATA_DIR, read_jupyter_notebook, get_case_path +from black.handle_ipynb_magics import jupyter_dependencies_are_installed +from tests.util import DATA_DIR, get_case_path, read_jupyter_notebook with contextlib.suppress(ModuleNotFoundError): import IPython diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index a3c897760fb..3e0b1593bf0 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -1,10 +1,11 @@ -import pytest import pathlib -from tests.util import get_case_path -from black import main, jupyter_dependencies_are_installed +import pytest from click.testing import CliRunner +from black import jupyter_dependencies_are_installed, main +from tests.util import get_case_path + pytestmark = pytest.mark.no_jupyter runner = CliRunner() diff --git a/tests/test_trans.py b/tests/test_trans.py index a1666a9c166..dce8a939677 100644 --- a/tests/test_trans.py +++ b/tests/test_trans.py @@ -1,4 +1,5 @@ from typing import List, Tuple + from black.trans import iter_fexpr_spans From 411ed778d53244a9d0b9c1913266fd03aee89123 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Wed, 27 Jul 2022 22:04:14 -0400 Subject: [PATCH 07/20] Bump pre-commit hooks (#3191) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a9c0ceda85..87bb6e62987 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,7 +38,7 @@ repos: - flake8-simplify - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.940 + rev: v0.971 hooks: - id: mypy exclude: ^docs/conf.py @@ -51,13 +51,13 @@ repos: - platformdirs >= 2.1.0 - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.5.1 + rev: v2.7.1 hooks: - id: prettier exclude: \.github/workflows/diff_shades\.yml - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.3.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace From ef8deb6d4a729192d7b7818d91530d462e769b7d Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:55:36 -0400 Subject: [PATCH 08/20] Consolidate test CI and add concurrency limits (#3189) --- .github/workflows/doc.yml | 2 +- .github/workflows/fuzz.yml | 4 ++ .github/workflows/lint.yml | 6 +-- .github/workflows/test.yml | 64 +++++++++++++++++++++---------- .github/workflows/uvloop_test.yml | 50 ------------------------ 5 files changed, 52 insertions(+), 74 deletions(-) delete mode 100644 .github/workflows/uvloop_test.yml diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 97f5f01e1b5..fc94dea62d9 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -1,4 +1,4 @@ -name: Documentation Build +name: Documentation on: [push, pull_request] diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 4ee6c839b48..a2810e25f77 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -2,6 +2,10 @@ name: Fuzz on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + permissions: contents: read diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1dd5ab5d35e..90c48013080 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python + - name: Set up latest Python uses: actions/setup-python@v4 with: python-version: "*" @@ -27,9 +27,9 @@ jobs: python -m pip install -e '.[d]' python -m pip install tox - - name: Lint + - name: Run pre-commit hooks uses: pre-commit/action@v3.0.0 - - name: Run On Self + - name: Format ourselves run: | tox -e run_self diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b4716c5493..7cc55d1bf76 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,8 +11,15 @@ on: - "docs/**" - "*.md" +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: - build: + main: # We want to run on external PRs, but not on our own internal PRs as they'll be run # by the push to the branch. Without this if check, checks are duplicated since # internal PRs match both the push and pull_request events. @@ -35,29 +42,23 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install tox run: | python -m pip install --upgrade pip python -m pip install --upgrade tox - name: Unit tests if: "!startsWith(matrix.python-version, 'pypy')" - run: | - tox -e ci-py -- -v --color=yes + run: tox -e ci-py -- -v --color=yes - - name: Unit tests pypy + - name: Unit tests (pypy) if: "startsWith(matrix.python-version, 'pypy')" - run: | - tox -e ci-pypy3 -- -v --color=yes + run: tox -e ci-pypy3 -- -v --color=yes - - name: Publish coverage to Coveralls - # If pushed / is a pull request against main repo AND + - name: Upload coverage to Coveralls + # Upload coverage if we are on the main repository and # we're running on Linux (this action only supports Linux) - if: - ((github.event_name == 'push' && github.repository == 'psf/black') || - github.event.pull_request.base.repo.full_name == 'psf/black') && matrix.os == - 'ubuntu-latest' - + if: github.repository == 'psf/black' && matrix.os == 'ubuntu-latest' uses: AndreMiras/coveralls-python-action@v20201129 with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -66,17 +67,40 @@ jobs: debug: true coveralls-finish: - needs: build - # If pushed / is a pull request against main repo - if: - (github.event_name == 'push' && github.repository == 'psf/black') || - github.event.pull_request.base.repo.full_name == 'psf/black' + needs: main + if: github.repository == 'psf/black' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Coveralls finished + - name: Send finished signal to Coveralls uses: AndreMiras/coveralls-python-action@v20201129 with: parallel-finished: true debug: true + + uvloop: + if: + github.event_name == 'push' || github.event.pull_request.head.repo.full_name != + github.repository + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest] + + steps: + - uses: actions/checkout@v3 + + - name: Set up latest Python + uses: actions/setup-python@v4 + with: + python-version: "*" + + - name: Install black with uvloop + run: | + python -m pip install pip --upgrade --disable-pip-version-check + python -m pip install -e ".[uvloop]" + + - name: Format ourselves + run: python -m black --check src/ diff --git a/.github/workflows/uvloop_test.yml b/.github/workflows/uvloop_test.yml deleted file mode 100644 index 9f247826969..00000000000 --- a/.github/workflows/uvloop_test.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: test uvloop - -on: - push: - paths-ignore: - - "docs/**" - - "*.md" - - pull_request: - paths-ignore: - - "docs/**" - - "*.md" - -permissions: - contents: read - -jobs: - build: - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. Without this if check, checks are duplicated since - # internal PRs match both the push and pull_request events. - if: - github.event_name == 'push' || github.event.pull_request.head.repo.full_name != - github.repository - - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macOS-latest] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: "*" - - - name: Install latest pip - run: | - python -m pip install --upgrade pip - - - name: Test uvloop Extra Install - run: | - python -m pip install -e ".[uvloop]" - - - name: Format ourselves - run: | - python -m black --check src/ From 4f1772e2aed8356e57b923eacf45f813ec3324a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Bastian?= Date: Fri, 29 Jul 2022 01:49:00 +0200 Subject: [PATCH 09/20] Vim plugin: prefix messages with "Black: " (#3194) As mentioned in GH-3185, when using Black as a Vim plugin, especially automatically on save, the plugin's messages can be confusing, as nothing indicates that they come from Black. --- CHANGES.md | 2 ++ autoload/black.vim | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 94c3bdda68e..e027b2cae71 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -46,6 +46,8 @@ +- Vim plugin: prefix messages with `Black: ` so it's clear they come from Black (#3194) + ### Output diff --git a/autoload/black.vim b/autoload/black.vim index 6c381b431a3..ed657be7bd3 100644 --- a/autoload/black.vim +++ b/autoload/black.vim @@ -158,9 +158,9 @@ def Black(**kwargs): ) except black.NothingChanged: if not quiet: - print(f'Already well formatted, good job. (took {time.time() - start:.4f}s)') + print(f'Black: already well formatted, good job. (took {time.time() - start:.4f}s)') except Exception as exc: - print(exc) + print(f'Black: {exc}') else: current_buffer = vim.current.window.buffer cursors = [] @@ -177,7 +177,7 @@ def Black(**kwargs): except vim.error: window.cursor = (len(window.buffer), 0) if not quiet: - print(f'Reformatted in {time.time() - start:.4f}s.') + print(f'Black: reformatted in {time.time() - start:.4f}s.') def get_configs(): filename = vim.eval("@%") From d85cf00ee80f00b25a819afef7f466dc871fa68d Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Fri, 29 Jul 2022 23:28:43 -0400 Subject: [PATCH 10/20] Remove blib2to3 grammar cache logging (#3193) As error logs are emitted often (they happen when Black's cache directory is created after blib2to3 tries to write its cache) and cause issues to be filed by users who think Black isn't working correctly. These errors are expected for now and aren't a cause for concern so let's remove them to stop worrying users (and new issues from being opened). We can improve the blib2to3 caching mechanism to write its cache at the end of a successful command line invocation later. --- CHANGES.md | 2 ++ src/blib2to3/pgen2/driver.py | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e027b2cae71..a30ac7f25e1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -54,6 +54,8 @@ - Change from deprecated `asyncio.get_event_loop()` to create our event loop which removes DeprecationWarning (#3164) +- Remove logging from internal `blib2to3` library since it regularily emits error logs + about failed caching that can and should be ignored (#3193) ### Packaging diff --git a/src/blib2to3/pgen2/driver.py b/src/blib2to3/pgen2/driver.py index 8fe820651da..daf271dfa9a 100644 --- a/src/blib2to3/pgen2/driver.py +++ b/src/blib2to3/pgen2/driver.py @@ -263,14 +263,13 @@ def load_grammar( logger = logging.getLogger(__name__) gp = _generate_pickle_name(gt) if gp is None else gp if force or not _newer(gp, gt): - logger.info("Generating grammar tables from %s", gt) g: grammar.Grammar = pgen.generate_grammar(gt) if save: - logger.info("Writing grammar tables to %s", gp) try: g.dump(gp) - except OSError as e: - logger.info("Writing failed: %s", e) + except OSError: + # Ignore error, caching is not vital. + pass else: g = grammar.Grammar() g.load(gp) From eaa048925e4443cc0e2b57b795f2852bedb4287f Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Fri, 29 Jul 2022 23:38:39 -0400 Subject: [PATCH 11/20] Add sanity check to executable CD + more (#3190) Building executables without any testing is quite sketchy, let's at least verify they won't crash on startup and format Black's own codebase. Also replaced "binaries" with "executables" since it's clearer and won't be confused with mypyc. Finally, I added colorama so all Windows users can get colour. --- .github/workflows/upload_binary.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/upload_binary.yml b/.github/workflows/upload_binary.yml index ed5ed961e67..22535a64c67 100644 --- a/.github/workflows/upload_binary.yml +++ b/.github/workflows/upload_binary.yml @@ -1,16 +1,14 @@ -name: Upload self-contained binaries +name: Publish executables on: release: types: [published] permissions: - contents: read + contents: write # actions/upload-release-asset needs this. jobs: build: - permissions: - contents: write # for actions/upload-release-asset to upload release asset runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -38,15 +36,21 @@ jobs: with: python-version: "*" - - name: Install dependencies + - name: Install Black and PyInstaller run: | - python -m pip install --upgrade pip wheel setuptools - python -m pip install . + python -m pip install --upgrade pip wheel + python -m pip install .[colorama] python -m pip install pyinstaller - - name: Build binary + - name: Build executable with PyInstaller + run: > + python -m PyInstaller -F --name ${{ matrix.asset_name }} --add-data + 'src/blib2to3${{ matrix.pathsep }}blib2to3' src/black/__main__.py + + - name: Quickly test executable run: | - python -m PyInstaller -F --name ${{ matrix.asset_name }} --add-data 'src/blib2to3${{ matrix.pathsep }}blib2to3' src/black/__main__.py + ./dist/${{ matrix.asset_name }} --version + ./dist/${{ matrix.asset_name }} src --verbose - name: Upload binary as release asset uses: actions/upload-release-asset@v1 From ca0dbb8fa6cca8c1fc2650cde9e71402c03a3324 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Sun, 31 Jul 2022 10:34:29 -0400 Subject: [PATCH 12/20] Move fuzz.py to scripts/ (#3199) --- fuzz.py => scripts/fuzz.py | 0 tox.ini | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename fuzz.py => scripts/fuzz.py (100%) diff --git a/fuzz.py b/scripts/fuzz.py similarity index 100% rename from fuzz.py rename to scripts/fuzz.py diff --git a/tox.ini b/tox.ini index 7af9e48d6f0..51ff4872db0 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,7 @@ deps = commands = pip install -e .[d] coverage erase - coverage run fuzz.py + coverage run {toxinidir}/scripts/fuzz.py coverage report [testenv:run_self] From b776bf92adb7f47cd92f550e33c2db445d226f78 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 13:51:46 -0400 Subject: [PATCH 13/20] Bump sphinx from 5.1.0 to 5.1.1 in /docs (#3201) Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/5.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v5.1.0...v5.1.1) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index e843a68566a..121df45e6c2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ # Used by ReadTheDocs; pinned requirements for stability. myst-parser==0.18.0 -Sphinx==5.1.0 +Sphinx==5.1.1 # Older versions break Sphinx even though they're declared to be supported. docutils==0.18.1 sphinxcontrib-programoutput==0.17 From f066e3fcae49554118962d2bbd9ec92a9958acf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Intrieri?= <81313286+n-borges@users.noreply.github.com> Date: Tue, 2 Aug 2022 18:01:15 +0200 Subject: [PATCH 14/20] makes install available for all users in docker image (#3202) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * makes install available for all users in docker image moves the installation path from /root/.local to a virtualenv. this way we still get the lightweight multistage build without excluding non-root users. * adds changelog entry for docker-image fix A changelog entry has been added under the Integration subheader * changes dockerfile to use the venv activate script we are now using the inbuilt venv activate script, as well as explicitly mentioning the binary location in the entrypoint cmd. Co-authored-by: Nicolò Co-authored-by: Cooper Lees --- CHANGES.md | 2 ++ Dockerfile | 12 +++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a30ac7f25e1..5b29f20bfff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -47,6 +47,8 @@ - Vim plugin: prefix messages with `Black: ` so it's clear they come from Black (#3194) +- Docker: changed to a /opt/venv installation + added to PATH to be available to + non-root users (#3202) ### Output diff --git a/Dockerfile b/Dockerfile index c393e29f632..4e8f12f9798 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,16 +2,18 @@ FROM python:3-slim AS builder RUN mkdir /src COPY . /src/ -RUN pip install --no-cache-dir --upgrade pip setuptools wheel \ +ENV VIRTUAL_ENV=/opt/venv +RUN python -m venv $VIRTUAL_ENV +RUN . /opt/venv/bin/activate && pip install --no-cache-dir --upgrade pip setuptools wheel \ # Install build tools to compile dependencies that don't have prebuilt wheels && apt update && apt install -y git build-essential \ && cd /src \ - && pip install --user --no-cache-dir .[colorama,d] + && pip install --no-cache-dir .[colorama,d] FROM python:3-slim # copy only Python packages to limit the image size -COPY --from=builder /root/.local /root/.local -ENV PATH=/root/.local/bin:$PATH +COPY --from=builder /opt/venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" -CMD ["black"] +CMD ["/opt/venv/bin/black"] From 507234c47d39f5b1d8289cdd49994e03dd97bcb4 Mon Sep 17 00:00:00 2001 From: Tom Fryers <61272761+TomFryers@users.noreply.github.com> Date: Tue, 2 Aug 2022 22:22:04 +0100 Subject: [PATCH 15/20] Remove invalid syntax in docstrings -S --preview test (#3205) uR is not a legal string prefix, so this test breaks (AssertionError: cannot use --safe with this file; failed to parse source file AST: invalid syntax) if changed to one in which the file is changed. I've changed the last test to have u alone, and added an R to the test above instead. --- .../docstring_preview_no_string_normalization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/miscellaneous/docstring_preview_no_string_normalization.py b/tests/data/miscellaneous/docstring_preview_no_string_normalization.py index 0957231eb9c..338cc01d33e 100644 --- a/tests/data/miscellaneous/docstring_preview_no_string_normalization.py +++ b/tests/data/miscellaneous/docstring_preview_no_string_normalization.py @@ -3,8 +3,8 @@ def do_not_touch_this_prefix(): def do_not_touch_this_prefix2(): - F'There was a bug where docstring prefixes would be normalized even with -S.' + FR'There was a bug where docstring prefixes would be normalized even with -S.' def do_not_touch_this_prefix3(): - uR'''There was a bug where docstring prefixes would be normalized even with -S.''' + u'''There was a bug where docstring prefixes would be normalized even with -S.''' From 6064a435453cdba47c43d71f3d0ea1aa19a29206 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 10 Aug 2022 14:29:47 -0700 Subject: [PATCH 16/20] Use debug f-strings for feature detection (#3215) Fixes GH-2907. --- CHANGES.md | 2 ++ src/black/__init__.py | 7 +++++++ src/black/mode.py | 5 +++++ tests/test_black.py | 20 ++++++++++++++++++++ 4 files changed, 34 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 5b29f20bfff..1fc8c65d6d8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -33,6 +33,8 @@ +- Black now uses the presence of debug f-strings to detect target version. (#3215) + ### Documentation +- `blackd` now supports preview style via `X-Preview` header (#3217) + ### Configuration diff --git a/docs/usage_and_configuration/black_as_a_server.md b/docs/usage_and_configuration/black_as_a_server.md index fc9d1cab716..a2d4252109a 100644 --- a/docs/usage_and_configuration/black_as_a_server.md +++ b/docs/usage_and_configuration/black_as_a_server.md @@ -54,8 +54,11 @@ The headers controlling how source code is formatted are: command line flag. If present and its value is not the empty string, no string normalization will be performed. - `X-Skip-Magic-Trailing-Comma`: corresponds to the `--skip-magic-trailing-comma` - command line flag. If present and its value is not the empty string, trailing commas + command line flag. If present and its value is not an empty string, trailing commas will not be used as a reason to split lines. +- `X-Preview`: corresponds to the `--preview` command line flag. If present and its + value is not an empty string, experimental and potentially disruptive style changes + will be used. - `X-Fast-Or-Safe`: if set to `fast`, `blackd` will act as _Black_ does when passed the `--fast` command line flag. - `X-Python-Variant`: if set to `pyi`, `blackd` will act as _Black_ does when passed the diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index a6de79fbeaa..e52a9917cf3 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -32,6 +32,7 @@ PYTHON_VARIANT_HEADER = "X-Python-Variant" SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization" SKIP_MAGIC_TRAILING_COMMA = "X-Skip-Magic-Trailing-Comma" +PREVIEW = "X-Preview" FAST_OR_SAFE_HEADER = "X-Fast-Or-Safe" DIFF_HEADER = "X-Diff" @@ -41,6 +42,7 @@ PYTHON_VARIANT_HEADER, SKIP_STRING_NORMALIZATION_HEADER, SKIP_MAGIC_TRAILING_COMMA, + PREVIEW, FAST_OR_SAFE_HEADER, DIFF_HEADER, ] @@ -109,6 +111,7 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: skip_magic_trailing_comma = bool( request.headers.get(SKIP_MAGIC_TRAILING_COMMA, False) ) + preview = bool(request.headers.get(PREVIEW, False)) fast = False if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast": fast = True @@ -118,6 +121,7 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: line_length=line_length, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, + preview=preview, ) req_bytes = await request.content.read() charset = request.charset if request.charset is not None else "utf8" diff --git a/tests/test_blackd.py b/tests/test_blackd.py index 18b2c98ac1f..1d12113a3f3 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -167,6 +167,13 @@ async def test_blackd_invalid_line_length(self) -> None: ) self.assertEqual(response.status, 400) + @unittest_run_loop + async def test_blackd_preview(self) -> None: + response = await self.client.post( + "/", data=b'print("hello")\n', headers={blackd.PREVIEW: "true"} + ) + self.assertEqual(response.status, 204) + @unittest_run_loop async def test_blackd_response_black_version_header(self) -> None: response = await self.client.post("/") From e7b967132fdbb9e2e4c4e9916530d238848ab183 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Fri, 12 Aug 2022 23:33:17 -0400 Subject: [PATCH 19/20] Port & upstream mypyc wheel build workflow (#3197) --- .github/mypyc-requirements.txt | 2 +- .github/workflows/pypi_upload.yml | 56 ++++++++++++++++++++++++++----- pyproject.toml | 52 ++++++++++++++++++++++++++++ setup.py | 5 ++- 4 files changed, 105 insertions(+), 10 deletions(-) diff --git a/.github/mypyc-requirements.txt b/.github/mypyc-requirements.txt index 4542673174c..352d36c0070 100644 --- a/.github/mypyc-requirements.txt +++ b/.github/mypyc-requirements.txt @@ -1,4 +1,4 @@ -mypy == 0.920 +mypy == 0.971 # A bunch of packages for type information mypy-extensions >= 0.4.3 diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index cda215aa5d6..31a83266345 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -1,4 +1,4 @@ -name: pypi_upload +name: Publish to PyPI on: release: @@ -8,14 +8,14 @@ permissions: contents: read jobs: - build: - name: PyPI Upload + main: + name: sdist + pure wheel runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python + - name: Set up latest Python uses: actions/setup-python@v4 with: python-version: "*" @@ -26,11 +26,51 @@ jobs: python -m pip install --upgrade build twine - name: Build wheel and source distributions - run: | - python -m build + run: python -m build - name: Upload to PyPI via Twine env: TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: | - twine upload --verbose -u '__token__' dist/* + run: twine upload --verbose -u '__token__' dist/* + + mypyc: + name: mypyc wheels (${{ matrix.name }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + name: linux-x86_64 + - os: windows-2019 + name: windows-amd64 + - os: macos-11 + name: macos-x86_64 + macos_arch: "x86_64" + - os: macos-11 + name: macos-arm64 + macos_arch: "arm64" + - os: macos-11 + name: macos-universal2 + macos_arch: "universal2" + + steps: + - uses: actions/checkout@v3 + + - name: Build wheels via cibuildwheel + uses: pypa/cibuildwheel@v2.8.1 + env: + CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" + # This isn't supported in pyproject.toml which makes sense (but is annoying). + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6.2" + + - name: Upload wheels as workflow artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.name }}-mypyc-wheels + path: ./wheelhouse/*.whl + + - name: Upload wheels to PyPI via Twine + env: + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: pipx run twine upload --verbose -u '__token__' wheelhouse/*.whl diff --git a/pyproject.toml b/pyproject.toml index 36765072056..813e86b2e93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,58 @@ preview = true requires = ["setuptools>=45.0", "setuptools_scm[toml]>=6.3.1", "wheel"] build-backend = "setuptools.build_meta" +[tool.cibuildwheel] +build-verbosity = 1 +# So these are the environments we target: +# - Python: CPython 3.6+ only +# - Architecture (64-bit only): amd64 / x86_64, universal2, and arm64 +# - OS: Linux (no musl), Windows, and macOS +build = "cp3*-*" +skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp-*"] +before-build = ["pip install -r .github/mypyc-requirements.txt"] +# This is the bare minimum needed to run the test suite. Pulling in the full +# test_requirements.txt would download a bunch of other packages not necessary +# here and would slow down the testing step a fair bit. +test-requires = ["pytest>=6.1.1"] +test-command = 'pytest {project} -k "not incompatible_with_mypyc"' +test-extras = ["d"," jupyter"] +# Skip trying to test arm64 builds on Intel Macs. (so cross-compilation doesn't +# straight up crash) +test-skip = ["*-macosx_arm64", "*-macosx_universal2:arm64"] + +[tool.cibuildwheel.environment] +BLACK_USE_MYPYC = "1" +MYPYC_OPT_LEVEL = "3" +MYPYC_DEBUG_LEVEL = "0" +# The dependencies required to build wheels with mypyc aren't specified in +# [build-system].requires so we'll have to manage the build environment ourselves. +PIP_NO_BUILD_ISOLATION = "no" + +[tool.cibuildwheel.linux] +before-build = [ + "pip install -r .github/mypyc-requirements.txt", + "yum install -y clang", +] +# Newer images break the builds, not sure why. We'll need to investigate more later. +manylinux-x86_64-image = "quay.io/pypa/manylinux2014_x86_64:2021-11-20-f410d11" + +[tool.cibuildwheel.linux.environment] +BLACK_USE_MYPYC = "1" +MYPYC_OPT_LEVEL = "3" +MYPYC_DEBUG_LEVEL = "0" +PIP_NO_BUILD_ISOLATION = "no" +# Black needs Clang to compile successfully on Linux. +CC = "clang" + +[tool.cibuildwheel.windows] +# For some reason, (compiled) mypyc is failing to start up with "ImportError: DLL load +# failed: A dynamic link library (DLL) initialization routine failed." on Windows for +# at least 3.6. Let's just use interpreted mypy[c]. +# See also: https://github.com/mypyc/mypyc/issues/819. +before-build = [ + "pip install -r .github/mypyc-requirements.txt --no-binary mypy" +] + [tool.isort] atomic = true profile = "black" diff --git a/setup.py b/setup.py index 3accdf433bc..bc0cc32352e 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,10 @@ def find_python_files(base: Path) -> List[Path]: ] opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") - ext_modules = mypycify(mypyc_targets, opt_level=opt_level, verbose=True) + debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "3") + ext_modules = mypycify( + mypyc_targets, opt_level=opt_level, debug_level=debug_level, verbose=True + ) else: ext_modules = [] From 4ebf14d17ed544be893be5706c02116fd8b83b4c Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 13 Aug 2022 06:41:34 -0700 Subject: [PATCH 20/20] Strip trailing commas in subscripts with -C (#3209) Fixes #2296, #3204 --- CHANGES.md | 2 ++ src/black/lines.py | 18 +++++++++- src/black/mode.py | 1 + .../data/preview/skip_magic_trailing_comma.py | 34 +++++++++++++++++++ tests/test_format.py | 7 +++- 5 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 tests/data/preview/skip_magic_trailing_comma.py diff --git a/CHANGES.md b/CHANGES.md index 79f4ce59187..fb7a2723b67 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,6 +24,8 @@ this is invalid. This was a bug introduced in version 22.6.0. (#3166) - `--skip-string-normalization` / `-S` now prevents docstring prefixes from being normalized as expected (#3168) +- When using `--skip-magic-trailing-comma` or `-C`, trailing commas are stripped from + subscript expressions with more than 1 element (#3209) ### _Blackd_ diff --git a/src/black/lines.py b/src/black/lines.py index 1ebc7808901..30622650d53 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -273,6 +273,8 @@ def has_magic_trailing_comma( - it's not a single-element subscript Additionally, if ensure_removable: - it's not from square bracket indexing + (specifically, single-element square bracket indexing with + Preview.skip_magic_trailing_comma_in_subscript) """ if not ( closing.type in CLOSING_BRACKETS @@ -301,8 +303,22 @@ def has_magic_trailing_comma( if not ensure_removable: return True + comma = self.leaves[-1] - return bool(comma.parent and comma.parent.type == syms.listmaker) + if comma.parent is None: + return False + if Preview.skip_magic_trailing_comma_in_subscript in self.mode: + return ( + comma.parent.type != syms.subscriptlist + or closing.opening_bracket is None + or not is_one_sequence_between( + closing.opening_bracket, + closing, + self.leaves, + brackets=(token.LSQB, token.RSQB), + ) + ) + return comma.parent.type == syms.listmaker if self.is_import: return True diff --git a/src/black/mode.py b/src/black/mode.py index f6d0cbf62bd..6c0847e8bcc 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -156,6 +156,7 @@ class Preview(Enum): remove_block_trailing_newline = auto() remove_redundant_parens = auto() string_processing = auto() + skip_magic_trailing_comma_in_subscript = auto() class Deprecated(UserWarning): diff --git a/tests/data/preview/skip_magic_trailing_comma.py b/tests/data/preview/skip_magic_trailing_comma.py new file mode 100644 index 00000000000..e98174af427 --- /dev/null +++ b/tests/data/preview/skip_magic_trailing_comma.py @@ -0,0 +1,34 @@ +# We should not remove the trailing comma in a single-element subscript. +a: tuple[int,] +b = tuple[int,] + +# But commas in multiple element subscripts should be removed. +c: tuple[int, int,] +d = tuple[int, int,] + +# Remove commas for non-subscripts. +small_list = [1,] +list_of_types = [tuple[int,],] +small_set = {1,} +set_of_types = {tuple[int,],} + +# Except single element tuples +small_tuple = (1,) + +# output +# We should not remove the trailing comma in a single-element subscript. +a: tuple[int,] +b = tuple[int,] + +# But commas in multiple element subscripts should be removed. +c: tuple[int, int] +d = tuple[int, int] + +# Remove commas for non-subscripts. +small_list = [1] +list_of_types = [tuple[int,]] +small_set = {1} +set_of_types = {tuple[int,]} + +# Except single element tuples +small_tuple = (1,) diff --git a/tests/test_format.py b/tests/test_format.py index 3645934721f..01cd61eef63 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -36,7 +36,12 @@ def test_simple_format(filename: str) -> None: @pytest.mark.parametrize("filename", all_data_cases("preview")) def test_preview_format(filename: str) -> None: - check_file("preview", filename, black.Mode(preview=True)) + magic_trailing_comma = filename != "skip_magic_trailing_comma" + check_file( + "preview", + filename, + black.Mode(preview=True, magic_trailing_comma=magic_trailing_comma), + ) @pytest.mark.parametrize("filename", all_data_cases("preview_39"))