Merge branch 'main' into strconcat

.github/workflows/test.yml

@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.7"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.7", "pypy-3.8"]
         os: [ubuntu-latest, macOS-latest, windows-latest]
 
     steps:

CHANGES.md

@@ -10,10 +10,17 @@
 
 <!-- Changes that affect Black's stable style -->
 
+- Comments are no longer deleted when a line had spaces removed around power operators
+  (#2874)
+
 ### Preview style
 
 <!-- Changes that affect Black's preview style -->
 
+- Single-character closing docstring quotes are no longer moved to their own line as
+  this is invalid. This was a bug introduced in version 22.6.0. (#3166)
+- `--skip-string-normalization` / `-S` now prevents docstring prefixes from being
+  normalized as expected (#3168)
 - Implicitly concatenated strings inside a list, set, or function call are now wrapped
   inside parentheses (#3162)
 
@@ -42,6 +49,9 @@
 
 <!-- Changes to Black's terminal output and error messages -->
 
+- Change from deprecated `asyncio.get_event_loop()` to create our event loop which
+  removes DeprecationWarning (#3164)
+
 ### Packaging
 
 <!-- Changes to how Black is packaged, such as dependency requirements -->
@@ -50,6 +60,10 @@
 
 <!-- Changes to the parser or to version autodetection -->
 
+- Type comments are now included in the AST equivalence check consistently so accidental
+  deletion raises an error. Though type comments can't be tracked when running on PyPy
+  3.7 due to standard library limitations. (#2874)
+
 ### Performance
 
 <!-- Changes that improve Black's performance. -->

docs/usage_and_configuration/black_as_a_server.md

@@ -4,6 +4,11 @@
 protocol. The main benefit of using it is to avoid the cost of starting up a new _Black_
 process every time you want to blacken a file.
 
+```{warning}
+`blackd` should not be run as a publicly accessible server as there are no security
+precautions in place to prevent abuse. **It is intended for local use only**.
+```
+
 ## Usage
 
 `blackd` is not packaged alongside _Black_ by default because it has additional

docs/usage_and_configuration/the_basics.md

@@ -265,7 +265,7 @@ extend-exclude = '''
 # in the root of the project.
 (
   ^/foo.py    # exclude a file named foo.py in the root of the project
-  | *_pb2.py  # exclude autogenerated Protocol Buffer files anywhere in the project
+  | .*_pb2.py  # exclude autogenerated Protocol Buffer files anywhere in the project
 )
 '''
 ```

pyproject.toml

@@ -31,10 +31,21 @@ build-backend = "setuptools.build_meta"
 
 [tool.pytest.ini_options]
 # Option below requires `tests/optional.py`
+addopts = "--strict-config --strict-markers"
 optional-tests = [
   "no_blackd: run when `d` extra NOT installed",
   "no_jupyter: run when `jupyter` extra NOT installed",
 ]
 markers = [
   "incompatible_with_mypyc: run when testing mypyc compiled black"
 ]
+xfail_strict = true
+filterwarnings = [
+    "error",
+    # this is mitigated by a try/catch in https://github.com/psf/black/pull/2974/
+    # this ignore can be removed when support for aiohttp 3.7 is dropped.
+    '''ignore:Decorator `@unittest_run_loop` is no longer needed in aiohttp 3\.8\+:DeprecationWarning''',
+    # this is mitigated by https://github.com/python/cpython/issues/79071 in python 3.8+
+    # this ignore can be removed when support for 3.7 is dropped.
+    '''ignore:Bare functions are deprecated, use async ones:DeprecationWarning''',
+]

src/black/__init__.py

@@ -775,7 +775,6 @@ def reformat_many(
     from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
 
     executor: Executor
-    loop = asyncio.get_event_loop()
     worker_count = workers if workers is not None else DEFAULT_WORKERS
     if sys.platform == "win32":
         # Work around https://bugs.python.org/issue26903
@@ -790,6 +789,8 @@ def reformat_many(
         # any good due to the Global Interpreter Lock)
         executor = ThreadPoolExecutor(max_workers=1)
 
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
     try:
         loop.run_until_complete(
             schedule_formatting(
@@ -803,7 +804,10 @@ def reformat_many(
             )
         )
     finally:
-        shutdown(loop)
+        try:
+            shutdown(loop)
+        finally:
+            asyncio.set_event_loop(None)
         if executor is not None:
             executor.shutdown()
 

src/black/linegen.py

@@ -293,7 +293,24 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
         if is_docstring(leaf) and "\\\n" not in leaf.value:
             # We're ignoring docstrings with backslash newline escapes because changing
             # indentation of those changes the AST representation of the code.
-            docstring = normalize_string_prefix(leaf.value)
+            if Preview.normalize_docstring_quotes_and_prefixes_properly in self.mode:
+                # There was a bug where --skip-string-normalization wouldn't stop us
+                # from normalizing docstring prefixes. To maintain stability, we can
+                # only address this buggy behaviour while the preview style is enabled.
+                if self.mode.string_normalization:
+                    docstring = normalize_string_prefix(leaf.value)
+                    # visit_default() does handle string normalization for us, but
+                    # since this method acts differently depending on quote style (ex.
+                    # see padding logic below), there's a possibility for unstable
+                    # formatting as visit_default() is called *after*. To avoid a
+                    # situation where this function formats a docstring differently on
+                    # the second pass, normalize it early.
+                    docstring = normalize_string_quotes(docstring)
+                else:
+                    docstring = leaf.value
+            else:
+                # ... otherwise, we'll keep the buggy behaviour >.<
+                docstring = normalize_string_prefix(leaf.value)
             prefix = get_string_prefix(docstring)
             docstring = docstring[len(prefix) :]  # Remove the prefix
             quote_char = docstring[0]
@@ -330,13 +347,14 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:
             # We could enforce triple quotes at this point.
             quote = quote_char * quote_len
 
-            if Preview.long_docstring_quotes_on_newline in self.mode:
+            # It's invalid to put closing single-character quotes on a new line.
+            if Preview.long_docstring_quotes_on_newline in self.mode and quote_len == 3:
                 # We need to find the length of the last line of the docstring
                 # to find if we can add the closing quotes to the line without
                 # exceeding the maximum line length.
                 # If docstring is one line, then we need to add the length
-                # of the indent, prefix, and starting quotes. Ending quote are
-                # handled later
+                # of the indent, prefix, and starting quotes. Ending quotes are
+                # handled later.
                 lines = docstring.splitlines()
                 last_line_length = len(lines[-1]) if docstring else 0
 

src/black/mode.py

@@ -145,12 +145,13 @@ def supports_feature(target_versions: Set[TargetVersion], feature: Feature) -> b
 class Preview(Enum):
     """Individual preview style features."""
 
-    string_processing = auto()
-    remove_redundant_parens = auto()
-    one_element_subscript = auto()
     annotation_parens = auto()
     long_docstring_quotes_on_newline = auto()
+    normalize_docstring_quotes_and_prefixes_properly = auto()
+    one_element_subscript = auto()
     remove_block_trailing_newline = auto()
+    remove_redundant_parens = auto()
+    string_processing = auto()
 
 
 class Deprecated(UserWarning):

src/black/parsing.py

@@ -154,14 +154,22 @@ def parse_single_version(
     src: str, version: Tuple[int, int]
 ) -> Union[ast.AST, ast3.AST]:
     filename = "<unknown>"
-    # typed_ast is needed because of feature version limitations in the builtin ast
+    # typed-ast is needed because of feature version limitations in the builtin ast 3.8>
     if sys.version_info >= (3, 8) and version >= (3,):
-        return ast.parse(src, filename, feature_version=version)
-    elif version >= (3,):
-        if _IS_PYPY:
-            return ast3.parse(src, filename)
+        return ast.parse(src, filename, feature_version=version, type_comments=True)
+
+    if _IS_PYPY:
+        # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's
+        # not much we can do as typed-ast won't work either.
+        if sys.version_info >= (3, 8):
+            return ast3.parse(src, filename, type_comments=True)
         else:
-            return ast3.parse(src, filename, feature_version=version[1])
+            return ast3.parse(src, filename)
+    else:
+        # Typed-ast is guaranteed to be used here and automatically tracks type
+        # comments separately.
+        return ast3.parse(src, filename, feature_version=version[1])
+
     raise AssertionError("INTERNAL ERROR: Tried parsing unsupported Python version!")
 
 

src/black/trans.py

@@ -121,7 +121,7 @@ def is_simple_operand(index: int, kind: Literal["base", "exponent"]) -> bool:
 
         return False
 
-    leaves: List[Leaf] = []
+    new_line = line.clone()
     should_hug = False
     for idx, leaf in enumerate(line.leaves):
         new_leaf = leaf.clone()
@@ -139,18 +139,14 @@ def is_simple_operand(index: int, kind: Literal["base", "exponent"]) -> bool:
         if should_hug:
             new_leaf.prefix = ""
 
-        leaves.append(new_leaf)
-
-    yield Line(
-        mode=line.mode,
-        depth=line.depth,
-        leaves=leaves,
-        comments=line.comments,
-        bracket_tracker=line.bracket_tracker,
-        inside_brackets=line.inside_brackets,
-        should_split_rhs=line.should_split_rhs,
-        magic_trailing_comma=line.magic_trailing_comma,
-    )
+        # We have to be careful to make a new line properly:
+        # - bracket related metadata must be maintained (handled by Line.append)
+        # - comments need to copied over, updating the leaf IDs they're attached to
+        new_line.append(new_leaf, preformatted=True)
+        for comment_leaf in line.comments_after(leaf):
+            new_line.append(comment_leaf, preformatted=True)
+
+    yield new_line
 
 
 class StringTransformer(ABC):

src/blib2to3/pytree.py

@@ -451,7 +451,6 @@ def clone(self) -> "Leaf":
             self.value,
             (self.prefix, (self.lineno, self.column)),
             fixers_applied=self.fixers_applied,
-            opening_bracket=self.opening_bracket,
         )
 
     def leaves(self) -> Iterator["Leaf"]:

tests/data/miscellaneous/docstring_preview_no_string_normalization.py

@@ -0,0 +1,10 @@
+def do_not_touch_this_prefix():
+    R"""There was a bug where docstring prefixes would be normalized even with -S."""
+
+
+def do_not_touch_this_prefix2():
+    F'There was a bug where docstring prefixes would be normalized even with -S.'
+
+
+def do_not_touch_this_prefix3():
+    uR'''There was a bug where docstring prefixes would be normalized even with -S.'''

tests/data/preview/docstring_preview.py

@@ -42,6 +42,14 @@ def multiline_docstring_at_line_limit_with_prefix():
     second line----------------------------------------------------------------------"""
 
 
+def single_quote_docstring_over_line_limit():
+    "We do not want to put the closing quote on a new line as that is invalid (see GH-3141)."
+
+
+def single_quote_docstring_over_line_limit2():
+    'We do not want to put the closing quote on a new line as that is invalid (see GH-3141).'
+
+
 # output
 
 
@@ -87,3 +95,11 @@ def multiline_docstring_at_line_limit_with_prefix():
     f"""first line----------------------------------------------------------------------
 
     second line----------------------------------------------------------------------"""
+
+
+def single_quote_docstring_over_line_limit():
+    "We do not want to put the closing quote on a new line as that is invalid (see GH-3141)."
+
+
+def single_quote_docstring_over_line_limit2():
+    "We do not want to put the closing quote on a new line as that is invalid (see GH-3141)."

tests/data/simple_cases/docstring.py

@@ -209,6 +209,13 @@ def multiline_docstring_at_line_limit():
     second line----------------------------------------------------------------------"""
 
 
+def stable_quote_normalization_with_immediate_inner_single_quote(self):
+    ''''<text here>
+
+    <text here, since without another non-empty line black is stable>
+    '''
+
+
 # output
 
 class MyClass:
@@ -417,3 +424,10 @@ def multiline_docstring_at_line_limit():
     """first line-----------------------------------------------------------------------
 
     second line----------------------------------------------------------------------"""
+
+
+def stable_quote_normalization_with_immediate_inner_single_quote(self):
+    """'<text here>
+
+    <text here, since without another non-empty line black is stable>
+    """

tests/data/simple_cases/power_op_spacing.py

@@ -49,6 +49,20 @@ def function_dont_replace_spaces():
 q = [10.5**i for i in range(6)]
 
 
+# WE SHOULD DEFINITELY NOT EAT THESE COMMENTS (https://github.com/psf/black/issues/2873)
+if hasattr(view, "sum_of_weights"):
+    return np.divide(  # type: ignore[no-any-return]
+        view.variance,  # type: ignore[union-attr]
+        view.sum_of_weights,  # type: ignore[union-attr]
+        out=np.full(view.sum_of_weights.shape, np.nan),  # type: ignore[union-attr]
+        where=view.sum_of_weights**2 > view.sum_of_weights_squared,  # type: ignore[union-attr]
+    )
+
+return np.divide(
+    where=view.sum_of_weights_of_weight_long**2 > view.sum_of_weights_squared,  # type: ignore
+)
+
+
 # output
 
 
@@ -101,3 +115,17 @@ def function_dont_replace_spaces():
 o = settings(max_examples=10**6.0)
 p = {(k, k**2): v**2.0 for k, v in pairs}
 q = [10.5**i for i in range(6)]
+
+
+# WE SHOULD DEFINITELY NOT EAT THESE COMMENTS (https://github.com/psf/black/issues/2873)
+if hasattr(view, "sum_of_weights"):
+    return np.divide(  # type: ignore[no-any-return]
+        view.variance,  # type: ignore[union-attr]
+        view.sum_of_weights,  # type: ignore[union-attr]
+        out=np.full(view.sum_of_weights.shape, np.nan),  # type: ignore[union-attr]
+        where=view.sum_of_weights**2 > view.sum_of_weights_squared,  # type: ignore[union-attr]
+    )
+
+return np.divide(
+    where=view.sum_of_weights_of_weight_long**2 > view.sum_of_weights_squared,  # type: ignore
+)

tests/test_format.py

@@ -28,6 +28,7 @@ def check_file(
     assert_format(source, expected, mode, fast=False)
 
 
+@pytest.mark.filterwarnings("ignore:invalid escape sequence.*:DeprecationWarning")
 @pytest.mark.parametrize("filename", all_data_cases("simple_cases"))
 def test_simple_format(filename: str) -> None:
     check_file("simple_cases", filename, DEFAULT_MODE)
@@ -132,13 +133,26 @@ def test_python_2_hint() -> None:
     exc_info.match(black.parsing.PY2_HINT)
 
 
+@pytest.mark.filterwarnings("ignore:invalid escape sequence.*:DeprecationWarning")
 def test_docstring_no_string_normalization() -> None:
     """Like test_docstring but with string normalization off."""
     source, expected = read_data("miscellaneous", "docstring_no_string_normalization")
     mode = replace(DEFAULT_MODE, string_normalization=False)
     assert_format(source, expected, mode)
 
 
+def test_preview_docstring_no_string_normalization() -> None:
+    """
+    Like test_docstring but with string normalization off *and* the preview style
+    enabled.
+    """
+    source, expected = read_data(
+        "miscellaneous", "docstring_preview_no_string_normalization"
+    )
+    mode = replace(DEFAULT_MODE, string_normalization=False, preview=True)
+    assert_format(source, expected, mode)
+
+
 def test_long_strings_flag_disabled() -> None:
     """Tests for turning off the string processing logic."""
     source, expected = read_data("miscellaneous", "long_strings_flag_disabled")