From ef1048d5f8205cb03358a6a373710c2a71d047b4 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Mon, 23 Oct 2023 23:26:40 -0700 Subject: [PATCH 01/33] Add Unreleased template to CHANGES.md (#3973) Add Unreleased template to CHANGES.md - Did this via tool working on in another branch --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 1e90c12b4fb..c4ae056b1b9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 23.10.1 ### Highlights From 1d4c31aa589dc0c8633af7531f8cc09192917b38 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Wed, 25 Oct 2023 18:35:37 +0300 Subject: [PATCH 02/33] [925] Improve multiline dictionary and list indentation for sole function parameter (#3964) --- CHANGES.md | 3 +- docs/the_black_code_style/future_style.md | 26 ++ src/black/linegen.py | 13 + src/black/mode.py | 1 + ..._parens_with_braces_and_square_brackets.py | 273 ++++++++++++++++++ .../cases/preview_long_strings__regression.py | 22 +- 6 files changed, 325 insertions(+), 13 deletions(-) create mode 100644 tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py diff --git a/CHANGES.md b/CHANGES.md index c4ae056b1b9..f7d02af187d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,7 +12,8 @@ ### Preview style - +- Multiline dictionaries and lists that are the sole argument to a function are now + indented less (#3964) ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 367ff98537c..e73c16ba26e 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -113,6 +113,32 @@ my_dict = { } ``` +### Improved multiline dictionary and list indentation for sole function parameter + +For better readability and less verticality, _Black_ now pairs parantheses ("(", ")") +with braces ("{", "}") and square brackets ("[", "]") on the same line for single +parameter function calls. For example: + +```python +foo( + [ + 1, + 2, + 3, + ] +) +``` + +will be changed to: + +```python +foo([ + 1, + 2, + 3, +]) +``` + ### Improved multiline string handling _Black_ is smarter when formatting multiline strings, especially in function arguments, diff --git a/src/black/linegen.py b/src/black/linegen.py index 2bfe587fa0e..5f5a69152d5 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -815,6 +815,19 @@ def _first_right_hand_split( tail_leaves.reverse() body_leaves.reverse() head_leaves.reverse() + + if Preview.hug_parens_with_braces_and_square_brackets in line.mode: + if ( + tail_leaves[0].type == token.RPAR + and tail_leaves[0].value + and tail_leaves[0].opening_bracket is head_leaves[-1] + and body_leaves[-1].type in [token.RBRACE, token.RSQB] + and body_leaves[-1].opening_bracket is body_leaves[0] + ): + head_leaves = head_leaves + body_leaves[:1] + tail_leaves = body_leaves[-1:] + tail_leaves + body_leaves = body_leaves[1:-1] + head = bracket_split_build_line( head_leaves, line, opening_bracket, component=_BracketSplitComponent.head ) diff --git a/src/black/mode.py b/src/black/mode.py index 4effeef3e7c..99b2a84a63d 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -190,6 +190,7 @@ class Preview(Enum): module_docstring_newlines = auto() accept_raw_docstrings = auto() fix_power_op_line_length = auto() + hug_parens_with_braces_and_square_brackets = auto() allow_empty_first_line_before_new_block_or_comment = auto() diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py new file mode 100644 index 00000000000..98ed342fcbc --- /dev/null +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -0,0 +1,273 @@ +# flags: --preview +def foo_brackets(request): + return JsonResponse( + { + "var_1": foo, + "var_2": bar, + } + ) + +def foo_square_brackets(request): + return JsonResponse( + [ + "var_1", + "var_2", + ] + ) + +func({"a": 37, "b": 42, "c": 927, "aaaaaaaaaaaaaaaaaaaaaaaaa": 11111111111111111111111111111111111111111}) + +func(["random_string_number_one","random_string_number_two","random_string_number_three","random_string_number_four"]) + +func( + { + # expand me + 'a':37, + 'b':42, + 'c':927 + } +) + +func( + [ + 'a', + 'b', + 'c', + ] +) + +func( # a + [ # b + "c", # c + "d", # d + "e", # e + ] # f +) # g + +func( # a + { # b + "c": 1, # c + "d": 2, # d + "e": 3, # e + } # f +) # g + +func( + # preserve me + [ + "c", + "d", + "e", + ] +) + +func( + [ # preserve me but hug brackets + "c", + "d", + "e", + ] +) + +func( + [ + # preserve me but hug brackets + "c", + "d", + "e", + ] +) + +func( + [ + "c", + # preserve me but hug brackets + "d", + "e", + ] +) + +func( + [ + "c", + "d", + "e", + # preserve me but hug brackets + ] +) + +func( + [ + "c", + "d", + "e", + ] # preserve me but hug brackets +) + +func( + [ + "c", + "d", + "e", + ] + # preserve me +) + +func([x for x in "short line"]) +func([x for x in "long line long line long line long line long line long line long line"]) +func([x for x in [x for x in "long line long line long line long line long line long line long line"]]) + +func({"short line"}) +func({"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}) +func({{"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}}) + +foooooooooooooooooooo( + [{c: n + 1 for c in range(256)} for n in range(100)] + [{}], {size} +) + +baaaaaaaaaaaaar( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +) + +# output +def foo_brackets(request): + return JsonResponse({ + "var_1": foo, + "var_2": bar, + }) + + +def foo_square_brackets(request): + return JsonResponse([ + "var_1", + "var_2", + ]) + + +func({ + "a": 37, + "b": 42, + "c": 927, + "aaaaaaaaaaaaaaaaaaaaaaaaa": 11111111111111111111111111111111111111111, +}) + +func([ + "random_string_number_one", + "random_string_number_two", + "random_string_number_three", + "random_string_number_four", +]) + +func({ + # expand me + "a": 37, + "b": 42, + "c": 927, +}) + +func([ + "a", + "b", + "c", +]) + +func([ # a # b + "c", # c + "d", # d + "e", # e +]) # f # g + +func({ # a # b + "c": 1, # c + "d": 2, # d + "e": 3, # e +}) # f # g + +func( + # preserve me + [ + "c", + "d", + "e", + ] +) + +func([ # preserve me but hug brackets + "c", + "d", + "e", +]) + +func([ + # preserve me but hug brackets + "c", + "d", + "e", +]) + +func([ + "c", + # preserve me but hug brackets + "d", + "e", +]) + +func([ + "c", + "d", + "e", + # preserve me but hug brackets +]) + +func([ + "c", + "d", + "e", +]) # preserve me but hug brackets + +func( + [ + "c", + "d", + "e", + ] + # preserve me +) + +func([x for x in "short line"]) +func([ + x for x in "long line long line long line long line long line long line long line" +]) +func([ + x + for x in [ + x + for x in "long line long line long line long line long line long line long line" + ] +]) + +func({"short line"}) +func({ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +}) +func({ + { + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", + } +}) + +foooooooooooooooooooo( + [{c: n + 1 for c in range(256)} for n in range(100)] + [{}], {size} +) + +baaaaaaaaaaaaar( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +) diff --git a/tests/data/cases/preview_long_strings__regression.py b/tests/data/cases/preview_long_strings__regression.py index 436157f4e05..313d898cd83 100644 --- a/tests/data/cases/preview_long_strings__regression.py +++ b/tests/data/cases/preview_long_strings__regression.py @@ -962,19 +962,17 @@ def who(self): ) -xxxxxxx_xxxxxx_xxxxxxx = xxx( - [ - xxxxxxxxxxxx( - xxxxxx_xxxxxxx=( - '((x.aaaaaaaaa = "xxxxxx.xxxxxxxxxxxxxxxxxxxxx") || (x.xxxxxxxxx =' - ' "xxxxxxxxxxxx")) && ' - # xxxxx xxxxxxxxxxxx xxxx xxx (xxxxxxxxxxxxxxxx) xx x xxxxxxxxx xx xxxxxx. - "(x.bbbbbbbbbbbb.xxx != " - '"xxx:xxx:xxx::cccccccccccc:xxxxxxx-xxxx/xxxxxxxxxxx/xxxxxxxxxxxxxxxxx") && ' - ) +xxxxxxx_xxxxxx_xxxxxxx = xxx([ + xxxxxxxxxxxx( + xxxxxx_xxxxxxx=( + '((x.aaaaaaaaa = "xxxxxx.xxxxxxxxxxxxxxxxxxxxx") || (x.xxxxxxxxx =' + ' "xxxxxxxxxxxx")) && ' + # xxxxx xxxxxxxxxxxx xxxx xxx (xxxxxxxxxxxxxxxx) xx x xxxxxxxxx xx xxxxxx. + "(x.bbbbbbbbbbbb.xxx != " + '"xxx:xxx:xxx::cccccccccccc:xxxxxxx-xxxx/xxxxxxxxxxx/xxxxxxxxxxxxxxxxx") && ' ) - ] -) + ) +]) if __name__ == "__main__": for i in range(4, 8): From 878937bcc3282319081057e2f1dbee5e24d69d68 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Wed, 25 Oct 2023 19:47:21 +0300 Subject: [PATCH 03/33] [2213] Add support for single line format skip with other comments on the same line (#3959) --- CHANGES.md | 2 +- docs/the_black_code_style/current_style.md | 12 ++-- src/black/__init__.py | 2 +- src/black/comments.py | 63 +++++++++++++++---- src/black/mode.py | 1 + ...line_format_skip_with_multiple_comments.py | 20 ++++++ 6 files changed, 81 insertions(+), 19 deletions(-) create mode 100644 tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py diff --git a/CHANGES.md b/CHANGES.md index f7d02af187d..c96186c93cc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,7 +17,7 @@ ### Configuration - +- Add support for single line format skip with other comments on the same line (#3959) ### Packaging diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index ff757a8276b..f59c1853f72 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -8,12 +8,14 @@ deliberately limited and rarely added. Previous formatting is taken into account little as possible, with rare exceptions like the magic trailing comma. The coding style used by _Black_ can be viewed as a strict subset of PEP 8. -_Black_ reformats entire files in place. It doesn't reformat lines that end with +_Black_ reformats entire files in place. It doesn't reformat lines that contain `# fmt: skip` or blocks that start with `# fmt: off` and end with `# fmt: on`. -`# fmt: on/off` must be on the same level of indentation and in the same block, meaning -no unindents beyond the initial indentation level between them. It also recognizes -[YAPF](https://github.com/google/yapf)'s block comments to the same effect, as a -courtesy for straddling code. +`# fmt: skip` can be mixed with other pragmas/comments either with multiple comments +(e.g. `# fmt: skip # pylint # noqa`) or as a semicolon separeted list (e.g. +`# fmt: skip; pylint; noqa`). `# fmt: on/off` must be on the same level of indentation +and in the same block, meaning no unindents beyond the initial indentation level between +them. It also recognizes [YAPF](https://github.com/google/yapf)'s block comments to the +same effect, as a courtesy for straddling code. The rest of this document describes the current formatting style. If you're interested in trying out where the style is heading, see [future style](./future_style.md) and try diff --git a/src/black/__init__.py b/src/black/__init__.py index 188a4f79f0e..7cf93b89e42 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1099,7 +1099,7 @@ def _format_str_once(src_contents: str, *, mode: Mode) -> str: for feature in {Feature.PARENTHESIZED_CONTEXT_MANAGERS} if supports_feature(versions, feature) } - normalize_fmt_off(src_node) + normalize_fmt_off(src_node, mode) lines = LineGenerator(mode=mode, features=context_manager_features) elt = EmptyLineTracker(mode=mode) split_line_features = { diff --git a/src/black/comments.py b/src/black/comments.py index 226968bff98..862fc7607cc 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -3,6 +3,7 @@ from functools import lru_cache from typing import Final, Iterator, List, Optional, Union +from black.mode import Mode, Preview from black.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, @@ -20,10 +21,11 @@ FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"} FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"} -FMT_PASS: Final = {*FMT_OFF, *FMT_SKIP} FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"} COMMENT_EXCEPTIONS = " !:#'" +_COMMENT_PREFIX = "# " +_COMMENT_LIST_SEPARATOR = ";" @dataclass @@ -130,14 +132,14 @@ def make_comment(content: str) -> str: return "#" + content -def normalize_fmt_off(node: Node) -> None: +def normalize_fmt_off(node: Node, mode: Mode) -> None: """Convert content between `# fmt: off`/`# fmt: on` into standalone comments.""" try_again = True while try_again: - try_again = convert_one_fmt_off_pair(node) + try_again = convert_one_fmt_off_pair(node, mode) -def convert_one_fmt_off_pair(node: Node) -> bool: +def convert_one_fmt_off_pair(node: Node, mode: Mode) -> bool: """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment. Returns True if a pair was converted. @@ -145,21 +147,27 @@ def convert_one_fmt_off_pair(node: Node) -> bool: for leaf in node.leaves(): previous_consumed = 0 for comment in list_comments(leaf.prefix, is_endmarker=False): - if comment.value not in FMT_PASS: + should_pass_fmt = comment.value in FMT_OFF or _contains_fmt_skip_comment( + comment.value, mode + ) + if not should_pass_fmt: previous_consumed = comment.consumed continue # We only want standalone comments. If there's no previous leaf or # the previous leaf is indentation, it's a standalone comment in # disguise. - if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT: + if should_pass_fmt and comment.type != STANDALONE_COMMENT: prev = preceding_leaf(leaf) if prev: if comment.value in FMT_OFF and prev.type not in WHITESPACE: continue - if comment.value in FMT_SKIP and prev.type in WHITESPACE: + if ( + _contains_fmt_skip_comment(comment.value, mode) + and prev.type in WHITESPACE + ): continue - ignored_nodes = list(generate_ignored_nodes(leaf, comment)) + ignored_nodes = list(generate_ignored_nodes(leaf, comment, mode)) if not ignored_nodes: continue @@ -168,7 +176,7 @@ def convert_one_fmt_off_pair(node: Node) -> bool: prefix = first.prefix if comment.value in FMT_OFF: first.prefix = prefix[comment.consumed :] - if comment.value in FMT_SKIP: + if _contains_fmt_skip_comment(comment.value, mode): first.prefix = "" standalone_comment_prefix = prefix else: @@ -178,7 +186,7 @@ def convert_one_fmt_off_pair(node: Node) -> bool: hidden_value = "".join(str(n) for n in ignored_nodes) if comment.value in FMT_OFF: hidden_value = comment.value + "\n" + hidden_value - if comment.value in FMT_SKIP: + if _contains_fmt_skip_comment(comment.value, mode): hidden_value += " " + comment.value if hidden_value.endswith("\n"): # That happens when one of the `ignored_nodes` ended with a NEWLINE @@ -205,13 +213,15 @@ def convert_one_fmt_off_pair(node: Node) -> bool: return False -def generate_ignored_nodes(leaf: Leaf, comment: ProtoComment) -> Iterator[LN]: +def generate_ignored_nodes( + leaf: Leaf, comment: ProtoComment, mode: Mode +) -> Iterator[LN]: """Starting from the container of `leaf`, generate all leaves until `# fmt: on`. If comment is skip, returns leaf only. Stops at the end of the block. """ - if comment.value in FMT_SKIP: + if _contains_fmt_skip_comment(comment.value, mode): yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment) return container: Optional[LN] = container_of(leaf) @@ -327,3 +337,32 @@ def contains_pragma_comment(comment_list: List[Leaf]) -> bool: return True return False + + +def _contains_fmt_skip_comment(comment_line: str, mode: Mode) -> bool: + """ + Checks if the given comment contains FMT_SKIP alone or paired with other comments. + Matching styles: + # fmt:skip <-- single comment + # noqa:XXX # fmt:skip # a nice line <-- multiple comments (Preview) + # pylint:XXX; fmt:skip <-- list of comments (; separated, Preview) + """ + semantic_comment_blocks = ( + [ + comment_line, + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.split(_COMMENT_PREFIX)[1:] + ], + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.strip(_COMMENT_PREFIX).split( + _COMMENT_LIST_SEPARATOR + ) + ], + ] + if Preview.single_line_format_skip_with_multiple_comments in mode + else [comment_line] + ) + + return any(comment in FMT_SKIP for comment in semantic_comment_blocks) diff --git a/src/black/mode.py b/src/black/mode.py index 99b2a84a63d..4e4effffb86 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -192,6 +192,7 @@ class Preview(Enum): fix_power_op_line_length = auto() hug_parens_with_braces_and_square_brackets = auto() allow_empty_first_line_before_new_block_or_comment = auto() + single_line_format_skip_with_multiple_comments = auto() class Deprecated(UserWarning): diff --git a/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py b/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py new file mode 100644 index 00000000000..efde662baa8 --- /dev/null +++ b/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py @@ -0,0 +1,20 @@ +# flags: --preview +foo = 123 # fmt: skip # noqa: E501 # pylint +bar = ( + 123 , + ( 1 + 5 ) # pylint # fmt:skip +) +baz = "a" + "b" # pylint; fmt: skip; noqa: E501 +skip_will_not_work = "a" + "b" # pylint fmt:skip +skip_will_not_work2 = "a" + "b" # some text; fmt:skip happens to be part of it + +# output + +foo = 123 # fmt: skip # noqa: E501 # pylint +bar = ( + 123 , + ( 1 + 5 ) # pylint # fmt:skip +) +baz = "a" + "b" # pylint; fmt: skip; noqa: E501 +skip_will_not_work = "a" + "b" # pylint fmt:skip +skip_will_not_work2 = "a" + "b" # some text; fmt:skip happens to be part of it From f7174bfc431e22f38b502579d1234989c3c5ce15 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Fri, 27 Oct 2023 01:43:42 +0900 Subject: [PATCH 04/33] Fix typo in future_style.md (#3979) parantheses -> parentheses --- docs/the_black_code_style/future_style.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index e73c16ba26e..f2534b0f0d0 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -115,7 +115,7 @@ my_dict = { ### Improved multiline dictionary and list indentation for sole function parameter -For better readability and less verticality, _Black_ now pairs parantheses ("(", ")") +For better readability and less verticality, _Black_ now pairs parentheses ("(", ")") with braces ("{", "}") and square brackets ("[", "]") on the same line for single parameter function calls. For example: From de701fe6aa0d61526b806dd31610da5cf8b67ab9 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 26 Oct 2023 21:13:25 -0700 Subject: [PATCH 05/33] Fix CI by running on Python 3.11 (#3984) aiohttp doesn't yet support 3.12 --- .github/workflows/diff_shades.yml | 4 ++-- .github/workflows/doc.yml | 2 +- .github/workflows/lint.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index 97db907abc8..6bfc6ca9ed8 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: "*" + python-version: "3.11" - name: Install diff-shades and support dependencies run: | @@ -59,7 +59,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "*" + python-version: "3.11" - name: Install diff-shades and support dependencies run: | diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index fa3d87c70f5..9a23e19cadd 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -26,7 +26,7 @@ jobs: - name: Set up latest Python uses: actions/setup-python@v4 with: - python-version: "*" + python-version: "3.11" - name: Install dependencies run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3eaf5785f5a..7fe1b04eb02 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,7 +26,7 @@ jobs: - name: Set up latest Python uses: actions/setup-python@v4 with: - python-version: "*" + python-version: "3.11" - name: Install dependencies run: | From 7bfa35cca88a2a6b875fb8564c19164143a46f1d Mon Sep 17 00:00:00 2001 From: Surav Shrestha <148626286+shresthasurav@users.noreply.github.com> Date: Fri, 27 Oct 2023 10:11:47 +0545 Subject: [PATCH 06/33] docs: fix typos in change log and documentations (#3985) --- CHANGES.md | 2 +- docs/the_black_code_style/current_style.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c96186c93cc..7703223a119 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -52,7 +52,7 @@ ### Highlights -- Maintanence release to get a fix out for GitHub Action edge case (#3957) +- Maintenance release to get a fix out for GitHub Action edge case (#3957) ### Preview style diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index f59c1853f72..431bae525f6 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -11,7 +11,7 @@ used by _Black_ can be viewed as a strict subset of PEP 8. _Black_ reformats entire files in place. It doesn't reformat lines that contain `# fmt: skip` or blocks that start with `# fmt: off` and end with `# fmt: on`. `# fmt: skip` can be mixed with other pragmas/comments either with multiple comments -(e.g. `# fmt: skip # pylint # noqa`) or as a semicolon separeted list (e.g. +(e.g. `# fmt: skip # pylint # noqa`) or as a semicolon separated list (e.g. `# fmt: skip; pylint; noqa`). `# fmt: on/off` must be on the same level of indentation and in the same block, meaning no unindents beyond the initial indentation level between them. It also recognizes [YAPF](https://github.com/google/yapf)'s block comments to the From c369e446f9dbff313ebb555bf461b4e7778ca78d Mon Sep 17 00:00:00 2001 From: sth Date: Fri, 27 Oct 2023 09:43:51 +0200 Subject: [PATCH 07/33] Fix matching of absolute paths in `--include` (#3976) --- CHANGES.md | 2 ++ src/black/files.py | 2 +- tests/test_black.py | 59 +++++++++++++++++++++++++++++++++++++++------ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7703223a119..71f62d0e11f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,8 @@ - Add support for single line format skip with other comments on the same line (#3959) +- Fix a bug in the matching of absolute path names in `--include` (#3976) + ### Packaging diff --git a/src/black/files.py b/src/black/files.py index 362898dc0fd..1eed7eda828 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -389,7 +389,7 @@ def gen_python_files( warn=verbose or not quiet ): continue - include_match = include.search(normalized_path) if include else True + include_match = include.search(root_relative_path) if include else True if include_match: yield child diff --git a/tests/test_black.py b/tests/test_black.py index 537ca80d432..56c20243020 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2388,6 +2388,27 @@ def test_empty_include(self) -> None: # Setting exclude explicitly to an empty string to block .gitignore usage. assert_collected_sources(src, expected, include="", exclude="") + def test_include_absolute_path(self) -> None: + path = DATA_DIR / "include_exclude_tests" + src = [path] + expected = [ + Path(path / "b/dont_exclude/a.pie"), + ] + assert_collected_sources( + src, expected, root=path, include=r"^/b/dont_exclude/a\.pie$", exclude="" + ) + + def test_exclude_absolute_path(self) -> None: + path = DATA_DIR / "include_exclude_tests" + src = [path] + expected = [ + Path(path / "b/dont_exclude/a.py"), + Path(path / "b/.definitely_exclude/a.py"), + ] + assert_collected_sources( + src, expected, root=path, include=r"\.py$", exclude=r"^/b/exclude/a\.py$" + ) + def test_extend_exclude(self) -> None: path = DATA_DIR / "include_exclude_tests" src = [path] @@ -2401,7 +2422,6 @@ def test_extend_exclude(self) -> None: @pytest.mark.incompatible_with_mypyc def test_symlinks(self) -> None: - path = MagicMock() root = THIS_DIR.resolve() include = re.compile(black.DEFAULT_INCLUDES) exclude = re.compile(black.DEFAULT_EXCLUDES) @@ -2409,19 +2429,44 @@ def test_symlinks(self) -> None: gitignore = PathSpec.from_lines("gitwildmatch", []) regular = MagicMock() - outside_root_symlink = MagicMock() - ignored_symlink = MagicMock() - - path.iterdir.return_value = [regular, outside_root_symlink, ignored_symlink] - regular.absolute.return_value = root / "regular.py" regular.resolve.return_value = root / "regular.py" regular.is_dir.return_value = False + regular.is_file.return_value = True + outside_root_symlink = MagicMock() outside_root_symlink.absolute.return_value = root / "symlink.py" outside_root_symlink.resolve.return_value = Path("/nowhere") + outside_root_symlink.is_dir.return_value = False + outside_root_symlink.is_file.return_value = True + ignored_symlink = MagicMock() ignored_symlink.absolute.return_value = root / ".mypy_cache" / "symlink.py" + ignored_symlink.is_dir.return_value = False + ignored_symlink.is_file.return_value = True + + # A symlink that has an excluded name, but points to an included name + symlink_excluded_name = MagicMock() + symlink_excluded_name.absolute.return_value = root / "excluded_name" + symlink_excluded_name.resolve.return_value = root / "included_name.py" + symlink_excluded_name.is_dir.return_value = False + symlink_excluded_name.is_file.return_value = True + + # A symlink that has an included name, but points to an excluded name + symlink_included_name = MagicMock() + symlink_included_name.absolute.return_value = root / "included_name.py" + symlink_included_name.resolve.return_value = root / "excluded_name" + symlink_included_name.is_dir.return_value = False + symlink_included_name.is_file.return_value = True + + path = MagicMock() + path.iterdir.return_value = [ + regular, + outside_root_symlink, + ignored_symlink, + symlink_excluded_name, + symlink_included_name, + ] files = list( black.gen_python_files( @@ -2437,7 +2482,7 @@ def test_symlinks(self) -> None: quiet=False, ) ) - assert files == [regular] + assert files == [regular, symlink_included_name] path.iterdir.assert_called_once() outside_root_symlink.resolve.assert_called_once() From caef19689b153f3a7baea1764a5adccae8bf1f1e Mon Sep 17 00:00:00 2001 From: Gabriel Perren Date: Fri, 27 Oct 2023 15:54:31 -0300 Subject: [PATCH 08/33] Update current_style.md (#3990) Fix small typo --- docs/the_black_code_style/current_style.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index 431bae525f6..2a5e10162f2 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -178,7 +178,7 @@ If you use Flake8, you have a few options: extend-ignore = E203, E501, E704 ``` - The rationale for E950 is explained in + The rationale for B950 is explained in [Bugbear's documentation](https://github.com/PyCQA/flake8-bugbear#opinionated-warnings). 2. For a minimally compatible config: From c712d57ca9e30ba0db61c2fd7e4a2bf67f58bcc2 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 27 Oct 2023 12:17:54 -0700 Subject: [PATCH 09/33] Add trailing comma test case for hugging parens (#3991) --- docs/the_black_code_style/future_style.md | 13 +++++++++++++ ...hug_parens_with_braces_and_square_brackets.py | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index f2534b0f0d0..c744902577d 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -139,6 +139,19 @@ foo([ ]) ``` +You can use a magic trailing comma to avoid this compacting behavior; by default, +_Black_ will not reformat the following code: + +```python +foo( + [ + 1, + 2, + 3, + ], +) +``` + ### Improved multiline string handling _Black_ is smarter when formatting multiline strings, especially in function arguments, diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py index 98ed342fcbc..6d10518133c 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -36,6 +36,14 @@ def foo_square_brackets(request): ] ) +func( + [ + 'a', + 'b', + 'c', + ], +) + func( # a [ # b "c", # c @@ -171,6 +179,14 @@ def foo_square_brackets(request): "c", ]) +func( + [ + "a", + "b", + "c", + ], +) + func([ # a # b "c", # c "d", # d From 53c4278a4c9b81baa86630ffda5f680f33968d1e Mon Sep 17 00:00:00 2001 From: Satyam Namdev <111422209+Spyrosigma@users.noreply.github.com> Date: Sat, 28 Oct 2023 01:57:19 +0530 Subject: [PATCH 10/33] Update CHANGES.md (#3988) Fixed a grammatical mistake --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 71f62d0e11f..84d9061135a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,7 +17,7 @@ ### Configuration -- Add support for single line format skip with other comments on the same line (#3959) +- Add support for single-line format skip with other comments on the same line (#3959) - Fix a bug in the matching of absolute path names in `--include` (#3976) From 7686989fc89aad5ea235a34977ebf8c81c26c4eb Mon Sep 17 00:00:00 2001 From: David Culley <6276049+davidculley@users.noreply.github.com> Date: Sat, 28 Oct 2023 00:43:34 +0200 Subject: [PATCH 11/33] confine pre-commit to stages (#3940) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://pre-commit.com/#confining-hooks-to-run-at-certain-stages > If you are authoring a tool, it is usually a good idea to provide an appropriate `stages` property. For example a reasonable setting for a linter or code formatter would be `stages: [pre-commit, pre-merge-commit, pre-push, manual]`. Co-authored-by: Jelle Zijlstra --- .pre-commit-hooks.yaml | 2 ++ CHANGES.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index a1ff41fded8..54a03efe7a1 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -4,6 +4,7 @@ name: black description: "Black: The uncompromising Python code formatter" entry: black + stages: [pre-commit, pre-merge-commit, pre-push, manual] language: python minimum_pre_commit_version: 2.9.2 require_serial: true @@ -13,6 +14,7 @@ description: "Black: The uncompromising Python code formatter (with Jupyter Notebook support)" entry: black + stages: [pre-commit, pre-merge-commit, pre-push, manual] language: python minimum_pre_commit_version: 2.9.2 require_serial: true diff --git a/CHANGES.md b/CHANGES.md index 84d9061135a..60231468bdf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,6 +45,9 @@ +- Black's pre-commit integration will now run only on git hooks appropriate for a code + formatter (#3940) + ### Documentation in CHANGES.md to delete ... - Update ci to run out of scripts dir too - Update test_tuple_calver --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- .github/workflows/release_tests.yml | 56 ++++++ docs/contributing/release_process.md | 70 ++------ scripts/release.py | 243 +++++++++++++++++++++++++++ scripts/release_tests.py | 69 ++++++++ 4 files changed, 383 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/release_tests.yml create mode 100755 scripts/release.py create mode 100644 scripts/release_tests.py diff --git a/.github/workflows/release_tests.yml b/.github/workflows/release_tests.yml new file mode 100644 index 00000000000..74729445052 --- /dev/null +++ b/.github/workflows/release_tests.yml @@ -0,0 +1,56 @@ +name: Release tool CI + +on: + push: + paths: + - .github/workflows/release_tests.yml + - release.py + - release_tests.py + pull_request: + paths: + - .github/workflows/release_tests.yml + - release.py + - release_tests.py + +jobs: + build: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. Without this if check, checks are duplicated since + # internal PRs match both the push and pull_request events. + if: + github.event_name == 'push' || github.event.pull_request.head.repo.full_name != + github.repository + + name: Running python ${{ matrix.python-version }} on ${{matrix.os}} + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: ["3.12"] + os: [macOS-latest, ubuntu-latest, windows-latest] + + steps: + - uses: actions/checkout@v4 + with: + # Give us all history, branches and tags + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Print Python Version + run: python --version --version && which python + + - name: Print Git Version + run: git --version && which git + + - name: Update pip, setuptools + wheels + run: | + python -m pip install --upgrade pip setuptools wheel + + - name: Run unit tests via coverage + print report + run: | + python -m pip install coverage + coverage run scripts/release_tests.py + coverage report --show-missing diff --git a/docs/contributing/release_process.md b/docs/contributing/release_process.md index 02865d6f4bd..c66ffae8ace 100644 --- a/docs/contributing/release_process.md +++ b/docs/contributing/release_process.md @@ -32,21 +32,29 @@ The 10,000 foot view of the release process is that you prepare a release PR and publish a [GitHub Release]. This triggers [release automation](#release-workflows) that builds all release artifacts and publishes them to the various platforms we publish to. +We now have a `scripts/release.py` script to help with cutting the release PRs. + +- `python3 scripts/release.py --help` is your friend. + - `release.py` has only been tested in Python 3.12 (so get with the times :D) + To cut a release: 1. Determine the release's version number - **_Black_ follows the [CalVer] versioning standard using the `YY.M.N` format** - So unless there already has been a release during this month, `N` should be `0` - Example: the first release in January, 2022 → `22.1.0` + - `release.py` will calculate this and log to stderr for you copy paste pleasure 1. File a PR editing `CHANGES.md` and the docs to version the latest changes + - Run `python3 scripts/release.py [--debug]` to generate most changes + - Sub headings in the template, if they have no bullet points need manual removal + _PR welcome to improve :D_ +1. If `release.py` fail manually edit; otherwise, yay, skip this step! 1. Replace the `## Unreleased` header with the version number 1. Remove any empty sections for the current release 1. (_optional_) Read through and copy-edit the changelog (eg. by moving entries, fixing typos, or rephrasing entries) 1. Double-check that no changelog entries since the last release were put in the wrong section (e.g., run `git diff CHANGES.md`) - 1. Add a new empty template for the next release above - ([template below](#changelog-template)) 1. Update references to the latest version in {doc}`/integrations/source_version_control` and {doc}`/usage_and_configuration/the_basics` @@ -63,6 +71,11 @@ To cut a release: description box 1. Publish the GitHub Release, triggering [release automation](#release-workflows) that will handle the rest +1. Once CI is done add + commit (git push - No review) a new empty template for the next + release to CHANGES.md _(Template is able to be copy pasted from release.py should we + fail)_ + 1. `python3 scripts/release.py --add-changes-template|-a [--debug]` + 1. Should that fail, please return to copy + paste 1. At this point, you're basically done. It's good practice to go and [watch and verify that all the release workflows pass][black-actions], although you will receive a GitHub notification should something fail. @@ -81,59 +94,6 @@ release is probably unnecessary. In the end, use your best judgement and ask other maintainers for their thoughts. ``` -### Changelog template - -Use the following template for a clean changelog after the release: - -``` -## Unreleased - -### Highlights - - - -### Stable style - - - -### Preview style - - - -### Configuration - - - -### Packaging - - - -### Parser - - - -### Performance - - - -### Output - - - -### _Blackd_ - - - -### Integrations - - - -### Documentation - - -``` - ## Release workflows All of _Black_'s release automation uses [GitHub Actions]. All workflows are therefore diff --git a/scripts/release.py b/scripts/release.py new file mode 100755 index 00000000000..d588429c2d3 --- /dev/null +++ b/scripts/release.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +""" +Tool to help automate changes needed in commits during and after releases +""" + +import argparse +import logging +import sys +from datetime import datetime +from pathlib import Path +from subprocess import PIPE, run +from typing import List + +LOG = logging.getLogger(__name__) +NEW_VERSION_CHANGELOG_TEMPLATE = """\ +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + +""" + + +class NoGitTagsError(Exception): ... # noqa: E701,E761 + + +# TODO: Do better with alpha + beta releases +# Maybe we vendor packaging library +def get_git_tags(versions_only: bool = True) -> List[str]: + """Pull out all tags or calvers only""" + cp = run(["git", "tag"], stdout=PIPE, stderr=PIPE, check=True, encoding="utf8") + if not cp.stdout: + LOG.error(f"Returned no git tags stdout: {cp.stderr}") + raise NoGitTagsError + git_tags = cp.stdout.splitlines() + if versions_only: + return [t for t in git_tags if t[0].isdigit()] + return git_tags + + +# TODO: Support sorting alhpa/beta releases correctly +def tuple_calver(calver: str) -> tuple[int, ...]: # mypy can't notice maxsplit below + """Convert a calver string into a tuple of ints for sorting""" + try: + return tuple(map(int, calver.split(".", maxsplit=2))) + except ValueError: + return (0, 0, 0) + + +class SourceFiles: + def __init__(self, black_repo_dir: Path): + # File path fun all pathlib to be platform agnostic + self.black_repo_path = black_repo_dir + self.changes_path = self.black_repo_path / "CHANGES.md" + self.docs_path = self.black_repo_path / "docs" + self.version_doc_paths = ( + self.docs_path / "integrations" / "source_version_control.md", + self.docs_path / "usage_and_configuration" / "the_basics.md", + ) + self.current_version = self.get_current_version() + self.next_version = self.get_next_version() + + def __str__(self) -> str: + return f"""\ +> SourceFiles ENV: + Repo path: {self.black_repo_path} + CHANGES.md path: {self.changes_path} + docs path: {self.docs_path} + Current version: {self.current_version} + Next version: {self.next_version} +""" + + def add_template_to_changes(self) -> int: + """Add the template to CHANGES.md if it does not exist""" + LOG.info(f"Adding template to {self.changes_path}") + + with self.changes_path.open("r") as cfp: + changes_string = cfp.read() + + if "## Unreleased" in changes_string: + LOG.error(f"{self.changes_path} already has unreleased template") + return 1 + + templated_changes_string = changes_string.replace( + "# Change Log\n", + f"# Change Log\n\n{NEW_VERSION_CHANGELOG_TEMPLATE}", + ) + + with self.changes_path.open("w") as cfp: + cfp.write(templated_changes_string) + + LOG.info(f"Added template to {self.changes_path}") + return 0 + + def cleanup_changes_template_for_release(self) -> None: + LOG.info(f"Cleaning up {self.changes_path}") + + with self.changes_path.open("r") as cfp: + changes_string = cfp.read() + + # Change Unreleased to next version + versioned_changes = changes_string.replace( + "## Unreleased", f"## {self.next_version}" + ) + + # Remove all comments (subheadings are harder - Human required still) + no_comments_changes = [] + for line in versioned_changes.splitlines(): + if line.startswith(""): + continue + no_comments_changes.append(line) + + with self.changes_path.open("w") as cfp: + cfp.write("\n".join(no_comments_changes) + "\n") + + LOG.debug(f"Finished Cleaning up {self.changes_path}") + + def get_current_version(self) -> str: + """Get the latest git (version) tag as latest version""" + return sorted(get_git_tags(), key=lambda k: tuple_calver(k))[-1] + + def get_next_version(self) -> str: + """Workout the year and month + version number we need to move to""" + base_calver = datetime.today().strftime("%y.%m") + calver_parts = base_calver.split(".") + base_calver = f"{calver_parts[0]}.{int(calver_parts[1])}" # Remove leading 0 + git_tags = get_git_tags() + same_month_releases = [t for t in git_tags if t.startswith(base_calver)] + if len(same_month_releases) < 1: + return f"{base_calver}.0" + same_month_version = same_month_releases[-1].split(".", 2)[-1] + return f"{base_calver}.{int(same_month_version) + 1}" + + def update_repo_for_release(self) -> int: + """Update CHANGES.md + doc files ready for release""" + self.cleanup_changes_template_for_release() + self.update_version_in_docs() + return 0 # return 0 if no exceptions hit + + def update_version_in_docs(self) -> None: + for doc_path in self.version_doc_paths: + LOG.info(f"Updating black version to {self.next_version} in {doc_path}") + + with doc_path.open("r") as dfp: + doc_string = dfp.read() + + next_version_doc = doc_string.replace( + self.current_version, self.next_version + ) + + with doc_path.open("w") as dfp: + dfp.write(next_version_doc) + + LOG.debug( + f"Finished updating black version to {self.next_version} in {doc_path}" + ) + + +def _handle_debug(debug: bool) -> None: + """Turn on debugging if asked otherwise INFO default""" + log_level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + format="[%(asctime)s] %(levelname)s: %(message)s (%(filename)s:%(lineno)d)", + level=log_level, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "-a", + "--add-changes-template", + action="store_true", + help="Add the Unreleased template to CHANGES.md", + ) + parser.add_argument( + "-d", "--debug", action="store_true", help="Verbose debug output" + ) + args = parser.parse_args() + _handle_debug(args.debug) + return args + + +def main() -> int: + args = parse_args() + + # Need parent.parent cause script is in scripts/ directory + sf = SourceFiles(Path(__file__).parent.parent) + + if args.add_changes_template: + return sf.add_template_to_changes() + + LOG.info(f"Current version detected to be {sf.current_version}") + LOG.info(f"Next version will be {sf.next_version}") + return sf.update_repo_for_release() + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/scripts/release_tests.py b/scripts/release_tests.py new file mode 100644 index 00000000000..bd72cb4b48a --- /dev/null +++ b/scripts/release_tests.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import unittest +from pathlib import Path +from shutil import rmtree +from tempfile import TemporaryDirectory +from typing import Any +from unittest.mock import Mock, patch + +from release import SourceFiles, tuple_calver # type: ignore + + +class FakeDateTime: + """Used to mock the date to test generating next calver function""" + + def today(*args: Any, **kwargs: Any) -> "FakeDateTime": # noqa + return FakeDateTime() + + # Add leading 0 on purpose to ensure we remove it + def strftime(*args: Any, **kwargs: Any) -> str: # noqa + return "69.01" + + +class TestRelease(unittest.TestCase): + def setUp(self) -> None: + # We only test on >= 3.12 + self.tempdir = TemporaryDirectory(delete=False) # type: ignore + self.tempdir_path = Path(self.tempdir.name) + self.sf = SourceFiles(self.tempdir_path) + + def tearDown(self) -> None: + rmtree(self.tempdir.name) + return super().tearDown() + + @patch("release.get_git_tags") + def test_get_current_version(self, mocked_git_tags: Mock) -> None: + mocked_git_tags.return_value = ["1.1.0", "69.1.0", "69.1.1", "2.2.0"] + self.assertEqual("69.1.1", self.sf.get_current_version()) + + @patch("release.get_git_tags") + @patch("release.datetime", FakeDateTime) + def test_get_next_version(self, mocked_git_tags: Mock) -> None: + # test we handle no args + mocked_git_tags.return_value = [] + self.assertEqual( + "69.1.0", + self.sf.get_next_version(), + "Unable to get correct next version with no git tags", + ) + + # test we handle + mocked_git_tags.return_value = ["1.1.0", "69.1.0", "69.1.1", "2.2.0"] + self.assertEqual( + "69.1.2", + self.sf.get_next_version(), + "Unable to get correct version with 2 previous versions released this" + " month", + ) + + def test_tuple_calver(self) -> None: + first_month_release = tuple_calver("69.1.0") + second_month_release = tuple_calver("69.1.1") + self.assertEqual((69, 1, 0), first_month_release) + self.assertEqual((0, 0, 0), tuple_calver("69.1.1a0")) # Hack for alphas/betas + self.assertTrue(first_month_release < second_month_release) + + +if __name__ == "__main__": + unittest.main() From ddfecf06c13dd86205c851e340124e325ed82c5c Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Mon, 30 Oct 2023 17:35:26 +0200 Subject: [PATCH 16/33] Hug parens also with multiline unpacking (#3992) --- CHANGES.md | 2 ++ docs/the_black_code_style/future_style.md | 20 +++++++++++ src/black/cache.py | 6 ++-- src/black/linegen.py | 7 ++-- ..._parens_with_braces_and_square_brackets.py | 36 +++++++++++++++++++ 5 files changed, 65 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 60231468bdf..dd5f52cf706 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,8 @@ - Multiline dictionaries and lists that are the sole argument to a function are now indented less (#3964) +- Multiline list and dict unpacking as the sole argument to a function is now also + indented less (#3992) ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index c744902577d..944ffad033e 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -139,6 +139,26 @@ foo([ ]) ``` +This also applies to list and dictionary unpacking: + +```python +foo( + *[ + a_long_function_name(a_long_variable_name) + for a_long_variable_name in some_generator + ] +) +``` + +will become: + +```python +foo(*[ + a_long_function_name(a_long_variable_name) + for a_long_variable_name in some_generator +]) +``` + You can use a magic trailing comma to avoid this compacting behavior; by default, _Black_ will not reformat the following code: diff --git a/src/black/cache.py b/src/black/cache.py index 6baa096baca..6a332304981 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -124,9 +124,9 @@ def filtered_cached(self, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path] def write(self, sources: Iterable[Path]) -> None: """Update the cache file data and write a new cache file.""" - self.file_data.update( - **{str(src.resolve()): Cache.get_file_data(src) for src in sources} - ) + self.file_data.update(**{ + str(src.resolve()): Cache.get_file_data(src) for src in sources + }) try: CACHE_DIR.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile( diff --git a/src/black/linegen.py b/src/black/linegen.py index 5f5a69152d5..43bc08efbbd 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -817,16 +817,17 @@ def _first_right_hand_split( head_leaves.reverse() if Preview.hug_parens_with_braces_and_square_brackets in line.mode: + is_unpacking = 1 if body_leaves[0].type in [token.STAR, token.DOUBLESTAR] else 0 if ( tail_leaves[0].type == token.RPAR and tail_leaves[0].value and tail_leaves[0].opening_bracket is head_leaves[-1] and body_leaves[-1].type in [token.RBRACE, token.RSQB] - and body_leaves[-1].opening_bracket is body_leaves[0] + and body_leaves[-1].opening_bracket is body_leaves[is_unpacking] ): - head_leaves = head_leaves + body_leaves[:1] + head_leaves = head_leaves + body_leaves[: 1 + is_unpacking] tail_leaves = body_leaves[-1:] + tail_leaves - body_leaves = body_leaves[1:-1] + body_leaves = body_leaves[1 + is_unpacking : -1] head = bracket_split_build_line( head_leaves, line, opening_bracket, component=_BracketSplitComponent.head diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py index 6d10518133c..51fe516add5 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -137,6 +137,21 @@ def foo_square_brackets(request): [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] ) +foo(*["long long long long long line", "long long long long long line", "long long long long long line"]) + +foo(*[str(i) for i in range(100000000000000000000000000000000000000000000000000000000000)]) + +foo( + **{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": 1, + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": 2, + "ccccccccccccccccccccccccccccccccc": 3, + **other, + } +) + +foo(**{x: y for x, y in enumerate(["long long long long line","long long long long line"])}) + # output def foo_brackets(request): return JsonResponse({ @@ -287,3 +302,24 @@ def foo_square_brackets(request): baaaaaaaaaaaaar( [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] ) + +foo(*[ + "long long long long long line", + "long long long long long line", + "long long long long long line", +]) + +foo(*[ + str(i) for i in range(100000000000000000000000000000000000000000000000000000000000) +]) + +foo(**{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": 1, + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": 2, + "ccccccccccccccccccccccccccccccccc": 3, + **other, +}) + +foo(**{ + x: y for x, y in enumerate(["long long long long line", "long long long long line"]) +}) From e50110353ab81b539aaee686453c18c707b5f045 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Tue, 31 Oct 2023 17:27:11 +0200 Subject: [PATCH 17/33] Produce equivalent code for docstrings containing backslash followed by whitespace(s) before newline (#4008) Fixes #3727 --- CHANGES.md | 3 ++- src/black/linegen.py | 3 ++- tests/data/cases/docstring.py | 13 +++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dd5f52cf706..e910fbed162 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,8 @@ ### Stable style - +- Fix a crash when whitespace(s) followed a backslash before newline in a docstring + (#4008) ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index 43bc08efbbd..121c6e314fe 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -2,6 +2,7 @@ Generating lines of code. """ +import re import sys from dataclasses import replace from enum import Enum, auto @@ -420,7 +421,7 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: if Preview.hex_codes_in_unicode_sequences in self.mode: normalize_unicode_escape_sequences(leaf) - if is_docstring(leaf) and "\\\n" not in leaf.value: + if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value): # We're ignoring docstrings with backslash newline escapes because changing # indentation of those changes the AST representation of the code. if self.mode.string_normalization: diff --git a/tests/data/cases/docstring.py b/tests/data/cases/docstring.py index c31d6a68783..e983c5bd438 100644 --- a/tests/data/cases/docstring.py +++ b/tests/data/cases/docstring.py @@ -221,6 +221,12 @@ def stable_quote_normalization_with_immediate_inner_single_quote(self): ''' +def foo(): + """ + Docstring with a backslash followed by a space\ + and then another line + """ + # output class MyClass: @@ -442,3 +448,10 @@ def stable_quote_normalization_with_immediate_inner_single_quote(self): """ + + +def foo(): + """ + Docstring with a backslash followed by a space\ + and then another line + """ From 5758da6e3cda4ec037c5dbb7867373cf694edd03 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 31 Oct 2023 17:11:28 -0700 Subject: [PATCH 18/33] Fix bytes strings being treated as docstrings (#4003) Fixes #4002 --- CHANGES.md | 4 +-- src/black/nodes.py | 9 ++++++- tests/data/cases/bytes_docstring.py | 34 +++++++++++++++++++++++++ tests/data/{ => cases}/raw_docstring.py | 0 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/data/cases/bytes_docstring.py rename tests/data/{ => cases}/raw_docstring.py (100%) diff --git a/CHANGES.md b/CHANGES.md index e910fbed162..f365f1c239b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,8 +8,8 @@ ### Stable style -- Fix a crash when whitespace(s) followed a backslash before newline in a docstring - (#4008) +- Fix crash on formatting bytes strings that look like docstrings (#4003) +- Fix crash when whitespace followed a backslash before newline in a docstring (#4008) ### Preview style diff --git a/src/black/nodes.py b/src/black/nodes.py index b2e96cb9edf..5f6b280c035 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -14,7 +14,7 @@ from black.cache import CACHE_DIR from black.mode import Mode, Preview -from black.strings import has_triple_quotes +from black.strings import get_string_prefix, has_triple_quotes from blib2to3 import pygram from blib2to3.pgen2 import token from blib2to3.pytree import NL, Leaf, Node, type_repr @@ -525,6 +525,13 @@ def is_arith_like(node: LN) -> bool: def is_docstring(leaf: Leaf) -> bool: + if leaf.type != token.STRING: + return False + + prefix = get_string_prefix(leaf.value) + if "b" in prefix or "B" in prefix: + return False + if prev_siblings_are( leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] ): diff --git a/tests/data/cases/bytes_docstring.py b/tests/data/cases/bytes_docstring.py new file mode 100644 index 00000000000..2326e95293a --- /dev/null +++ b/tests/data/cases/bytes_docstring.py @@ -0,0 +1,34 @@ +def bitey(): + b" not a docstring" + +def bitey2(): + b' also not a docstring' + +def triple_quoted_bytes(): + b""" not a docstring""" + +def triple_quoted_bytes2(): + b''' also not a docstring''' + +def capitalized_bytes(): + B" NOT A DOCSTRING" + +# output +def bitey(): + b" not a docstring" + + +def bitey2(): + b" also not a docstring" + + +def triple_quoted_bytes(): + b""" not a docstring""" + + +def triple_quoted_bytes2(): + b""" also not a docstring""" + + +def capitalized_bytes(): + b" NOT A DOCSTRING" \ No newline at end of file diff --git a/tests/data/raw_docstring.py b/tests/data/cases/raw_docstring.py similarity index 100% rename from tests/data/raw_docstring.py rename to tests/data/cases/raw_docstring.py From e2f2bd076fbc19d4adb90b70b5a7be32b08d5dbe Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 1 Nov 2023 06:20:14 -0700 Subject: [PATCH 19/33] Minor refactoring in get_sources and gen_python_files (#4013) --- src/black/__init__.py | 39 ++++++++++++++++++--------------------- src/black/files.py | 7 ++----- tests/test_black.py | 5 +++-- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 7cf93b89e42..c11a66b7bc8 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -50,6 +50,7 @@ get_gitignore, normalize_path_maybe_ignore, parse_pyproject_toml, + path_is_excluded, wrap_stream_for_windows, ) from black.handle_ipynb_magics import ( @@ -632,15 +633,15 @@ def get_sources( for s in src: if s == "-" and stdin_filename: - p = Path(stdin_filename) + path = Path(stdin_filename) is_stdin = True else: - p = Path(s) + path = Path(s) is_stdin = False - if is_stdin or p.is_file(): + if is_stdin or path.is_file(): normalized_path: Optional[str] = normalize_path_maybe_ignore( - p, root, report + path, root, report ) if normalized_path is None: if verbose: @@ -651,38 +652,34 @@ def get_sources( normalized_path = "/" + normalized_path # Hard-exclude any files that matches the `--force-exclude` regex. - if force_exclude: - force_exclude_match = force_exclude.search(normalized_path) - else: - force_exclude_match = None - if force_exclude_match and force_exclude_match.group(0): - report.path_ignored(p, "matches the --force-exclude regular expression") + if path_is_excluded(normalized_path, force_exclude): + report.path_ignored( + path, "matches the --force-exclude regular expression" + ) continue if is_stdin: - p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") + path = Path(f"{STDIN_PLACEHOLDER}{str(path)}") - if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed( + if path.suffix == ".ipynb" and not jupyter_dependencies_are_installed( warn=verbose or not quiet ): continue - sources.add(p) - elif p.is_dir(): - p_relative = normalize_path_maybe_ignore(p, root, report) - assert p_relative is not None - p = root / p_relative + sources.add(path) + elif path.is_dir(): + path = root / (path.resolve().relative_to(root)) if verbose: - out(f'Found input source directory: "{p}"', fg="blue") + out(f'Found input source directory: "{path}"', fg="blue") if using_default_exclude: gitignore = { root: root_gitignore, - p: get_gitignore(p), + path: get_gitignore(path), } sources.update( gen_python_files( - p.iterdir(), + path.iterdir(), root, include, exclude, @@ -697,7 +694,7 @@ def get_sources( elif s == "-": if verbose: out("Found input source stdin", fg="blue") - sources.add(p) + sources.add(path) else: err(f"invalid path: {s}") diff --git a/src/black/files.py b/src/black/files.py index 1eed7eda828..858303ca1a3 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -280,7 +280,6 @@ def _path_is_ignored( root_relative_path: str, root: Path, gitignore_dict: Dict[Path, PathSpec], - report: Report, ) -> bool: path = root / root_relative_path # Note that this logic is sensitive to the ordering of gitignore_dict. Callers must @@ -291,9 +290,6 @@ def _path_is_ignored( except ValueError: break if pattern.match_file(relative_path): - report.path_ignored( - path.relative_to(root), "matches a .gitignore file content" - ) return True return False @@ -334,8 +330,9 @@ def gen_python_files( # First ignore files matching .gitignore, if passed if gitignore_dict and _path_is_ignored( - root_relative_path, root, gitignore_dict, report + root_relative_path, root, gitignore_dict ): + report.path_ignored(child, "matches a .gitignore file content") continue # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options. diff --git a/tests/test_black.py b/tests/test_black.py index 56c20243020..c7196098e14 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -504,7 +504,7 @@ def _mocked_calls() -> bool: return _mocked_calls with patch("pathlib.Path.iterdir", return_value=target_contents), patch( - "pathlib.Path.cwd", return_value=working_directory + "pathlib.Path.resolve", return_value=target_abspath ), patch("pathlib.Path.is_dir", side_effect=mock_n_calls([True])): # Note that the root folder (project_root) isn't the folder # named "root" (aka working_directory) @@ -526,7 +526,8 @@ def _mocked_calls() -> bool: for _, mock_args, _ in report.path_ignored.mock_calls ), "A symbolic link was reported." report.path_ignored.assert_called_once_with( - Path("root", "child", "b.py"), "matches a .gitignore file content" + Path(working_directory, "child", "b.py"), + "matches a .gitignore file content", ) def test_report_verbose(self) -> None: From c54c213d6a3132986feede0cf0525f5bae5b43d6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 2 Nov 2023 20:42:11 -0700 Subject: [PATCH 20/33] Fix crash on await (a ** b) (#3994) --- CHANGES.md | 2 ++ src/black/linegen.py | 22 ++++++++++------------ tests/data/cases/remove_await_parens.py | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f365f1c239b..5ce37943693 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,8 @@ - Fix crash on formatting bytes strings that look like docstrings (#4003) - Fix crash when whitespace followed a backslash before newline in a docstring (#4008) +- Fix crash on formatting code like `await (a ** b)` (#3994) + ### Preview style - Multiline dictionaries and lists that are the sole argument to a function are now diff --git a/src/black/linegen.py b/src/black/linegen.py index 121c6e314fe..b13b95d9b31 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1352,18 +1352,16 @@ def remove_await_parens(node: Node) -> None: opening_bracket = cast(Leaf, node.children[1].children[0]) closing_bracket = cast(Leaf, node.children[1].children[-1]) bracket_contents = node.children[1].children[1] - if isinstance(bracket_contents, Node): - if bracket_contents.type != syms.power: - ensure_visible(opening_bracket) - ensure_visible(closing_bracket) - elif ( - bracket_contents.type == syms.power - and bracket_contents.children[0].type == token.AWAIT - ): - ensure_visible(opening_bracket) - ensure_visible(closing_bracket) - # If we are in a nested await then recurse down. - remove_await_parens(bracket_contents) + if isinstance(bracket_contents, Node) and ( + bracket_contents.type != syms.power + or bracket_contents.children[0].type == token.AWAIT + or any( + isinstance(child, Leaf) and child.type == token.DOUBLESTAR + for child in bracket_contents.children + ) + ): + ensure_visible(opening_bracket) + ensure_visible(closing_bracket) def _maybe_wrap_cms_in_parens( diff --git a/tests/data/cases/remove_await_parens.py b/tests/data/cases/remove_await_parens.py index 8c7223d2f39..073150c5f08 100644 --- a/tests/data/cases/remove_await_parens.py +++ b/tests/data/cases/remove_await_parens.py @@ -80,6 +80,15 @@ async def main(): async def main(): await (yield) +async def main(): + await (a ** b) + await (a[b] ** c) + await (a ** b[c]) + await ((a + b) ** (c + d)) + await (a + b) + await (a[b]) + await (a[b ** c]) + # output import asyncio @@ -174,3 +183,13 @@ async def main(): async def main(): await (yield) + + +async def main(): + await (a**b) + await (a[b] ** c) + await (a ** b[c]) + await ((a + b) ** (c + d)) + await (a + b) + await a[b] + await a[b**c] From 448324637d12514b540efb33b4df7bf8af10c6d5 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sat, 4 Nov 2023 22:49:12 +0200 Subject: [PATCH 21/33] Enable branch coverage (#4022) When trying to understand the code logic, and looking at coverage reports, branch coverage is very helpful. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8c55076e4c9..f3689bfb746 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -224,6 +224,7 @@ omit = [ ] [tool.coverage.run] relative_files = true +branch = true [tool.mypy] # Specify the target platform details in config, so your developers are From 9e3daa1107a66f311a8367395a33ed5fc5d5e73d Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 5 Nov 2023 18:29:37 -0800 Subject: [PATCH 22/33] Fix arm wheels on macOS (#4017) --- .github/workflows/pypi_upload.yml | 7 ++++--- CHANGES.md | 2 ++ pyproject.toml | 13 ++++++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index a57013d67c1..07273f09508 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -68,9 +68,10 @@ jobs: - name: generate matrix (PR) if: github.event_name == 'pull_request' run: | - cibuildwheel --print-build-identifiers --platform linux \ - | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' \ - | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix + { + cibuildwheel --print-build-identifiers --platform linux \ + | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' + } | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix env: CIBW_BUILD: "cp38-* cp311-*" CIBW_ARCHS_LINUX: x86_64 diff --git a/CHANGES.md b/CHANGES.md index 5ce37943693..97084a2bfc1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -38,6 +38,8 @@ +- Fix mypyc builds on arm64 on macOS (#4017) + ### Output diff --git a/pyproject.toml b/pyproject.toml index f3689bfb746..c0302d2302a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,8 @@ exclude = ["/profiling"] [tool.hatch.build.targets.wheel] only-include = ["src"] sources = ["src"] +# Note that we change the behaviour of this flag below +macos-max-compat = true [tool.hatch.build.targets.wheel.hooks.mypyc] enable-by-default = false @@ -175,9 +177,18 @@ before-build = [ HATCH_BUILD_HOOKS_ENABLE = "1" MYPYC_OPT_LEVEL = "3" MYPYC_DEBUG_LEVEL = "0" +AIOHTTP_NO_EXTENSIONS = "1" + # Black needs Clang to compile successfully on Linux. CC = "clang" -AIOHTTP_NO_EXTENSIONS = "1" + +[tool.cibuildwheel.macos] +build-frontend = { name = "build", args = ["--no-isolation"] } +# Unfortunately, hatch doesn't respect MACOSX_DEPLOYMENT_TARGET +before-build = [ + "python -m pip install 'hatchling==1.18.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy==1.5.1' 'click==8.1.3'", + """sed -i '' -e "600,700s/'10_16'/os.environ['MACOSX_DEPLOYMENT_TARGET'].replace('.', '_')/" $(python -c 'import hatchling.builders.wheel as h; print(h.__file__)') """, +] [tool.isort] atomic = true From e808e61db8c7a8f9c7fd4b2fff2281141f6b2517 Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Mon, 6 Nov 2023 14:30:04 -0800 Subject: [PATCH 23/33] Preview: Keep requiring two empty lines between module-level docstring and first function or class definition (#4028) Fixes #4027. --- CHANGES.md | 2 ++ src/black/lines.py | 1 + .../data/cases/module_docstring_followed_by_class.py | 11 +++++++++++ .../cases/module_docstring_followed_by_function.py | 11 +++++++++++ 4 files changed, 25 insertions(+) create mode 100644 tests/data/cases/module_docstring_followed_by_class.py create mode 100644 tests/data/cases/module_docstring_followed_by_function.py diff --git a/CHANGES.md b/CHANGES.md index 97084a2bfc1..a68f87bfc12 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,8 @@ indented less (#3964) - Multiline list and dict unpacking as the sole argument to a function is now also indented less (#3992) +- Keep requiring two empty lines between module-level docstring and first function or + class definition. (#4028) ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index a73c429e3d9..23c1a93d3d4 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -578,6 +578,7 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: and self.previous_block.previous_block is None and len(self.previous_block.original_line.leaves) == 1 and self.previous_block.original_line.is_triple_quoted_string + and not (current_line.is_class or current_line.is_def) ): before = 1 diff --git a/tests/data/cases/module_docstring_followed_by_class.py b/tests/data/cases/module_docstring_followed_by_class.py new file mode 100644 index 00000000000..6fdbfc8c240 --- /dev/null +++ b/tests/data/cases/module_docstring_followed_by_class.py @@ -0,0 +1,11 @@ +# flags: --preview +"""Two blank lines between module docstring and a class.""" +class MyClass: + pass + +# output +"""Two blank lines between module docstring and a class.""" + + +class MyClass: + pass diff --git a/tests/data/cases/module_docstring_followed_by_function.py b/tests/data/cases/module_docstring_followed_by_function.py new file mode 100644 index 00000000000..5913a59e1fe --- /dev/null +++ b/tests/data/cases/module_docstring_followed_by_function.py @@ -0,0 +1,11 @@ +# flags: --preview +"""Two blank lines between module docstring and a function def.""" +def function(): + pass + +# output +"""Two blank lines between module docstring and a function def.""" + + +def function(): + pass From ecbd9e8cf71f13068c7e6803a534e00363114c91 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:58:43 -0800 Subject: [PATCH 24/33] Fix crash with f-string docstrings (#4019) Python does not consider f-strings to be docstrings, so we probably shouldn't be formatting them as such Fixes #4018 Co-authored-by: Alex Waygood --- CHANGES.md | 3 +++ src/black/nodes.py | 2 +- tests/data/cases/docstring_preview.py | 3 ++- tests/data/cases/f_docstring.py | 20 +++++++++++++++++++ ...view_docstring_no_string_normalization.py} | 0 5 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 tests/data/cases/f_docstring.py rename tests/data/cases/{docstring_preview_no_string_normalization.py => preview_docstring_no_string_normalization.py} (100%) diff --git a/CHANGES.md b/CHANGES.md index a68f87bfc12..b1fe25ef625 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,9 @@ - Fix crash on formatting code like `await (a ** b)` (#3994) +- No longer treat leading f-strings as docstrings. This matches Python's behaviour and + fixes a crash (#4019) + ### Preview style - Multiline dictionaries and lists that are the sole argument to a function are now diff --git a/src/black/nodes.py b/src/black/nodes.py index 5f6b280c035..fff8e05a118 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -529,7 +529,7 @@ def is_docstring(leaf: Leaf) -> bool: return False prefix = get_string_prefix(leaf.value) - if "b" in prefix or "B" in prefix: + if set(prefix).intersection("bBfF"): return False if prev_siblings_are( diff --git a/tests/data/cases/docstring_preview.py b/tests/data/cases/docstring_preview.py index ff4819acb67..a3c656be2f8 100644 --- a/tests/data/cases/docstring_preview.py +++ b/tests/data/cases/docstring_preview.py @@ -58,7 +58,8 @@ def docstring_almost_at_line_limit(): def docstring_almost_at_line_limit_with_prefix(): - f"""long docstring................................................................""" + f"""long docstring................................................................ + """ def mulitline_docstring_almost_at_line_limit(): diff --git a/tests/data/cases/f_docstring.py b/tests/data/cases/f_docstring.py new file mode 100644 index 00000000000..667f550b353 --- /dev/null +++ b/tests/data/cases/f_docstring.py @@ -0,0 +1,20 @@ +def foo(e): + f""" {'.'.join(e)}""" + +def bar(e): + f"{'.'.join(e)}" + +def baz(e): + F""" {'.'.join(e)}""" + +# output +def foo(e): + f""" {'.'.join(e)}""" + + +def bar(e): + f"{'.'.join(e)}" + + +def baz(e): + f""" {'.'.join(e)}""" diff --git a/tests/data/cases/docstring_preview_no_string_normalization.py b/tests/data/cases/preview_docstring_no_string_normalization.py similarity index 100% rename from tests/data/cases/docstring_preview_no_string_normalization.py rename to tests/data/cases/preview_docstring_no_string_normalization.py From 46be1f8e54ac9a7d67723c0fa28c7bec13a0a2bf Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Mon, 6 Nov 2023 18:05:25 -0800 Subject: [PATCH 25/33] Support formatting specified lines (#4020) --- CHANGES.md | 3 + docs/usage_and_configuration/the_basics.md | 17 + src/black/__init__.py | 130 ++++- src/black/nodes.py | 28 + src/black/ranges.py | 496 ++++++++++++++++++ tests/data/cases/line_ranges_basic.py | 107 ++++ tests/data/cases/line_ranges_fmt_off.py | 49 ++ .../cases/line_ranges_fmt_off_decorator.py | 27 + .../data/cases/line_ranges_fmt_off_overlap.py | 37 ++ tests/data/cases/line_ranges_imports.py | 9 + tests/data/cases/line_ranges_indentation.py | 27 + tests/data/cases/line_ranges_two_passes.py | 27 + tests/data/cases/line_ranges_unwrapping.py | 25 + tests/data/invalid_line_ranges.toml | 2 + tests/data/line_ranges_formatted/basic.py | 50 ++ .../line_ranges_formatted/pattern_matching.py | 25 + tests/test_black.py | 87 ++- tests/test_format.py | 26 +- tests/test_ranges.py | 185 +++++++ tests/util.py | 29 +- 20 files changed, 1358 insertions(+), 28 deletions(-) create mode 100644 src/black/ranges.py create mode 100644 tests/data/cases/line_ranges_basic.py create mode 100644 tests/data/cases/line_ranges_fmt_off.py create mode 100644 tests/data/cases/line_ranges_fmt_off_decorator.py create mode 100644 tests/data/cases/line_ranges_fmt_off_overlap.py create mode 100644 tests/data/cases/line_ranges_imports.py create mode 100644 tests/data/cases/line_ranges_indentation.py create mode 100644 tests/data/cases/line_ranges_two_passes.py create mode 100644 tests/data/cases/line_ranges_unwrapping.py create mode 100644 tests/data/invalid_line_ranges.toml create mode 100644 tests/data/line_ranges_formatted/basic.py create mode 100644 tests/data/line_ranges_formatted/pattern_matching.py create mode 100644 tests/test_ranges.py diff --git a/CHANGES.md b/CHANGES.md index b1fe25ef625..780a00247ce 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,9 @@ +- Support formatting ranges of lines with the new `--line-ranges` command-line option + (#4020). + ### Stable style - Fix crash on formatting bytes strings that look like docstrings (#4003) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index f25dbb13d4d..dbd8c7ba434 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -175,6 +175,23 @@ All done! ✨ 🍰 ✨ 1 file would be reformatted. ``` +### `--line-ranges` + +When specified, _Black_ will try its best to only format these lines. + +This option can be specified multiple times, and a union of the lines will be formatted. +Each range must be specified as two integers connected by a `-`: `-`. The +`` and `` integer indices are 1-based and inclusive on both ends. + +_Black_ may still format lines outside of the ranges for multi-line statements. +Formatting more than one file or any ipynb files with this option is not supported. This +option cannot be specified in the `pyproject.toml` config. + +Example: `black --line-ranges=1-10 --line-ranges=21-30 test.py` will format lines from +`1` to `10` and `21` to `30`. + +This option is mainly for editor integrations, such as "Format Selection". + #### `--color` / `--no-color` Show (or do not show) colored diff. Only applies when `--diff` is given. diff --git a/src/black/__init__.py b/src/black/__init__.py index c11a66b7bc8..5aca3316df0 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import ( Any, + Collection, Dict, Generator, Iterator, @@ -77,6 +78,7 @@ from black.output import color_diff, diff, dump_to_file, err, ipynb_diff, out from black.parsing import InvalidInput # noqa F401 from black.parsing import lib2to3_parse, parse_ast, stringify_ast +from black.ranges import adjusted_lines, convert_unchanged_lines, parse_line_ranges from black.report import Changed, NothingChanged, Report from black.trans import iter_fexpr_spans from blib2to3.pgen2 import token @@ -163,6 +165,12 @@ def read_pyproject_toml( "extend-exclude", "Config key extend-exclude must be a string" ) + line_ranges = config.get("line_ranges") + if line_ranges is not None: + raise click.BadOptionUsage( + "line-ranges", "Cannot use line-ranges in the pyproject.toml file." + ) + default_map: Dict[str, Any] = {} if ctx.default_map: default_map.update(ctx.default_map) @@ -304,6 +312,19 @@ def validate_regex( is_flag=True, help="Don't write the files back, just output a diff for each file on stdout.", ) +@click.option( + "--line-ranges", + multiple=True, + metavar="START-END", + help=( + "When specified, _Black_ will try its best to only format these lines. This" + " option can be specified multiple times, and a union of the lines will be" + " formatted. Each range must be specified as two integers connected by a `-`:" + " `-`. The `` and `` integer indices are 1-based and" + " inclusive on both ends." + ), + default=(), +) @click.option( "--color/--no-color", is_flag=True, @@ -443,6 +464,7 @@ def main( # noqa: C901 target_version: List[TargetVersion], check: bool, diff: bool, + line_ranges: Sequence[str], color: bool, fast: bool, pyi: bool, @@ -544,6 +566,18 @@ def main( # noqa: C901 python_cell_magics=set(python_cell_magics), ) + lines: List[Tuple[int, int]] = [] + if line_ranges: + if ipynb: + err("Cannot use --line-ranges with ipynb files.") + ctx.exit(1) + + try: + lines = parse_line_ranges(line_ranges) + except ValueError as e: + err(str(e)) + ctx.exit(1) + if code is not None: # Run in quiet mode by default with -c; the extra output isn't useful. # You can still pass -v to get verbose output. @@ -553,7 +587,12 @@ def main( # noqa: C901 if code is not None: reformat_code( - content=code, fast=fast, write_back=write_back, mode=mode, report=report + content=code, + fast=fast, + write_back=write_back, + mode=mode, + report=report, + lines=lines, ) else: assert root is not None # root is only None if code is not None @@ -588,10 +627,14 @@ def main( # noqa: C901 write_back=write_back, mode=mode, report=report, + lines=lines, ) else: from black.concurrency import reformat_many + if lines: + err("Cannot use --line-ranges to format multiple files.") + ctx.exit(1) reformat_many( sources=sources, fast=fast, @@ -714,7 +757,13 @@ def path_empty( def reformat_code( - content: str, fast: bool, write_back: WriteBack, mode: Mode, report: Report + content: str, + fast: bool, + write_back: WriteBack, + mode: Mode, + report: Report, + *, + lines: Collection[Tuple[int, int]] = (), ) -> None: """ Reformat and print out `content` without spawning child processes. @@ -727,7 +776,7 @@ def reformat_code( try: changed = Changed.NO if format_stdin_to_stdout( - content=content, fast=fast, write_back=write_back, mode=mode + content=content, fast=fast, write_back=write_back, mode=mode, lines=lines ): changed = Changed.YES report.done(path, changed) @@ -741,7 +790,13 @@ def reformat_code( # not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26 @mypyc_attr(patchable=True) def reformat_one( - src: Path, fast: bool, write_back: WriteBack, mode: Mode, report: "Report" + src: Path, + fast: bool, + write_back: WriteBack, + mode: Mode, + report: "Report", + *, + lines: Collection[Tuple[int, int]] = (), ) -> None: """Reformat a single file under `src` without spawning child processes. @@ -766,7 +821,9 @@ def reformat_one( mode = replace(mode, is_pyi=True) elif src.suffix == ".ipynb": mode = replace(mode, is_ipynb=True) - if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): + if format_stdin_to_stdout( + fast=fast, write_back=write_back, mode=mode, lines=lines + ): changed = Changed.YES else: cache = Cache.read(mode) @@ -774,7 +831,7 @@ def reformat_one( if not cache.is_changed(src): changed = Changed.CACHED if changed is not Changed.CACHED and format_file_in_place( - src, fast=fast, write_back=write_back, mode=mode + src, fast=fast, write_back=write_back, mode=mode, lines=lines ): changed = Changed.YES if (write_back is WriteBack.YES and changed is not Changed.CACHED) or ( @@ -794,6 +851,8 @@ def format_file_in_place( mode: Mode, write_back: WriteBack = WriteBack.NO, lock: Any = None, # multiprocessing.Manager().Lock() is some crazy proxy + *, + lines: Collection[Tuple[int, int]] = (), ) -> bool: """Format file under `src` path. Return True if changed. @@ -813,7 +872,9 @@ def format_file_in_place( header = buf.readline() src_contents, encoding, newline = decode_bytes(buf.read()) try: - dst_contents = format_file_contents(src_contents, fast=fast, mode=mode) + dst_contents = format_file_contents( + src_contents, fast=fast, mode=mode, lines=lines + ) except NothingChanged: return False except JSONDecodeError: @@ -858,6 +919,7 @@ def format_stdin_to_stdout( content: Optional[str] = None, write_back: WriteBack = WriteBack.NO, mode: Mode, + lines: Collection[Tuple[int, int]] = (), ) -> bool: """Format file on stdin. Return True if changed. @@ -876,7 +938,7 @@ def format_stdin_to_stdout( dst = src try: - dst = format_file_contents(src, fast=fast, mode=mode) + dst = format_file_contents(src, fast=fast, mode=mode, lines=lines) return True except NothingChanged: @@ -904,7 +966,11 @@ def format_stdin_to_stdout( def check_stability_and_equivalence( - src_contents: str, dst_contents: str, *, mode: Mode + src_contents: str, + dst_contents: str, + *, + mode: Mode, + lines: Collection[Tuple[int, int]] = (), ) -> None: """Perform stability and equivalence checks. @@ -913,10 +979,16 @@ def check_stability_and_equivalence( content differently. """ assert_equivalent(src_contents, dst_contents) - assert_stable(src_contents, dst_contents, mode=mode) + assert_stable(src_contents, dst_contents, mode=mode, lines=lines) -def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: +def format_file_contents( + src_contents: str, + *, + fast: bool, + mode: Mode, + lines: Collection[Tuple[int, int]] = (), +) -> FileContent: """Reformat contents of a file and return new contents. If `fast` is False, additionally confirm that the reformatted code is @@ -926,13 +998,15 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo if mode.is_ipynb: dst_contents = format_ipynb_string(src_contents, fast=fast, mode=mode) else: - dst_contents = format_str(src_contents, mode=mode) + dst_contents = format_str(src_contents, mode=mode, lines=lines) if src_contents == dst_contents: raise NothingChanged if not fast and not mode.is_ipynb: # Jupyter notebooks will already have been checked above. - check_stability_and_equivalence(src_contents, dst_contents, mode=mode) + check_stability_and_equivalence( + src_contents, dst_contents, mode=mode, lines=lines + ) return dst_contents @@ -1043,7 +1117,9 @@ def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileCon raise NothingChanged -def format_str(src_contents: str, *, mode: Mode) -> str: +def format_str( + src_contents: str, *, mode: Mode, lines: Collection[Tuple[int, int]] = () +) -> str: """Reformat a string and return new contents. `mode` determines formatting options, such as how many characters per line are @@ -1073,16 +1149,20 @@ def f( hey """ - dst_contents = _format_str_once(src_contents, mode=mode) + dst_contents = _format_str_once(src_contents, mode=mode, lines=lines) # Forced second pass to work around optional trailing commas (becoming # forced trailing commas on pass 2) interacting differently with optional # parentheses. Admittedly ugly. if src_contents != dst_contents: - return _format_str_once(dst_contents, mode=mode) + if lines: + lines = adjusted_lines(lines, src_contents, dst_contents) + return _format_str_once(dst_contents, mode=mode, lines=lines) return dst_contents -def _format_str_once(src_contents: str, *, mode: Mode) -> str: +def _format_str_once( + src_contents: str, *, mode: Mode, lines: Collection[Tuple[int, int]] = () +) -> str: src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions) dst_blocks: List[LinesBlock] = [] if mode.target_versions: @@ -1097,7 +1177,11 @@ def _format_str_once(src_contents: str, *, mode: Mode) -> str: if supports_feature(versions, feature) } normalize_fmt_off(src_node, mode) - lines = LineGenerator(mode=mode, features=context_manager_features) + if lines: + # This should be called after normalize_fmt_off. + convert_unchanged_lines(src_node, lines) + + line_generator = LineGenerator(mode=mode, features=context_manager_features) elt = EmptyLineTracker(mode=mode) split_line_features = { feature @@ -1105,7 +1189,7 @@ def _format_str_once(src_contents: str, *, mode: Mode) -> str: if supports_feature(versions, feature) } block: Optional[LinesBlock] = None - for current_line in lines.visit(src_node): + for current_line in line_generator.visit(src_node): block = elt.maybe_empty_lines(current_line) dst_blocks.append(block) for line in transform_line( @@ -1373,12 +1457,16 @@ def assert_equivalent(src: str, dst: str) -> None: ) from None -def assert_stable(src: str, dst: str, mode: Mode) -> None: +def assert_stable( + src: str, dst: str, mode: Mode, *, lines: Collection[Tuple[int, int]] = () +) -> None: """Raise AssertionError if `dst` reformats differently the second time.""" # We shouldn't call format_str() here, because that formats the string # twice and may hide a bug where we bounce back and forth between two # versions. - newdst = _format_str_once(dst, mode=mode) + if lines: + lines = adjusted_lines(lines, src, dst) + newdst = _format_str_once(dst, mode=mode, lines=lines) if dst != newdst: log = dump_to_file( str(mode), diff --git a/src/black/nodes.py b/src/black/nodes.py index fff8e05a118..9251b0defb0 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -935,3 +935,31 @@ def is_part_of_annotation(leaf: Leaf) -> bool: return True ancestor = ancestor.parent return False + + +def first_leaf(node: LN) -> Optional[Leaf]: + """Returns the first leaf of the ancestor node.""" + if isinstance(node, Leaf): + return node + elif not node.children: + return None + else: + return first_leaf(node.children[0]) + + +def last_leaf(node: LN) -> Optional[Leaf]: + """Returns the last leaf of the ancestor node.""" + if isinstance(node, Leaf): + return node + elif not node.children: + return None + else: + return last_leaf(node.children[-1]) + + +def furthest_ancestor_with_last_leaf(leaf: Leaf) -> LN: + """Returns the furthest ancestor that has this leaf node as the last leaf.""" + node: LN = leaf + while node.parent and node.parent.children and node is node.parent.children[-1]: + node = node.parent + return node diff --git a/src/black/ranges.py b/src/black/ranges.py new file mode 100644 index 00000000000..b0c312e6274 --- /dev/null +++ b/src/black/ranges.py @@ -0,0 +1,496 @@ +"""Functions related to Black's formatting by line ranges feature.""" + +import difflib +from dataclasses import dataclass +from typing import Collection, Iterator, List, Sequence, Set, Tuple, Union + +from black.nodes import ( + LN, + STANDALONE_COMMENT, + Leaf, + Node, + Visitor, + first_leaf, + furthest_ancestor_with_last_leaf, + last_leaf, + syms, +) +from blib2to3.pgen2.token import ASYNC, NEWLINE + + +def parse_line_ranges(line_ranges: Sequence[str]) -> List[Tuple[int, int]]: + lines: List[Tuple[int, int]] = [] + for lines_str in line_ranges: + parts = lines_str.split("-") + if len(parts) != 2: + raise ValueError( + "Incorrect --line-ranges format, expect 'START-END', found" + f" {lines_str!r}" + ) + try: + start = int(parts[0]) + end = int(parts[1]) + except ValueError: + raise ValueError( + "Incorrect --line-ranges value, expect integer ranges, found" + f" {lines_str!r}" + ) from None + else: + lines.append((start, end)) + return lines + + +def is_valid_line_range(lines: Tuple[int, int]) -> bool: + """Returns whether the line range is valid.""" + return not lines or lines[0] <= lines[1] + + +def adjusted_lines( + lines: Collection[Tuple[int, int]], + original_source: str, + modified_source: str, +) -> List[Tuple[int, int]]: + """Returns the adjusted line ranges based on edits from the original code. + + This computes the new line ranges by diffing original_source and + modified_source, and adjust each range based on how the range overlaps with + the diffs. + + Note the diff can contain lines outside of the original line ranges. This can + happen when the formatting has to be done in adjacent to maintain consistent + local results. For example: + + 1. def my_func(arg1, arg2, + 2. arg3,): + 3. pass + + If it restricts to line 2-2, it can't simply reformat line 2, it also has + to reformat line 1: + + 1. def my_func( + 2. arg1, + 3. arg2, + 4. arg3, + 5. ): + 6. pass + + In this case, we will expand the line ranges to also include the whole diff + block. + + Args: + lines: a collection of line ranges. + original_source: the original source. + modified_source: the modified source. + """ + lines_mappings = _calculate_lines_mappings(original_source, modified_source) + + new_lines = [] + # Keep an index of the current search. Since the lines and lines_mappings are + # sorted, this makes the search complexity linear. + current_mapping_index = 0 + for start, end in sorted(lines): + start_mapping_index = _find_lines_mapping_index( + start, + lines_mappings, + current_mapping_index, + ) + end_mapping_index = _find_lines_mapping_index( + end, + lines_mappings, + start_mapping_index, + ) + current_mapping_index = start_mapping_index + if start_mapping_index >= len(lines_mappings) or end_mapping_index >= len( + lines_mappings + ): + # Protect against invalid inputs. + continue + start_mapping = lines_mappings[start_mapping_index] + end_mapping = lines_mappings[end_mapping_index] + if start_mapping.is_changed_block: + # When the line falls into a changed block, expands to the whole block. + new_start = start_mapping.modified_start + else: + new_start = ( + start - start_mapping.original_start + start_mapping.modified_start + ) + if end_mapping.is_changed_block: + # When the line falls into a changed block, expands to the whole block. + new_end = end_mapping.modified_end + else: + new_end = end - end_mapping.original_start + end_mapping.modified_start + new_range = (new_start, new_end) + if is_valid_line_range(new_range): + new_lines.append(new_range) + return new_lines + + +def convert_unchanged_lines(src_node: Node, lines: Collection[Tuple[int, int]]) -> None: + """Converts unchanged lines to STANDALONE_COMMENT. + + The idea is similar to how `# fmt: on/off` is implemented. It also converts the + nodes between those markers as a single `STANDALONE_COMMENT` leaf node with + the unformatted code as its value. `STANDALONE_COMMENT` is a "fake" token + that will be formatted as-is with its prefix normalized. + + Here we perform two passes: + + 1. Visit the top-level statements, and convert them to a single + `STANDALONE_COMMENT` when unchanged. This speeds up formatting when some + of the top-level statements aren't changed. + 2. Convert unchanged "unwrapped lines" to `STANDALONE_COMMENT` nodes line by + line. "unwrapped lines" are divided by the `NEWLINE` token. e.g. a + multi-line statement is *one* "unwrapped line" that ends with `NEWLINE`, + even though this statement itself can span multiple lines, and the + tokenizer only sees the last '\n' as the `NEWLINE` token. + + NOTE: During pass (2), comment prefixes and indentations are ALWAYS + normalized even when the lines aren't changed. This is fixable by moving + more formatting to pass (1). However, it's hard to get it correct when + incorrect indentations are used. So we defer this to future optimizations. + """ + lines_set: Set[int] = set() + for start, end in lines: + lines_set.update(range(start, end + 1)) + visitor = _TopLevelStatementsVisitor(lines_set) + _ = list(visitor.visit(src_node)) # Consume all results. + _convert_unchanged_line_by_line(src_node, lines_set) + + +def _contains_standalone_comment(node: LN) -> bool: + if isinstance(node, Leaf): + return node.type == STANDALONE_COMMENT + else: + for child in node.children: + if _contains_standalone_comment(child): + return True + return False + + +class _TopLevelStatementsVisitor(Visitor[None]): + """ + A node visitor that converts unchanged top-level statements to + STANDALONE_COMMENT. + + This is used in addition to _convert_unchanged_lines_by_flatterning, to + speed up formatting when there are unchanged top-level + classes/functions/statements. + """ + + def __init__(self, lines_set: Set[int]): + self._lines_set = lines_set + + def visit_simple_stmt(self, node: Node) -> Iterator[None]: + # This is only called for top-level statements, since `visit_suite` + # won't visit its children nodes. + yield from [] + newline_leaf = last_leaf(node) + if not newline_leaf: + return + assert ( + newline_leaf.type == NEWLINE + ), f"Unexpectedly found leaf.type={newline_leaf.type}" + # We need to find the furthest ancestor with the NEWLINE as the last + # leaf, since a `suite` can simply be a `simple_stmt` when it puts + # its body on the same line. Example: `if cond: pass`. + ancestor = furthest_ancestor_with_last_leaf(newline_leaf) + if not _get_line_range(ancestor).intersection(self._lines_set): + _convert_node_to_standalone_comment(ancestor) + + def visit_suite(self, node: Node) -> Iterator[None]: + yield from [] + # If there is a STANDALONE_COMMENT node, it means parts of the node tree + # have fmt on/off/skip markers. Those STANDALONE_COMMENT nodes can't + # be simply converted by calling str(node). So we just don't convert + # here. + if _contains_standalone_comment(node): + return + # Find the semantic parent of this suite. For `async_stmt` and + # `async_funcdef`, the ASYNC token is defined on a separate level by the + # grammar. + semantic_parent = node.parent + if semantic_parent is not None: + if ( + semantic_parent.prev_sibling is not None + and semantic_parent.prev_sibling.type == ASYNC + ): + semantic_parent = semantic_parent.parent + if semantic_parent is not None and not _get_line_range( + semantic_parent + ).intersection(self._lines_set): + _convert_node_to_standalone_comment(semantic_parent) + + +def _convert_unchanged_line_by_line(node: Node, lines_set: Set[int]) -> None: + """Converts unchanged to STANDALONE_COMMENT line by line.""" + for leaf in node.leaves(): + if leaf.type != NEWLINE: + # We only consider "unwrapped lines", which are divided by the NEWLINE + # token. + continue + if leaf.parent and leaf.parent.type == syms.match_stmt: + # The `suite` node is defined as: + # match_stmt: "match" subject_expr ':' NEWLINE INDENT case_block+ DEDENT + # Here we need to check `subject_expr`. The `case_block+` will be + # checked by their own NEWLINEs. + nodes_to_ignore: List[LN] = [] + prev_sibling = leaf.prev_sibling + while prev_sibling: + nodes_to_ignore.insert(0, prev_sibling) + prev_sibling = prev_sibling.prev_sibling + if not _get_line_range(nodes_to_ignore).intersection(lines_set): + _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf) + elif leaf.parent and leaf.parent.type == syms.suite: + # The `suite` node is defined as: + # suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT + # We will check `simple_stmt` and `stmt+` separately against the lines set + parent_sibling = leaf.parent.prev_sibling + nodes_to_ignore = [] + while parent_sibling and not parent_sibling.type == syms.suite: + # NOTE: Multiple suite nodes can exist as siblings in e.g. `if_stmt`. + nodes_to_ignore.insert(0, parent_sibling) + parent_sibling = parent_sibling.prev_sibling + # Special case for `async_stmt` and `async_funcdef` where the ASYNC + # token is on the grandparent node. + grandparent = leaf.parent.parent + if ( + grandparent is not None + and grandparent.prev_sibling is not None + and grandparent.prev_sibling.type == ASYNC + ): + nodes_to_ignore.insert(0, grandparent.prev_sibling) + if not _get_line_range(nodes_to_ignore).intersection(lines_set): + _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf) + else: + ancestor = furthest_ancestor_with_last_leaf(leaf) + # Consider multiple decorators as a whole block, as their + # newlines have different behaviors than the rest of the grammar. + if ( + ancestor.type == syms.decorator + and ancestor.parent + and ancestor.parent.type == syms.decorators + ): + ancestor = ancestor.parent + if not _get_line_range(ancestor).intersection(lines_set): + _convert_node_to_standalone_comment(ancestor) + + +def _convert_node_to_standalone_comment(node: LN) -> None: + """Convert node to STANDALONE_COMMENT by modifying the tree inline.""" + parent = node.parent + if not parent: + return + first = first_leaf(node) + last = last_leaf(node) + if not first or not last: + return + if first is last: + # This can happen on the following edge cases: + # 1. A block of `# fmt: off/on` code except the `# fmt: on` is placed + # on the end of the last line instead of on a new line. + # 2. A single backslash on its own line followed by a comment line. + # Ideally we don't want to format them when not requested, but fixing + # isn't easy. These cases are also badly formatted code, so it isn't + # too bad we reformat them. + return + # The prefix contains comments and indentation whitespaces. They are + # reformatted accordingly to the correct indentation level. + # This also means the indentation will be changed on the unchanged lines, and + # this is actually required to not break incremental reformatting. + prefix = first.prefix + first.prefix = "" + index = node.remove() + if index is not None: + # Remove the '\n', as STANDALONE_COMMENT will have '\n' appended when + # genearting the formatted code. + value = str(node)[:-1] + parent.insert_child( + index, + Leaf( + STANDALONE_COMMENT, + value, + prefix=prefix, + fmt_pass_converted_first_leaf=first, + ), + ) + + +def _convert_nodes_to_standalone_comment(nodes: Sequence[LN], *, newline: Leaf) -> None: + """Convert nodes to STANDALONE_COMMENT by modifying the tree inline.""" + if not nodes: + return + parent = nodes[0].parent + first = first_leaf(nodes[0]) + if not parent or not first: + return + prefix = first.prefix + first.prefix = "" + value = "".join(str(node) for node in nodes) + # The prefix comment on the NEWLINE leaf is the trailing comment of the statement. + if newline.prefix: + value += newline.prefix + newline.prefix = "" + index = nodes[0].remove() + for node in nodes[1:]: + node.remove() + if index is not None: + parent.insert_child( + index, + Leaf( + STANDALONE_COMMENT, + value, + prefix=prefix, + fmt_pass_converted_first_leaf=first, + ), + ) + + +def _leaf_line_end(leaf: Leaf) -> int: + """Returns the line number of the leaf node's last line.""" + if leaf.type == NEWLINE: + return leaf.lineno + else: + # Leaf nodes like multiline strings can occupy multiple lines. + return leaf.lineno + str(leaf).count("\n") + + +def _get_line_range(node_or_nodes: Union[LN, List[LN]]) -> Set[int]: + """Returns the line range of this node or list of nodes.""" + if isinstance(node_or_nodes, list): + nodes = node_or_nodes + if not nodes: + return set() + first = first_leaf(nodes[0]) + last = last_leaf(nodes[-1]) + if first and last: + line_start = first.lineno + line_end = _leaf_line_end(last) + return set(range(line_start, line_end + 1)) + else: + return set() + else: + node = node_or_nodes + if isinstance(node, Leaf): + return set(range(node.lineno, _leaf_line_end(node) + 1)) + else: + first = first_leaf(node) + last = last_leaf(node) + if first and last: + return set(range(first.lineno, _leaf_line_end(last) + 1)) + else: + return set() + + +@dataclass +class _LinesMapping: + """1-based lines mapping from original source to modified source. + + Lines [original_start, original_end] from original source + are mapped to [modified_start, modified_end]. + + The ranges are inclusive on both ends. + """ + + original_start: int + original_end: int + modified_start: int + modified_end: int + # Whether this range corresponds to a changed block, or an unchanged block. + is_changed_block: bool + + +def _calculate_lines_mappings( + original_source: str, + modified_source: str, +) -> Sequence[_LinesMapping]: + """Returns a sequence of _LinesMapping by diffing the sources. + + For example, given the following diff: + import re + - def func(arg1, + - arg2, arg3): + + def func(arg1, arg2, arg3): + pass + It returns the following mappings: + original -> modified + (1, 1) -> (1, 1), is_changed_block=False (the "import re" line) + (2, 3) -> (2, 2), is_changed_block=True (the diff) + (4, 4) -> (3, 3), is_changed_block=False (the "pass" line) + + You can think of this visually as if it brings up a side-by-side diff, and tries + to map the line ranges from the left side to the right side: + + (1, 1)->(1, 1) 1. import re 1. import re + (2, 3)->(2, 2) 2. def func(arg1, 2. def func(arg1, arg2, arg3): + 3. arg2, arg3): + (4, 4)->(3, 3) 4. pass 3. pass + + Args: + original_source: the original source. + modified_source: the modified source. + """ + matcher = difflib.SequenceMatcher( + None, + original_source.splitlines(keepends=True), + modified_source.splitlines(keepends=True), + ) + matching_blocks = matcher.get_matching_blocks() + lines_mappings: List[_LinesMapping] = [] + # matching_blocks is a sequence of "same block of code ranges", see + # https://docs.python.org/3/library/difflib.html#difflib.SequenceMatcher.get_matching_blocks + # Each block corresponds to a _LinesMapping with is_changed_block=False, + # and the ranges between two blocks corresponds to a _LinesMapping with + # is_changed_block=True, + # NOTE: matching_blocks is 0-based, but _LinesMapping is 1-based. + for i, block in enumerate(matching_blocks): + if i == 0: + if block.a != 0 or block.b != 0: + lines_mappings.append( + _LinesMapping( + original_start=1, + original_end=block.a, + modified_start=1, + modified_end=block.b, + is_changed_block=False, + ) + ) + else: + previous_block = matching_blocks[i - 1] + lines_mappings.append( + _LinesMapping( + original_start=previous_block.a + previous_block.size + 1, + original_end=block.a, + modified_start=previous_block.b + previous_block.size + 1, + modified_end=block.b, + is_changed_block=True, + ) + ) + if i < len(matching_blocks) - 1: + lines_mappings.append( + _LinesMapping( + original_start=block.a + 1, + original_end=block.a + block.size, + modified_start=block.b + 1, + modified_end=block.b + block.size, + is_changed_block=False, + ) + ) + return lines_mappings + + +def _find_lines_mapping_index( + original_line: int, + lines_mappings: Sequence[_LinesMapping], + start_index: int, +) -> int: + """Returns the original index of the lines mappings for the original line.""" + index = start_index + while index < len(lines_mappings): + mapping = lines_mappings[index] + if ( + mapping.original_start <= original_line + and original_line <= mapping.original_end + ): + return index + index += 1 + return index diff --git a/tests/data/cases/line_ranges_basic.py b/tests/data/cases/line_ranges_basic.py new file mode 100644 index 00000000000..9f0fb2da70e --- /dev/null +++ b/tests/data/cases/line_ranges_basic.py @@ -0,0 +1,107 @@ +# flags: --line-ranges=5-6 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo2(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo3(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo4(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass + +# Adding some unformated code covering a wide range of syntaxes. + +if True: + # Incorrectly indented prefix comments. + pass + +import typing +from typing import ( + Any , + ) +class MyClass( object): # Trailing comment with extra leading space. + #NOTE: The following indentation is incorrect: + @decor( 1 * 3 ) + def my_func( arg): + pass + +try: # Trailing comment with extra leading space. + for i in range(10): # Trailing comment with extra leading space. + while condition: + if something: + then_something( ) + elif something_else: + then_something_else( ) +except ValueError as e: + unformatted( ) +finally: + unformatted( ) + +async def test_async_unformatted( ): # Trailing comment with extra leading space. + async for i in some_iter( unformatted ): # Trailing comment with extra leading space. + await asyncio.sleep( 1 ) + async with some_context( unformatted ): + print( "unformatted" ) + + +# output +# flags: --line-ranges=5-6 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo2( + parameter_1, + parameter_2, + parameter_3, + parameter_4, + parameter_5, + parameter_6, + parameter_7, +): + pass + + +def foo3( + parameter_1, + parameter_2, + parameter_3, + parameter_4, + parameter_5, + parameter_6, + parameter_7, +): + pass + + +def foo4(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass + +# Adding some unformated code covering a wide range of syntaxes. + +if True: + # Incorrectly indented prefix comments. + pass + +import typing +from typing import ( + Any , + ) +class MyClass( object): # Trailing comment with extra leading space. + #NOTE: The following indentation is incorrect: + @decor( 1 * 3 ) + def my_func( arg): + pass + +try: # Trailing comment with extra leading space. + for i in range(10): # Trailing comment with extra leading space. + while condition: + if something: + then_something( ) + elif something_else: + then_something_else( ) +except ValueError as e: + unformatted( ) +finally: + unformatted( ) + +async def test_async_unformatted( ): # Trailing comment with extra leading space. + async for i in some_iter( unformatted ): # Trailing comment with extra leading space. + await asyncio.sleep( 1 ) + async with some_context( unformatted ): + print( "unformatted" ) diff --git a/tests/data/cases/line_ranges_fmt_off.py b/tests/data/cases/line_ranges_fmt_off.py new file mode 100644 index 00000000000..b51cef58fe5 --- /dev/null +++ b/tests/data/cases/line_ranges_fmt_off.py @@ -0,0 +1,49 @@ +# flags: --line-ranges=7-7 --line-ranges=17-23 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# fmt: off +import os +def myfunc( ): # Intentionally unformatted. + pass +# fmt: on + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc( ): # This will be reformatted. + print( {"this will be reformatted"} ) + +# output + +# flags: --line-ranges=7-7 --line-ranges=17-23 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# fmt: off +import os +def myfunc( ): # Intentionally unformatted. + pass +# fmt: on + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc(): # This will be reformatted. + print({"this will be reformatted"}) diff --git a/tests/data/cases/line_ranges_fmt_off_decorator.py b/tests/data/cases/line_ranges_fmt_off_decorator.py new file mode 100644 index 00000000000..14aa1dda02d --- /dev/null +++ b/tests/data/cases/line_ranges_fmt_off_decorator.py @@ -0,0 +1,27 @@ +# flags: --line-ranges=12-12 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# Regression test for an edge case involving decorators and fmt: off/on. +class MyClass: + + # fmt: off + @decorator ( ) + # fmt: on + def method(): + print ( "str" ) + +# output + +# flags: --line-ranges=12-12 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# Regression test for an edge case involving decorators and fmt: off/on. +class MyClass: + + # fmt: off + @decorator ( ) + # fmt: on + def method(): + print("str") diff --git a/tests/data/cases/line_ranges_fmt_off_overlap.py b/tests/data/cases/line_ranges_fmt_off_overlap.py new file mode 100644 index 00000000000..0391d17a843 --- /dev/null +++ b/tests/data/cases/line_ranges_fmt_off_overlap.py @@ -0,0 +1,37 @@ +# flags: --line-ranges=11-17 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc( ): # This will be reformatted. + print( {"this will be reformatted"} ) + +# output + +# flags: --line-ranges=11-17 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc(): # This will be reformatted. + print({"this will be reformatted"}) diff --git a/tests/data/cases/line_ranges_imports.py b/tests/data/cases/line_ranges_imports.py new file mode 100644 index 00000000000..76b18ffecb3 --- /dev/null +++ b/tests/data/cases/line_ranges_imports.py @@ -0,0 +1,9 @@ +# flags: --line-ranges=8-8 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# This test ensures no empty lines are added around import lines. +# It caused an issue before https://github.com/psf/black/pull/3610 is merged. +import os +import re +import sys diff --git a/tests/data/cases/line_ranges_indentation.py b/tests/data/cases/line_ranges_indentation.py new file mode 100644 index 00000000000..82d3ad69a5e --- /dev/null +++ b/tests/data/cases/line_ranges_indentation.py @@ -0,0 +1,27 @@ +# flags: --line-ranges=5-5 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +if cond1: + print("first") + if cond2: + print("second") + else: + print("else") + +if another_cond: + print("will not be changed") + +# output + +# flags: --line-ranges=5-5 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +if cond1: + print("first") + if cond2: + print("second") + else: + print("else") + +if another_cond: + print("will not be changed") diff --git a/tests/data/cases/line_ranges_two_passes.py b/tests/data/cases/line_ranges_two_passes.py new file mode 100644 index 00000000000..aeed3260b8e --- /dev/null +++ b/tests/data/cases/line_ranges_two_passes.py @@ -0,0 +1,27 @@ +# flags: --line-ranges=9-11 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# This is a specific case for Black's two-pass formatting behavior in `format_str`. +# The second pass must respect the line ranges before the first pass. + + +def restrict_to_this_line(arg1, + arg2, + arg3): + print ( "This should not be formatted." ) + print ( "Note that in the second pass, the original line range 9-11 will cover these print lines.") + +# output + +# flags: --line-ranges=9-11 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# This is a specific case for Black's two-pass formatting behavior in `format_str`. +# The second pass must respect the line ranges before the first pass. + + +def restrict_to_this_line(arg1, arg2, arg3): + print ( "This should not be formatted." ) + print ( "Note that in the second pass, the original line range 9-11 will cover these print lines.") diff --git a/tests/data/cases/line_ranges_unwrapping.py b/tests/data/cases/line_ranges_unwrapping.py new file mode 100644 index 00000000000..cd7751b9417 --- /dev/null +++ b/tests/data/cases/line_ranges_unwrapping.py @@ -0,0 +1,25 @@ +# flags: --line-ranges=5-5 --line-ranges=9-9 --line-ranges=13-13 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +alist = [ + 1, 2 +] + +adict = { + "key" : "value" +} + +func_call ( + arg = value +) + +# output + +# flags: --line-ranges=5-5 --line-ranges=9-9 --line-ranges=13-13 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +alist = [1, 2] + +adict = {"key": "value"} + +func_call(arg=value) diff --git a/tests/data/invalid_line_ranges.toml b/tests/data/invalid_line_ranges.toml new file mode 100644 index 00000000000..791573f2625 --- /dev/null +++ b/tests/data/invalid_line_ranges.toml @@ -0,0 +1,2 @@ +[tool.black] +line-ranges = "1-1" diff --git a/tests/data/line_ranges_formatted/basic.py b/tests/data/line_ranges_formatted/basic.py new file mode 100644 index 00000000000..b419b1f16ae --- /dev/null +++ b/tests/data/line_ranges_formatted/basic.py @@ -0,0 +1,50 @@ +"""Module doc.""" + +from typing import ( + Callable, + Literal, +) + + +# fmt: off +class Unformatted: + def should_also_work(self): + pass +# fmt: on + + +a = [1, 2] # fmt: skip + + +# This should cover as many syntaxes as possible. +class Foo: + """Class doc.""" + + def __init__(self) -> None: + pass + + @add_logging + @memoize.memoize(max_items=2) + def plus_one( + self, + number: int, + ) -> int: + return number + 1 + + async def async_plus_one(self, number: int) -> int: + await asyncio.sleep(1) + async with some_context(): + return number + 1 + + +try: + for i in range(10): + while condition: + if something: + then_something() + elif something_else: + then_something_else() +except ValueError as e: + handle(e) +finally: + done() diff --git a/tests/data/line_ranges_formatted/pattern_matching.py b/tests/data/line_ranges_formatted/pattern_matching.py new file mode 100644 index 00000000000..cd98efdd504 --- /dev/null +++ b/tests/data/line_ranges_formatted/pattern_matching.py @@ -0,0 +1,25 @@ +# flags: --minimum-version=3.10 + + +def pattern_matching(): + match status: + case 1: + return "1" + case [single]: + return "single" + case [ + action, + obj, + ]: + return "act on obj" + case Point(x=0): + return "class pattern" + case {"text": message}: + return "mapping" + case { + "text": message, + "format": _, + }: + return "mapping" + case _: + return "fallback" diff --git a/tests/test_black.py b/tests/test_black.py index c7196098e14..c9819742425 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -8,6 +8,7 @@ import os import re import sys +import textwrap import types from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager, redirect_stderr @@ -1269,7 +1270,7 @@ def test_reformat_one_with_stdin_filename(self) -> None: report=report, ) fsts.assert_called_once_with( - fast=True, write_back=black.WriteBack.YES, mode=DEFAULT_MODE + fast=True, write_back=black.WriteBack.YES, mode=DEFAULT_MODE, lines=() ) # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) @@ -1295,6 +1296,7 @@ def test_reformat_one_with_stdin_filename_pyi(self) -> None: fast=True, write_back=black.WriteBack.YES, mode=replace(DEFAULT_MODE, is_pyi=True), + lines=(), ) # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) @@ -1320,6 +1322,7 @@ def test_reformat_one_with_stdin_filename_ipynb(self) -> None: fast=True, write_back=black.WriteBack.YES, mode=replace(DEFAULT_MODE, is_ipynb=True), + lines=(), ) # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) @@ -1941,6 +1944,88 @@ def test_equivalency_ast_parse_failure_includes_error(self) -> None: err.match("invalid character") err.match(r"\(, line 1\)") + def test_line_ranges_with_code_option(self) -> None: + code = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + args = ["--line-ranges=1-1", "--code", code] + result = CliRunner().invoke(black.main, args) + + expected = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + self.compare_results(result, expected, expected_exit_code=0) + + def test_line_ranges_with_stdin(self) -> None: + code = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + runner = BlackRunner() + result = runner.invoke( + black.main, ["--line-ranges=1-1", "-"], input=BytesIO(code.encode("utf-8")) + ) + + expected = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + self.compare_results(result, expected, expected_exit_code=0) + + def test_line_ranges_with_source(self) -> None: + with TemporaryDirectory() as workspace: + test_file = Path(workspace) / "test.py" + test_file.write_text( + textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """), + encoding="utf-8", + ) + args = ["--line-ranges=1-1", str(test_file)] + result = CliRunner().invoke(black.main, args) + assert not result.exit_code + + formatted = test_file.read_text(encoding="utf-8") + expected = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + assert expected == formatted + + def test_line_ranges_with_multiple_sources(self) -> None: + with TemporaryDirectory() as workspace: + test1_file = Path(workspace) / "test1.py" + test1_file.write_text("", encoding="utf-8") + test2_file = Path(workspace) / "test2.py" + test2_file.write_text("", encoding="utf-8") + args = ["--line-ranges=1-1", str(test1_file), str(test2_file)] + result = CliRunner().invoke(black.main, args) + assert result.exit_code == 1 + assert "Cannot use --line-ranges to format multiple files" in result.output + + def test_line_ranges_with_ipynb(self) -> None: + with TemporaryDirectory() as workspace: + test_file = Path(workspace) / "test.ipynb" + test_file.write_text("{}", encoding="utf-8") + args = ["--line-ranges=1-1", "--ipynb", str(test_file)] + result = CliRunner().invoke(black.main, args) + assert "Cannot use --line-ranges with ipynb files" in result.output + assert result.exit_code == 1 + + def test_line_ranges_in_pyproject_toml(self) -> None: + config = THIS_DIR / "data" / "invalid_line_ranges.toml" + result = BlackRunner().invoke( + black.main, ["--code", "print()", "--config", str(config)] + ) + assert result.exit_code == 2 + assert result.stderr_bytes is not None + assert ( + b"Cannot use line-ranges in the pyproject.toml file." in result.stderr_bytes + ) + class TestCaching: def test_get_cache_dir( diff --git a/tests/test_format.py b/tests/test_format.py index 4e863c6c54b..6c2eca8c618 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -29,13 +29,19 @@ def check_file(subdir: str, filename: str, *, data: bool = True) -> None: args.mode, fast=args.fast, minimum_version=args.minimum_version, + lines=args.lines, ) if args.minimum_version is not None: major, minor = args.minimum_version target_version = TargetVersion[f"PY{major}{minor}"] mode = replace(args.mode, target_versions={target_version}) assert_format( - source, expected, mode, fast=args.fast, minimum_version=args.minimum_version + source, + expected, + mode, + fast=args.fast, + minimum_version=args.minimum_version, + lines=args.lines, ) @@ -45,6 +51,24 @@ def test_simple_format(filename: str) -> None: check_file("cases", filename) +@pytest.mark.parametrize("filename", all_data_cases("line_ranges_formatted")) +def test_line_ranges_line_by_line(filename: str) -> None: + args, source, expected = read_data_with_mode("line_ranges_formatted", filename) + assert ( + source == expected + ), "Test cases in line_ranges_formatted must already be formatted." + line_count = len(source.splitlines()) + for line in range(1, line_count + 1): + assert_format( + source, + expected, + args.mode, + fast=args.fast, + minimum_version=args.minimum_version, + lines=[(line, line)], + ) + + # =============== # # Unusual cases # =============== # diff --git a/tests/test_ranges.py b/tests/test_ranges.py new file mode 100644 index 00000000000..d9fa9171a7f --- /dev/null +++ b/tests/test_ranges.py @@ -0,0 +1,185 @@ +"""Test the black.ranges module.""" + +from typing import List, Tuple + +import pytest + +from black.ranges import adjusted_lines + + +@pytest.mark.parametrize( + "lines", + [[(1, 1)], [(1, 3)], [(1, 1), (3, 4)]], +) +def test_no_diff(lines: List[Tuple[int, int]]) -> None: + source = """\ +import re + +def func(): +pass +""" + assert lines == adjusted_lines(lines, source, source) + + +@pytest.mark.parametrize( + "lines", + [ + [(1, 0)], + [(-8, 0)], + [(-8, 8)], + [(1, 100)], + [(2, 1)], + [(0, 8), (3, 1)], + ], +) +def test_invalid_lines(lines: List[Tuple[int, int]]) -> None: + original_source = """\ +import re +def foo(arg): +'''This is the foo function. + +This is foo function's +docstring with more descriptive texts. +''' + +def func(arg1, +arg2, arg3): +pass +""" + modified_source = """\ +import re +def foo(arg): +'''This is the foo function. + +This is foo function's +docstring with more descriptive texts. +''' + +def func(arg1, arg2, arg3): +pass +""" + assert not adjusted_lines(lines, original_source, modified_source) + + +@pytest.mark.parametrize( + "lines,adjusted", + [ + ( + [(1, 1)], + [(1, 1)], + ), + ( + [(1, 2)], + [(1, 1)], + ), + ( + [(1, 6)], + [(1, 2)], + ), + ( + [(6, 6)], + [], + ), + ], +) +def test_removals( + lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]] +) -> None: + original_source = """\ +1. first line +2. second line +3. third line +4. fourth line +5. fifth line +6. sixth line +""" + modified_source = """\ +2. second line +5. fifth line +""" + assert adjusted == adjusted_lines(lines, original_source, modified_source) + + +@pytest.mark.parametrize( + "lines,adjusted", + [ + ( + [(1, 1)], + [(2, 2)], + ), + ( + [(1, 2)], + [(2, 5)], + ), + ( + [(2, 2)], + [(5, 5)], + ), + ], +) +def test_additions( + lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]] +) -> None: + original_source = """\ +1. first line +2. second line +""" + modified_source = """\ +this is added +1. first line +this is added +this is added +2. second line +this is added +""" + assert adjusted == adjusted_lines(lines, original_source, modified_source) + + +@pytest.mark.parametrize( + "lines,adjusted", + [ + ( + [(1, 11)], + [(1, 10)], + ), + ( + [(1, 12)], + [(1, 11)], + ), + ( + [(10, 10)], + [(9, 9)], + ), + ([(1, 1), (9, 10)], [(1, 1), (9, 9)]), + ([(9, 10), (1, 1)], [(1, 1), (9, 9)]), + ], +) +def test_diffs(lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]]) -> None: + original_source = """\ + 1. import re + 2. def foo(arg): + 3. '''This is the foo function. + 4. + 5. This is foo function's + 6. docstring with more descriptive texts. + 7. ''' + 8. + 9. def func(arg1, +10. arg2, arg3): +11. pass +12. # last line +""" + modified_source = """\ + 1. import re # changed + 2. def foo(arg): + 3. '''This is the foo function. + 4. + 5. This is foo function's + 6. docstring with more descriptive texts. + 7. ''' + 8. + 9. def func(arg1, arg2, arg3): +11. pass +12. # last line changed +""" + assert adjusted == adjusted_lines(lines, original_source, modified_source) diff --git a/tests/util.py b/tests/util.py index a31ae0992c2..c8699d335ab 100644 --- a/tests/util.py +++ b/tests/util.py @@ -8,13 +8,14 @@ from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path -from typing import Any, Iterator, List, Optional, Tuple +from typing import Any, Collection, Iterator, List, Optional, Tuple import black from black.const import DEFAULT_LINE_LENGTH from black.debug import DebugVisitor from black.mode import TargetVersion from black.output import diff, err, out +from black.ranges import parse_line_ranges from . import conftest @@ -44,6 +45,7 @@ class TestCaseArgs: mode: black.Mode = field(default_factory=black.Mode) fast: bool = False minimum_version: Optional[Tuple[int, int]] = None + lines: Collection[Tuple[int, int]] = () def _assert_format_equal(expected: str, actual: str) -> None: @@ -93,6 +95,7 @@ def assert_format( *, fast: bool = False, minimum_version: Optional[Tuple[int, int]] = None, + lines: Collection[Tuple[int, int]] = (), ) -> None: """Convenience function to check that Black formats as expected. @@ -101,7 +104,7 @@ def assert_format( separate from TargetVerson Mode configuration. """ _assert_format_inner( - source, expected, mode, fast=fast, minimum_version=minimum_version + source, expected, mode, fast=fast, minimum_version=minimum_version, lines=lines ) # For both preview and non-preview tests, ensure that Black doesn't crash on @@ -113,6 +116,7 @@ def assert_format( replace(mode, preview=not mode.preview), fast=fast, minimum_version=minimum_version, + lines=lines, ) except Exception as e: text = "non-preview" if mode.preview else "preview" @@ -129,6 +133,7 @@ def assert_format( replace(mode, preview=False, line_length=1), fast=fast, minimum_version=minimum_version, + lines=lines, ) except Exception as e: raise FormatFailure( @@ -143,8 +148,9 @@ def _assert_format_inner( *, fast: bool = False, minimum_version: Optional[Tuple[int, int]] = None, + lines: Collection[Tuple[int, int]] = (), ) -> None: - actual = black.format_str(source, mode=mode) + actual = black.format_str(source, mode=mode, lines=lines) if expected is not None: _assert_format_equal(expected, actual) # It's not useful to run safety checks if we're expecting no changes anyway. The @@ -156,7 +162,7 @@ def _assert_format_inner( # when checking modern code on older versions. if minimum_version is None or sys.version_info >= minimum_version: black.assert_equivalent(source, actual) - black.assert_stable(source, actual, mode=mode) + black.assert_stable(source, actual, mode=mode, lines=lines) def dump_to_stderr(*output: str) -> str: @@ -239,6 +245,7 @@ def get_flags_parser() -> argparse.ArgumentParser: " version works correctly." ), ) + parser.add_argument("--line-ranges", action="append") return parser @@ -254,7 +261,13 @@ def parse_mode(flags_line: str) -> TestCaseArgs: magic_trailing_comma=not args.skip_magic_trailing_comma, preview=args.preview, ) - return TestCaseArgs(mode=mode, fast=args.fast, minimum_version=args.minimum_version) + if args.line_ranges: + lines = parse_line_ranges(args.line_ranges) + else: + lines = [] + return TestCaseArgs( + mode=mode, fast=args.fast, minimum_version=args.minimum_version, lines=lines + ) def read_data_from_file(file_name: Path) -> Tuple[TestCaseArgs, str, str]: @@ -267,6 +280,12 @@ def read_data_from_file(file_name: Path) -> Tuple[TestCaseArgs, str, str]: for line in lines: if not _input and line.startswith("# flags: "): mode = parse_mode(line[len("# flags: ") :]) + if mode.lines: + # Retain the `# flags: ` line when using --line-ranges=. This requires + # the `# output` section to also include this line, but retaining the + # line is important to make the line ranges match what you see in the + # test file. + result.append(line) continue line = line.replace(EMPTY_LINE, "") if line.rstrip() == "# output": From 50ed6221d97b265025abaa66116a7b185f2df5e2 Mon Sep 17 00:00:00 2001 From: rdrll <13176405+rdrll@users.noreply.github.com> Date: Tue, 7 Nov 2023 06:31:58 -0800 Subject: [PATCH 26/33] Fix long case blocks not split into multiple lines (#4024) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 + src/black/linegen.py | 24 +++++++++++- src/black/mode.py | 1 + tests/data/cases/pattern_matching_extras.py | 22 +---------- .../cases/preview_pattern_matching_long.py | 34 ++++++++++++++++ ...preview_pattern_matching_trailing_comma.py | 39 +++++++++++++++++++ 6 files changed, 101 insertions(+), 21 deletions(-) create mode 100644 tests/data/cases/preview_pattern_matching_long.py create mode 100644 tests/data/cases/preview_pattern_matching_trailing_comma.py diff --git a/CHANGES.md b/CHANGES.md index 780a00247ce..c8ba83b5ae9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,6 +25,8 @@ indented less (#3964) - Multiline list and dict unpacking as the sole argument to a function is now also indented less (#3992) +- Fix a bug where long `case` blocks were not split into multiple lines. Also enable + general trailing comma rules on `case` blocks (#4024) - Keep requiring two empty lines between module-level docstring and first function or class definition. (#4028) diff --git a/src/black/linegen.py b/src/black/linegen.py index b13b95d9b31..30cfff3e846 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1229,7 +1229,7 @@ def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: leaf.prefix = "" -def normalize_invisible_parens( +def normalize_invisible_parens( # noqa: C901 node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature] ) -> None: """Make existing optional parentheses invisible or create new ones. @@ -1260,6 +1260,17 @@ def normalize_invisible_parens( child, parens_after=parens_after, mode=mode, features=features ) + # Fixes a bug where invisible parens are not properly wrapped around + # case blocks. + if ( + isinstance(child, Node) + and child.type == syms.case_block + and Preview.long_case_block_line_splitting in mode + ): + normalize_invisible_parens( + child, parens_after={"case"}, mode=mode, features=features + ) + # Add parentheses around long tuple unpacking in assignments. if ( index == 0 @@ -1305,6 +1316,17 @@ def normalize_invisible_parens( # invisible parentheses to work more precisely. continue + elif ( + isinstance(child, Leaf) + and child.next_sibling is not None + and child.next_sibling.type == token.COLON + and child.value == "case" + and Preview.long_case_block_line_splitting in mode + ): + # A special patch for "case case:" scenario, the second occurrence + # of case will be not parsed as a Python keyword. + break + elif not (isinstance(child, Leaf) and is_multiline_string(child)): wrap_in_parentheses(node, child, visible=False) diff --git a/src/black/mode.py b/src/black/mode.py index 4e4effffb86..1aa5cbecc86 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -193,6 +193,7 @@ class Preview(Enum): hug_parens_with_braces_and_square_brackets = auto() allow_empty_first_line_before_new_block_or_comment = auto() single_line_format_skip_with_multiple_comments = auto() + long_case_block_line_splitting = auto() class Deprecated(UserWarning): diff --git a/tests/data/cases/pattern_matching_extras.py b/tests/data/cases/pattern_matching_extras.py index 1e1481d7bbe..1aef8f16b5a 100644 --- a/tests/data/cases/pattern_matching_extras.py +++ b/tests/data/cases/pattern_matching_extras.py @@ -30,22 +30,6 @@ def func(match: case, case: match) -> case: ... -match maybe, multiple: - case perhaps, 5: - pass - case perhaps, 6,: - pass - - -match more := (than, one), indeed,: - case _, (5, 6): - pass - case [[5], (6)], [7],: - pass - case _: - pass - - match a, *b, c: case [*_]: assert "seq" == _ @@ -67,12 +51,12 @@ def func(match: case, case: match) -> case: ), ): pass - case [a as match]: pass - case case: pass + case something: + pass match match: @@ -98,10 +82,8 @@ def func(match: case, case: match) -> case: match something: case 1 as a: pass - case 2 as b, 3 as c: pass - case 4 as d, (5 as e), (6 | 7 as g), *h: pass diff --git a/tests/data/cases/preview_pattern_matching_long.py b/tests/data/cases/preview_pattern_matching_long.py new file mode 100644 index 00000000000..df849fdc4f2 --- /dev/null +++ b/tests/data/cases/preview_pattern_matching_long.py @@ -0,0 +1,34 @@ +# flags: --preview --minimum-version=3.10 +match x: + case "abcd" | "abcd" | "abcd" : + pass + case "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd": + pass + case xxxxxxxxxxxxxxxxxxxxxxx: + pass + +# output + +match x: + case "abcd" | "abcd" | "abcd": + pass + case ( + "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + ): + pass + case xxxxxxxxxxxxxxxxxxxxxxx: + pass diff --git a/tests/data/cases/preview_pattern_matching_trailing_comma.py b/tests/data/cases/preview_pattern_matching_trailing_comma.py new file mode 100644 index 00000000000..e6c0d88bb80 --- /dev/null +++ b/tests/data/cases/preview_pattern_matching_trailing_comma.py @@ -0,0 +1,39 @@ +# flags: --preview --minimum-version=3.10 +match maybe, multiple: + case perhaps, 5: + pass + case perhaps, 6,: + pass + + +match more := (than, one), indeed,: + case _, (5, 6): + pass + case [[5], (6)], [7],: + pass + case _: + pass + + +# output + +match maybe, multiple: + case perhaps, 5: + pass + case ( + perhaps, + 6, + ): + pass + + +match more := (than, one), indeed,: + case _, (5, 6): + pass + case ( + [[5], (6)], + [7], + ): + pass + case _: + pass \ No newline at end of file From 66008fda5dc07f5626e5f5d0dcefc476a9c12ab8 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Tue, 7 Nov 2023 21:29:24 +0200 Subject: [PATCH 27/33] [563] Fix standalone comments inside complex blocks crashing Black (#4016) Bracket depth is not an accurate indicator of standalone comment position inside more complex blocks because bracket depth can be virtual (in loops' and lambdas' parameter blocks) or from optional parens. Here we try to stop cumulating lines upon standalone comments in complex blocks, and try to make standalone comment processing more simple. The fundamental idea is, that if we have a standalone comment, it needs to go on its own line, so we always have to split. This is not perfect, but at least a first step. --- CHANGES.md | 1 + src/black/brackets.py | 7 ++ src/black/linegen.py | 4 +- src/black/lines.py | 27 ++++- tests/data/cases/comments_in_blocks.py | 111 ++++++++++++++++++ ..._parens_with_braces_and_square_brackets.py | 20 ++++ 6 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 tests/data/cases/comments_in_blocks.py diff --git a/CHANGES.md b/CHANGES.md index c8ba83b5ae9..d30622b7786 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,7 @@ - Fix crash on formatting bytes strings that look like docstrings (#4003) - Fix crash when whitespace followed a backslash before newline in a docstring (#4008) +- Fix standalone comments inside complex blocks crashing Black (#4016) - Fix crash on formatting code like `await (a ** b)` (#3994) diff --git a/src/black/brackets.py b/src/black/brackets.py index 85dac6edd1e..3020cc0d390 100644 --- a/src/black/brackets.py +++ b/src/black/brackets.py @@ -127,6 +127,13 @@ def mark(self, leaf: Leaf) -> None: self.maybe_increment_lambda_arguments(leaf) self.maybe_increment_for_loop_variable(leaf) + def any_open_for_or_lambda(self) -> bool: + """Return True if there is an open for or lambda expression on the line. + + See maybe_increment_for_loop_variable and maybe_increment_lambda_arguments + for details.""" + return bool(self._for_loop_depths or self._lambda_argument_depths) + def any_open_brackets(self) -> bool: """Return True if there is an yet unmatched open bracket on the line.""" return bool(self.bracket_match) diff --git a/src/black/linegen.py b/src/black/linegen.py index 30cfff3e846..e2c961d7a01 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -861,8 +861,6 @@ def _maybe_split_omitting_optional_parens( # it's not an import (optional parens are the only thing we can split on # in this case; attempting a split without them is a waste of time) and not line.is_import - # there are no standalone comments in the body - and not rhs.body.contains_standalone_comments(0) # and we can actually remove the parens and can_omit_invisible_parens(rhs, mode.line_length) ): @@ -1181,7 +1179,7 @@ def standalone_comment_split( line: Line, features: Collection[Feature], mode: Mode ) -> Iterator[Line]: """Split standalone comments from the rest of the line.""" - if not line.contains_standalone_comments(0): + if not line.contains_standalone_comments(): raise CannotSplit("Line does not have any standalone comments") current_line = Line( diff --git a/src/black/lines.py b/src/black/lines.py index 23c1a93d3d4..f0cf25ba3e7 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -1,6 +1,5 @@ import itertools import math -import sys from dataclasses import dataclass, field from typing import ( Callable, @@ -103,7 +102,10 @@ def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None: Raises ValueError when any `leaf` is appended after a standalone comment or when a standalone comment is not the first leaf on the line. """ - if self.bracket_tracker.depth == 0: + if ( + self.bracket_tracker.depth == 0 + or self.bracket_tracker.any_open_for_or_lambda() + ): if self.is_comment: raise ValueError("cannot append to standalone comments") @@ -233,10 +235,10 @@ def is_fmt_pass_converted( leaf.fmt_pass_converted_first_leaf ) - def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool: + def contains_standalone_comments(self) -> bool: """If so, needs to be split before emitting.""" for leaf in self.leaves: - if leaf.type == STANDALONE_COMMENT and leaf.bracket_depth <= depth_limit: + if leaf.type == STANDALONE_COMMENT: return True return False @@ -982,6 +984,23 @@ def can_omit_invisible_parens( are too long. """ line = rhs.body + + # We need optional parens in order to split standalone comments to their own lines + # if there are no nested parens around the standalone comments + closing_bracket: Optional[Leaf] = None + for leaf in reversed(line.leaves): + if closing_bracket and leaf is closing_bracket.opening_bracket: + closing_bracket = None + if leaf.type == STANDALONE_COMMENT and not closing_bracket: + return False + if ( + not closing_bracket + and leaf.type in CLOSING_BRACKETS + and leaf.opening_bracket in line.leaves + and leaf.value + ): + closing_bracket = leaf + bt = line.bracket_tracker if not bt.delimiters: # Without delimiters the optional parentheses are useless. diff --git a/tests/data/cases/comments_in_blocks.py b/tests/data/cases/comments_in_blocks.py new file mode 100644 index 00000000000..1221139b6d8 --- /dev/null +++ b/tests/data/cases/comments_in_blocks.py @@ -0,0 +1,111 @@ +# Test cases from: +# - https://github.com/psf/black/issues/1798 +# - https://github.com/psf/black/issues/1499 +# - https://github.com/psf/black/issues/1211 +# - https://github.com/psf/black/issues/563 + +( + lambda + # a comment + : None +) + +( + lambda: + # b comment + None +) + +( + lambda + # a comment + : + # b comment + None +) + +[ + x + # Let's do this + for + # OK? + x + # Some comment + # And another + in + # One more + y +] + +return [ + (offers[offer_index], 1.0) + for offer_index, _ + # avoid returning any offers that don't match the grammar so + # that the return values here are consistent with what would be + # returned in AcceptValidHeader + in self._parse_and_normalize_offers(offers) +] + +from foo import ( + bar, + # qux +) + + +def convert(collection): + # replace all variables by integers + replacement_dict = { + variable: f"{index}" + for index, variable + # 0 is reserved as line terminator + in enumerate(collection.variables(), start=1) + } + + +{ + i: i + for i + # a comment + in range(5) +} + + +def get_subtree_proof_nodes( + chunk_index_groups: Sequence[Tuple[int, ...], ...], +) -> Tuple[int, ...]: + subtree_node_paths = ( + # We take a candidate element from each group and shift it to + # remove the bits that are not common to other group members, then + # we convert it to a tree path that all elements from this group + # have in common. + chunk_index + for chunk_index, bits_to_truncate + # Each group will contain an even "power-of-two" number of# elements. + # This tells us how many tailing bits each element has# which need to + # be truncated to get the group's common prefix. + in ((group[0], (len(group) - 1).bit_length()) for group in chunk_index_groups) + ) + return subtree_node_paths + + +if ( + # comment1 + a + # comment2 + or ( + # comment3 + ( + # comment4 + b + ) + # comment5 + and + # comment6 + c + or ( + # comment7 + d + ) + ) +): + print("Foo") diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py index 51fe516add5..97b5b2e8dd1 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -152,6 +152,16 @@ def foo_square_brackets(request): foo(**{x: y for x, y in enumerate(["long long long long line","long long long long line"])}) +for foo in ["a", "b"]: + output.extend([ + individual + for + # Foobar + container in xs_by_y[foo] + # Foobar + for individual in container["nested"] + ]) + # output def foo_brackets(request): return JsonResponse({ @@ -323,3 +333,13 @@ def foo_square_brackets(request): foo(**{ x: y for x, y in enumerate(["long long long long line", "long long long long line"]) }) + +for foo in ["a", "b"]: + output.extend([ + individual + for + # Foobar + container in xs_by_y[foo] + # Foobar + for individual in container["nested"] + ]) From 2e4fac9d87615e904a49e46a9cab2293e0b13126 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:31:44 -0800 Subject: [PATCH 28/33] Apply force exclude logic before symlink resolution (#4015) --- CHANGES.md | 2 +- src/black/__init__.py | 24 ++++++++++++++---------- tests/test_black.py | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d30622b7786..17882194aad 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,7 +34,7 @@ ### Configuration - Add support for single-line format skip with other comments on the same line (#3959) - +- Consistently apply force exclusion logic before resolving symlinks (#4015) - Fix a bug in the matching of absolute path names in `--include` (#3976) ### Packaging diff --git a/src/black/__init__.py b/src/black/__init__.py index 5aca3316df0..2455e8648fc 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -682,7 +682,19 @@ def get_sources( path = Path(s) is_stdin = False + # Compare the logic here to the logic in `gen_python_files`. if is_stdin or path.is_file(): + root_relative_path = path.absolute().relative_to(root).as_posix() + + root_relative_path = "/" + root_relative_path + + # Hard-exclude any files that matches the `--force-exclude` regex. + if path_is_excluded(root_relative_path, force_exclude): + report.path_ignored( + path, "matches the --force-exclude regular expression" + ) + continue + normalized_path: Optional[str] = normalize_path_maybe_ignore( path, root, report ) @@ -690,16 +702,6 @@ def get_sources( if verbose: out(f'Skipping invalid source: "{normalized_path}"', fg="red") continue - if verbose: - out(f'Found input source: "{normalized_path}"', fg="blue") - - normalized_path = "/" + normalized_path - # Hard-exclude any files that matches the `--force-exclude` regex. - if path_is_excluded(normalized_path, force_exclude): - report.path_ignored( - path, "matches the --force-exclude regular expression" - ) - continue if is_stdin: path = Path(f"{STDIN_PLACEHOLDER}{str(path)}") @@ -709,6 +711,8 @@ def get_sources( ): continue + if verbose: + out(f'Found input source: "{normalized_path}"', fg="blue") sources.add(path) elif path.is_dir(): path = root / (path.resolve().relative_to(root)) diff --git a/tests/test_black.py b/tests/test_black.py index c9819742425..899cbeb111d 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2637,6 +2637,23 @@ def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: stdin_filename=stdin_filename, ) + @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) + def test_get_sources_with_stdin_filename_and_force_exclude_and_symlink( + self, + ) -> None: + # Force exclude should exclude a symlink based on the symlink, not its target + path = THIS_DIR / "data" / "include_exclude_tests" + stdin_filename = str(path / "symlink.py") + expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] + target = path / "b/exclude/a.py" + with patch("pathlib.Path.resolve", return_value=target): + assert_collected_sources( + src=["-"], + expected=expected, + force_exclude=r"exclude/a\.py", + stdin_filename=stdin_filename, + ) + class TestDeFactoAPI: """Test that certain symbols that are commonly used externally keep working. From f4c7be5445c87d9af5eba3d12faea62d2635e3d8 Mon Sep 17 00:00:00 2001 From: Abdenour Madani <61651582+Ab2nour@users.noreply.github.com> Date: Wed, 8 Nov 2023 00:40:10 +0100 Subject: [PATCH 29/33] docs: fix minor typo (#4030) Replace "E950" with "B950" From 1a7d9c2f58de1ffcbbe6d133f60f283601ba3f54 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Wed, 8 Nov 2023 06:19:32 +0200 Subject: [PATCH 30/33] Preserve visible quote types for f-string debug expressions (#4005) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 + src/black/trans.py | 21 +++++-- tests/data/cases/preview_long_strings.py | 80 ++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 17882194aad..26e4db5848b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,8 @@ indented less (#3964) - Multiline list and dict unpacking as the sole argument to a function is now also indented less (#3992) +- In f-string debug expressions preserve quote types that are visible in the final + string (#4005) - Fix a bug where long `case` blocks were not split into multiple lines. Also enable general trailing comma rules on `case` blocks (#4024) - Keep requiring two empty lines between module-level docstring and first function or diff --git a/src/black/trans.py b/src/black/trans.py index a3f6467cc9e..ab3197fa6df 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -590,11 +590,22 @@ def make_naked(string: str, string_prefix: str) -> str: """ assert_is_leaf_string(string) if "f" in string_prefix: - string = _toggle_fexpr_quotes(string, QUOTE) - # After quotes toggling, quotes in expressions won't be escaped - # because quotes can't be reused in f-strings. So we can simply - # let the escaping logic below run without knowing f-string - # expressions. + f_expressions = ( + string[span[0] + 1 : span[1] - 1] # +-1 to get rid of curly braces + for span in iter_fexpr_spans(string) + ) + debug_expressions_contain_visible_quotes = any( + re.search(r".*[\'\"].*(? Date: Wed, 8 Nov 2023 06:21:33 +0200 Subject: [PATCH 31/33] Remove redundant condition from `has_magic_trailing_comma` (#4023) The second `if` cannot be true at its execution point, because it is already covered by the first `if`. The condition `comma.parent.type == syms.subscriptlist` always holds if `closing.parent.type == syms.trailer` holds, because `subscriptlist` only appears inside `trailer` in the grammar: ``` trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: (subscript|star_expr) (',' (subscript|star_expr))* [','] ``` --- src/black/lines.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/black/lines.py b/src/black/lines.py index f0cf25ba3e7..3ade0a5f4a5 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -353,9 +353,9 @@ def has_magic_trailing_comma( if closing.type == token.RSQB: if ( - closing.parent + closing.parent is not None and closing.parent.type == syms.trailer - and closing.opening_bracket + and closing.opening_bracket is not None and is_one_sequence_between( closing.opening_bracket, closing, @@ -365,22 +365,7 @@ def has_magic_trailing_comma( ): return False - if not ensure_removable: - return True - - comma = self.leaves[-1] - if comma.parent is None: - return False - return ( - comma.parent.type != syms.subscriptlist - or closing.opening_bracket is None - or not is_one_sequence_between( - closing.opening_bracket, - closing, - self.leaves, - brackets=(token.LSQB, token.RSQB), - ) - ) + return True if self.is_import: return True From 2a1c67e0b2f81df602ec1f6e7aeb030b9709dc7c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 Nov 2023 20:44:46 -0800 Subject: [PATCH 32/33] Prepare release 23.11.0 (#4032) --- CHANGES.md | 70 +++------------------ docs/integrations/source_version_control.md | 4 +- docs/usage_and_configuration/the_basics.md | 6 +- 3 files changed, 14 insertions(+), 66 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 26e4db5848b..b565d510a71 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,78 +1,49 @@ # Change Log -## Unreleased +## 23.11.0 ### Highlights - - - Support formatting ranges of lines with the new `--line-ranges` command-line option - (#4020). + (#4020) ### Stable style - Fix crash on formatting bytes strings that look like docstrings (#4003) - Fix crash when whitespace followed a backslash before newline in a docstring (#4008) - Fix standalone comments inside complex blocks crashing Black (#4016) - - Fix crash on formatting code like `await (a ** b)` (#3994) - - No longer treat leading f-strings as docstrings. This matches Python's behaviour and fixes a crash (#4019) ### Preview style -- Multiline dictionaries and lists that are the sole argument to a function are now - indented less (#3964) -- Multiline list and dict unpacking as the sole argument to a function is now also +- Multiline dicts and lists that are the sole argument to a function are now indented + less (#3964) +- Multiline unpacked dicts and lists as the sole argument to a function are now also indented less (#3992) -- In f-string debug expressions preserve quote types that are visible in the final - string (#4005) +- In f-string debug expressions, quote types that are visible in the final string are + now preserved (#4005) - Fix a bug where long `case` blocks were not split into multiple lines. Also enable general trailing comma rules on `case` blocks (#4024) - Keep requiring two empty lines between module-level docstring and first function or - class definition. (#4028) + class definition (#4028) +- Add support for single-line format skip with other comments on the same line (#3959) ### Configuration -- Add support for single-line format skip with other comments on the same line (#3959) - Consistently apply force exclusion logic before resolving symlinks (#4015) - Fix a bug in the matching of absolute path names in `--include` (#3976) -### Packaging - - - -### Parser - - - ### Performance - - - Fix mypyc builds on arm64 on macOS (#4017) -### Output - - - -### _Blackd_ - - - ### Integrations - - - Black's pre-commit integration will now run only on git hooks appropriate for a code formatter (#3940) -### Documentation - - - ## 23.10.1 ### Highlights @@ -327,8 +298,6 @@ versions separately. ### Stable style - - - Introduce the 2023 stable style, which incorporates most aspects of last year's preview style (#3418). Specific changes: - Enforce empty lines before classes and functions with sticky leading comments @@ -362,8 +331,6 @@ versions separately. ### Preview style - - - Format hex codes in unicode escape sequences in string literals (#2916) - Add parentheses around `if`-`else` expressions (#2278) - Improve performance on large expressions that contain many strings (#3467) @@ -394,15 +361,11 @@ versions separately. ### Configuration - - - Black now tries to infer its `--target-version` from the project metadata specified in `pyproject.toml` (#3219) ### Packaging - - - Upgrade mypyc from `0.971` to `0.991` so mypycified _Black_ can be built on armv7 (#3380) - This also fixes some crashes while using compiled Black with a debug build of @@ -415,8 +378,6 @@ versions separately. ### Output - - - Calling `black --help` multiple times will return the same help contents each time (#3516) - Verbose logging now shows the values of `pyproject.toml` configuration variables @@ -426,25 +387,18 @@ versions separately. ### Integrations - - - Move 3.11 CI to normal flow now that all dependencies support 3.11 (#3446) - Docker: Add new `latest_prerelease` tag automation to follow latest black alpha release on docker images (#3465) ### Documentation - - - Expand `vim-plug` installation instructions to offer more explicit options (#3468) ## 22.12.0 ### Preview style - - - Enforce empty lines before classes and functions with sticky leading comments (#3302) - Reformat empty and whitespace-only files as either an empty file (if no newline is present) or as a single newline character (if a newline is present) (#3348) @@ -457,8 +411,6 @@ versions separately. ### Configuration - - - Fix incorrectly applied `.gitignore` rules by considering the `.gitignore` location and the relative path to the target file (#3338) - Fix incorrectly ignoring `.gitignore` presence when more than one source directory is @@ -466,16 +418,12 @@ versions separately. ### Parser - - - Parsing support has been added for walruses inside generator expression that are passed as function args (for example, `any(match := my_re.match(text) for text in texts)`) (#3327). ### Integrations - - - Vim plugin: Optionally allow using the system installation of Black via `let g:black_use_virtualenv = 0`(#3309) diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 597a6b993c7..3c7ef89918f 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.10.1 + rev: 23.11.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.10.1 + rev: 23.11.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index dbd8c7ba434..6e7ee584cf9 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -211,8 +211,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.10.1 (compiled: yes) -$ black --required-version 23.10.1 -c "format = 'this'" +black, 23.11.0 (compiled: yes) +$ black --required-version 23.11.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -303,7 +303,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.10.1 +black, 23.11.0 ``` #### `--config` From 58f31a70efe6509ce8213afac998bc5d5bb7e34d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 Nov 2023 22:10:35 -0800 Subject: [PATCH 33/33] Add new release template --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index b565d510a71..9446927b8d1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 23.11.0 ### Highlights