From eb2e4ab90d8661a69a3b6830fc272e37a0304869 Mon Sep 17 00:00:00 2001 From: Matt Angus Date: Fri, 9 Aug 2024 23:37:41 +0100 Subject: [PATCH 1/5] don't generate stubs from invalid signatures --- mypy/stubdoc.py | 25 ++++++++++++- mypy/test/teststubgen.py | 79 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py index 928d024514f3..6897706d1415 100644 --- a/mypy/stubdoc.py +++ b/mypy/stubdoc.py @@ -180,6 +180,16 @@ def __init__(self, function_name: str) -> None: def add_token(self, token: tokenize.TokenInfo) -> None: """Process next token from the token stream.""" + + if ( + self.state[-1] == STATE_ARGUMENT_TYPE + and token.type != tokenize.NAME + and len(self.accumulator) == 0 + ): + # the next token after : must be a name + self.reset() + return + if ( token.type == tokenize.NAME and token.string == self.function_name @@ -282,8 +292,13 @@ def add_token(self, token: tokenize.TokenInfo) -> None: self.accumulator = "" elif token.type == tokenize.OP and token.string == "->" and self.state[-1] == STATE_INIT: + if len(self.accumulator) == 0: + self.state.append(STATE_RETURN_VALUE) + else: + # ) is not directly followed by -> + self.reset() + return self.accumulator = "" - self.state.append(STATE_RETURN_VALUE) # ENDMAKER is necessary for python 3.4 and 3.5. elif token.type in (tokenize.NEWLINE, tokenize.ENDMARKER) and self.state[-1] in ( @@ -306,6 +321,14 @@ def add_token(self, token: tokenize.TokenInfo) -> None: self.args = [] self.ret_type = "Any" # Leave state as INIT. + elif ( + token.type == tokenize.NAME + and self.state[-1] == STATE_ARGUMENT_LIST + and len(self.accumulator) != 0 + ): + # not the first name in a row + self.reset() + return else: self.accumulator += token.string diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index e65a16c8f395..f481d66b7781 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -398,6 +398,85 @@ def test_infer_sig_from_docstring_bad_indentation(self) -> None: None, ) + def test_infer_sig_from_docstring_invalid_signature(self) -> None: + + assert_equal(infer_sig_from_docstring("\nfunc() --> None", "func"), []) + + assert_equal( + infer_sig_from_docstring("\nfunc(name1 name2) -> None", "func"), [] + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(name1, name2 name3) -> None", "func"), [] + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(name2, name3) -> None, None", "func"), + [], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(invalid::name) -> None", "func"), + [], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(invalid: [type]) -> None", "func"), + [], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(invalid: (type)) -> None", "func"), + [], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(invalid: -type) -> None", "func"), + [], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(invalid None", "func"), + [], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(cpp::type name) -> None", "func"), + [], + ) + assert_equal( + infer_sig_from_docstring("\nfunc(name) -> cpp::type", "func"), + [], + ) + assert_equal( + infer_sig_from_docstring("\nvoid func(int name) {", "func"), + [], + ) + assert_equal( + infer_sig_from_docstring("\nvoid func(std::vector name)", "func"), + [], + ) + + def test_infer_sig_from_docstring_deeply_nested_types(self) -> None: + assert_equal( + infer_sig_from_docstring( + "\nfunc(name: dict[str, dict[str, list[tuple[int, float]]]]) -> None", + "func", + ), + [ + FunctionSig( + name="func", + args=[ + ArgSig( + name="name", + type="dict[str,dict[str,list[tuple[int,float]]]]", + ) + ], + ret_type="None", + ) + ], + ) + def test_infer_arg_sig_from_anon_docstring(self) -> None: assert_equal( infer_arg_sig_from_anon_docstring("(*args, **kwargs)"), From 9de1f2ee3bd67358e8f1d2b03ec3b12935308078 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Aug 2024 23:00:19 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/test/teststubgen.py | 68 +++++++++------------------------------- 1 file changed, 14 insertions(+), 54 deletions(-) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index f481d66b7781..9424b9ffd9cf 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -402,76 +402,36 @@ def test_infer_sig_from_docstring_invalid_signature(self) -> None: assert_equal(infer_sig_from_docstring("\nfunc() --> None", "func"), []) - assert_equal( - infer_sig_from_docstring("\nfunc(name1 name2) -> None", "func"), [] - ) + assert_equal(infer_sig_from_docstring("\nfunc(name1 name2) -> None", "func"), []) - assert_equal( - infer_sig_from_docstring("\nfunc(name1, name2 name3) -> None", "func"), [] - ) + assert_equal(infer_sig_from_docstring("\nfunc(name1, name2 name3) -> None", "func"), []) - assert_equal( - infer_sig_from_docstring("\nfunc(name2, name3) -> None, None", "func"), - [], - ) + assert_equal(infer_sig_from_docstring("\nfunc(name2, name3) -> None, None", "func"), []) - assert_equal( - infer_sig_from_docstring("\nfunc(invalid::name) -> None", "func"), - [], - ) + assert_equal(infer_sig_from_docstring("\nfunc(invalid::name) -> None", "func"), []) - assert_equal( - infer_sig_from_docstring("\nfunc(invalid: [type]) -> None", "func"), - [], - ) + assert_equal(infer_sig_from_docstring("\nfunc(invalid: [type]) -> None", "func"), []) - assert_equal( - infer_sig_from_docstring("\nfunc(invalid: (type)) -> None", "func"), - [], - ) + assert_equal(infer_sig_from_docstring("\nfunc(invalid: (type)) -> None", "func"), []) - assert_equal( - infer_sig_from_docstring("\nfunc(invalid: -type) -> None", "func"), - [], - ) + assert_equal(infer_sig_from_docstring("\nfunc(invalid: -type) -> None", "func"), []) - assert_equal( - infer_sig_from_docstring("\nfunc(invalid None", "func"), - [], - ) + assert_equal(infer_sig_from_docstring("\nfunc(invalid None", "func"), []) - assert_equal( - infer_sig_from_docstring("\nfunc(cpp::type name) -> None", "func"), - [], - ) - assert_equal( - infer_sig_from_docstring("\nfunc(name) -> cpp::type", "func"), - [], - ) - assert_equal( - infer_sig_from_docstring("\nvoid func(int name) {", "func"), - [], - ) - assert_equal( - infer_sig_from_docstring("\nvoid func(std::vector name)", "func"), - [], - ) + assert_equal(infer_sig_from_docstring("\nfunc(cpp::type name) -> None", "func"), []) + assert_equal(infer_sig_from_docstring("\nfunc(name) -> cpp::type", "func"), []) + assert_equal(infer_sig_from_docstring("\nvoid func(int name) {", "func"), []) + assert_equal(infer_sig_from_docstring("\nvoid func(std::vector name)", "func"), []) def test_infer_sig_from_docstring_deeply_nested_types(self) -> None: assert_equal( infer_sig_from_docstring( - "\nfunc(name: dict[str, dict[str, list[tuple[int, float]]]]) -> None", - "func", + "\nfunc(name: dict[str, dict[str, list[tuple[int, float]]]]) -> None", "func" ), [ FunctionSig( name="func", - args=[ - ArgSig( - name="name", - type="dict[str,dict[str,list[tuple[int,float]]]]", - ) - ], + args=[ArgSig(name="name", type="dict[str,dict[str,list[tuple[int,float]]]]")], ret_type="None", ) ], From e23a08c4d6fb71d4de17373a976feb77cefea6d9 Mon Sep 17 00:00:00 2001 From: Matt Angus Date: Sat, 10 Aug 2024 00:28:26 +0100 Subject: [PATCH 3/5] Fixed regression tests --- mypy/stubdoc.py | 5 ++++- mypy/test/teststubgen.py | 42 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py index 6897706d1415..bfcbbdf49233 100644 --- a/mypy/stubdoc.py +++ b/mypy/stubdoc.py @@ -325,8 +325,11 @@ def add_token(self, token: tokenize.TokenInfo) -> None: token.type == tokenize.NAME and self.state[-1] == STATE_ARGUMENT_LIST and len(self.accumulator) != 0 + and self.accumulator != "*" + and self.accumulator != "**" ): - # not the first name in a row + # not the token after the argument list started + # special case for *args and **kwargs self.reset() return else: diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 9424b9ffd9cf..29ccccce10bd 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -379,6 +379,21 @@ def test_infer_sig_from_docstring(self) -> None: ], ) + assert_equal( + infer_sig_from_docstring("\nfunc(*args)", "func"), + [FunctionSig(name="func", args=[ArgSig(name="*args")], ret_type="Any")], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(**kwargs)", "func"), + [FunctionSig(name="func", args=[ArgSig(name="**kwargs")], ret_type="Any")], + ) + + assert_equal( + infer_sig_from_docstring("\nfunc(*args, **kwargs)", "func"), + [FunctionSig(name="func", args=[ArgSig(name="*args"), ArgSig(name="**kwargs")], ret_type="Any")], + ) + def test_infer_sig_from_docstring_duplicate_args(self) -> None: assert_equal( infer_sig_from_docstring("\nfunc(x, x) -> str\nfunc(x, y) -> int", "func"), @@ -423,6 +438,21 @@ def test_infer_sig_from_docstring_invalid_signature(self) -> None: assert_equal(infer_sig_from_docstring("\nvoid func(int name) {", "func"), []) assert_equal(infer_sig_from_docstring("\nvoid func(std::vector name)", "func"), []) + def test_infer_sig_from_docstring_multiple_overloads(self) -> None: + input = """ +func(*args, **kwargs) +Overloaded function. + +1. func(self: class_type) -> None + +2. func(self: class_type, a: float, b: int) -> None +""" + assert_equal(infer_sig_from_docstring(input, "func"), [ + FunctionSig(name="func", args=[ArgSig(name="self", type="class_type")], ret_type="None"), + FunctionSig(name="func", args=[ArgSig(name="self", type="class_type"), ArgSig(name="a", type="float"), ArgSig(name="b", type="int")], ret_type="None"), + FunctionSig(name="func", args=[ArgSig(name="*args"), ArgSig(name="**kwargs")], ret_type="Any") + ]) + def test_infer_sig_from_docstring_deeply_nested_types(self) -> None: assert_equal( infer_sig_from_docstring( @@ -1489,3 +1519,15 @@ def module_to_path(out_dir: str, module: str) -> str: if os.path.exists(alt_fnam): return alt_fnam return fnam + +def main(): + suite = StubgenUtilSuite() + suite.test_infer_sig_from_docstring_invalid_signature() + suite.test_infer_sig_from_docstring() + suite.test_infer_sig_from_docstring_bad_indentation() + suite.test_infer_sig_from_docstring_duplicate_args() + suite.test_infer_sig_from_docstring_deeply_nested_types() + suite.test_infer_sig_from_docstring_multiple_overloads() + +if __name__ == "__main__": + main() \ No newline at end of file From b30ee2be925156fb9b7ed17718ab31ae33cc3336 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Aug 2024 23:28:51 +0000 Subject: [PATCH 4/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/test/teststubgen.py | 41 ++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 29ccccce10bd..a91d4397f46f 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -391,7 +391,13 @@ def test_infer_sig_from_docstring(self) -> None: assert_equal( infer_sig_from_docstring("\nfunc(*args, **kwargs)", "func"), - [FunctionSig(name="func", args=[ArgSig(name="*args"), ArgSig(name="**kwargs")], ret_type="Any")], + [ + FunctionSig( + name="func", + args=[ArgSig(name="*args"), ArgSig(name="**kwargs")], + ret_type="Any", + ) + ], ) def test_infer_sig_from_docstring_duplicate_args(self) -> None: @@ -447,11 +453,28 @@ def test_infer_sig_from_docstring_multiple_overloads(self) -> None: 2. func(self: class_type, a: float, b: int) -> None """ - assert_equal(infer_sig_from_docstring(input, "func"), [ - FunctionSig(name="func", args=[ArgSig(name="self", type="class_type")], ret_type="None"), - FunctionSig(name="func", args=[ArgSig(name="self", type="class_type"), ArgSig(name="a", type="float"), ArgSig(name="b", type="int")], ret_type="None"), - FunctionSig(name="func", args=[ArgSig(name="*args"), ArgSig(name="**kwargs")], ret_type="Any") - ]) + assert_equal( + infer_sig_from_docstring(input, "func"), + [ + FunctionSig( + name="func", args=[ArgSig(name="self", type="class_type")], ret_type="None" + ), + FunctionSig( + name="func", + args=[ + ArgSig(name="self", type="class_type"), + ArgSig(name="a", type="float"), + ArgSig(name="b", type="int"), + ], + ret_type="None", + ), + FunctionSig( + name="func", + args=[ArgSig(name="*args"), ArgSig(name="**kwargs")], + ret_type="Any", + ), + ], + ) def test_infer_sig_from_docstring_deeply_nested_types(self) -> None: assert_equal( @@ -1520,6 +1543,7 @@ def module_to_path(out_dir: str, module: str) -> str: return alt_fnam return fnam + def main(): suite = StubgenUtilSuite() suite.test_infer_sig_from_docstring_invalid_signature() @@ -1528,6 +1552,7 @@ def main(): suite.test_infer_sig_from_docstring_duplicate_args() suite.test_infer_sig_from_docstring_deeply_nested_types() suite.test_infer_sig_from_docstring_multiple_overloads() - + + if __name__ == "__main__": - main() \ No newline at end of file + main() From 986b3a6bb20d482bed3cbc4873b4c2f279afdf7b Mon Sep 17 00:00:00 2001 From: Matt Angus Date: Sat, 10 Aug 2024 00:29:35 +0100 Subject: [PATCH 5/5] remove debugging code --- mypy/test/teststubgen.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index a91d4397f46f..8b6ce54013ac 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -1542,17 +1542,3 @@ def module_to_path(out_dir: str, module: str) -> str: if os.path.exists(alt_fnam): return alt_fnam return fnam - - -def main(): - suite = StubgenUtilSuite() - suite.test_infer_sig_from_docstring_invalid_signature() - suite.test_infer_sig_from_docstring() - suite.test_infer_sig_from_docstring_bad_indentation() - suite.test_infer_sig_from_docstring_duplicate_args() - suite.test_infer_sig_from_docstring_deeply_nested_types() - suite.test_infer_sig_from_docstring_multiple_overloads() - - -if __name__ == "__main__": - main()