diff --git a/python/google/protobuf/internal/_parameterized.py b/python/google/protobuf/internal/_parameterized.py index 6d4afec8e6f8..8ead39487e88 100755 --- a/python/google/protobuf/internal/_parameterized.py +++ b/python/google/protobuf/internal/_parameterized.py @@ -7,6 +7,8 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +# TODO: Replace this with the now-open-sourced absl-py version. + """Adds support for parameterized tests to Python's unittest TestCase class. A parameterized test is a method in a test case that is invoked with different @@ -139,6 +141,8 @@ def testIsNegative(self, arg): _SEPARATOR = uuid.uuid1().hex _FIRST_ARG = object() _ARGUMENT_REPR = object() +_NAMED = object() +_NAMED_DICT_KEY = 'testcase_name' def _CleanRepr(obj): @@ -156,6 +160,14 @@ def _NonStringIterable(obj): not isinstance(obj, str)) +def _NonStringOrBytesIterable(obj): + return ( + isinstance(obj, collections_abc.Iterable) + and not isinstance(obj, str) + and not isinstance(obj, bytes) + ) + + def _FormatParameterList(testcase_params): if isinstance(testcase_params, collections_abc.Mapping): return ', '.join('%s=%s' % (argname, _CleanRepr(value)) @@ -169,7 +181,7 @@ def _FormatParameterList(testcase_params): class _ParameterizedTestIter(object): """Callable and iterable class for producing new test cases.""" - def __init__(self, test_method, testcases, naming_type): + def __init__(self, test_method, testcases, naming_type, original_name=None): """Returns concrete test functions for a test and a list of parameters. The naming_type is used to determine the name of the concrete @@ -179,13 +191,22 @@ def __init__(self, test_method, testcases, naming_type): Args: test_method: The decorated test method. - testcases: (list of tuple/dict) A list of parameter - tuples/dicts for individual test invocations. + testcases: (list of tuple/dict) A list of parameter tuples/dicts for + individual test invocations. naming_type: The test naming type, either _NAMED or _ARGUMENT_REPR. + original_name: The original test method name. When decorated on a test + method, None is passed to __init__ and test_method.__name__ is used. + Note test_method.__name__ might be different than the original defined + test method because of the use of other decorators. A more accurate + value is set by TestGeneratorMetaclass.__new__ later. """ + if original_name is None: + original_name = test_method.__name__ + self._test_method = test_method self.testcases = testcases self._naming_type = naming_type + self._original_name = original_name def __call__(self, *args, **kwargs): raise RuntimeError('You appear to be running a parameterized test case ' @@ -207,12 +228,45 @@ def BoundParamTest(self): else: test_method(self, testcase_params) - if naming_type is _FIRST_ARG: + if naming_type is _NAMED: # Signal the metaclass that the name of the test function is unique # and descriptive. BoundParamTest.__x_use_name__ = True - BoundParamTest.__name__ += str(testcase_params[0]) - testcase_params = testcase_params[1:] + + testcase_name = None + if isinstance(testcase_params, collections_abc.Mapping): + if _NAMED_DICT_KEY not in testcase_params: + raise RuntimeError( + 'Dict for named tests must contain key "%s"' % _NAMED_DICT_KEY + ) + # Create a new dict to avoid modifying the supplied testcase_params. + testcase_name = testcase_params[_NAMED_DICT_KEY] + testcase_params = { + k: v for k, v in testcase_params.items() if k != _NAMED_DICT_KEY + } + elif _NonStringOrBytesIterable(testcase_params): + if not isinstance(testcase_params[0], str): + raise RuntimeError( + 'The first element of named test parameters is the test name ' + 'suffix and must be a string' + ) + testcase_name = testcase_params[0] + testcase_params = testcase_params[1:] + else: + raise RuntimeError( + 'Named tests must be passed a dict or non-string iterable.' + ) + + test_method_name = self._original_name + # Support PEP-8 underscore style for test naming if used. + if ( + test_method_name.startswith('test_') + and testcase_name + and not testcase_name.startswith('_') + ): + test_method_name += '_' + + BoundParamTest.__name__ = test_method_name + str(testcase_name) elif naming_type is _ARGUMENT_REPR: # __x_extra_id__ is used to pass naming information to the __new__ # method of TestGeneratorMetaclass. @@ -249,8 +303,11 @@ def _ModifyClass(class_object, testcases, naming_type): delattr(class_object, name) methods = {} _UpdateClassDictForParamTestCase( - methods, id_suffix, name, - _ParameterizedTestIter(obj, testcases, naming_type)) + methods, + id_suffix, + name, + _ParameterizedTestIter(obj, testcases, naming_type, name), + ) for name, meth in methods.items(): setattr(class_object, name, meth) @@ -302,18 +359,23 @@ def parameters(*testcases): # pylint: disable=invalid-name def named_parameters(*testcases): # pylint: disable=invalid-name """A decorator for creating parameterized tests. - See the module docstring for a usage example. The first element of - each parameter tuple should be a string and will be appended to the - name of the test method. + See the module docstring for a usage example. For every parameter tuple + passed, the first element of the tuple should be a string and will be appended + to the name of the test method. Each parameter dict passed must have a value + for the key "testcase_name", the string representation of that value will be + appended to the name of the test method. Args: - *testcases: Parameters for the decorated method, either a single - iterable, or a list of tuples. + *testcases: Parameters for the decorated method, either a single iterable, + or a list of tuples or dicts. + + Raises: + NoTestsError: Raised when the decorator generates no tests. Returns: A test generator to be handled by TestGeneratorMetaclass. """ - return _ParameterDecorator(_FIRST_ARG, testcases) + return _ParameterDecorator(_NAMED, testcases) class TestGeneratorMetaclass(type): @@ -334,6 +396,12 @@ def __new__(mcs, class_name, bases, dct): for name, obj in dct.copy().items(): if (name.startswith(unittest.TestLoader.testMethodPrefix) and _NonStringIterable(obj)): + if isinstance(obj, _ParameterizedTestIter): + # Update the original test method name so it's more accurate. + # The mismatch might happen when another decorator is used inside + # the parameterized decrators, and the inner decorator doesn't + # preserve its __name__. + obj._original_name = name iterator = iter(obj) dct.pop(name) _UpdateClassDictForParamTestCase(dct, id_suffix, name, iterator) diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index e44f59703e3c..b0bf98eb2886 100644 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -2141,84 +2141,232 @@ def testProto3Optional(self): self.assertEqual(text_format.MessageToString(msg2), text) -class TokenizerTest(unittest.TestCase): - - def testSimpleTokenCases(self): - text = ('identifier1:"string1"\n \n\n' - 'identifier2 : \n \n123 \n identifier3 :\'string\'\n' - 'identifiER_4 : 1.1e+2 ID5:-0.23 ID6:\'aaaa\\\'bbbb\'\n' - 'ID7 : "aa\\"bb"\n\n\n\n ID8: {A:inf B:-inf C:true D:false}\n' - 'ID9: 22 ID10: -111111111111111111 ID11: -22\n' - 'ID12: 2222222222222222222 ID13: 1.23456f ID14: 1.2e+2f ' - 'false_bool: 0 true_BOOL:t \n true_bool1: 1 false_BOOL1:f ' - 'False_bool: False True_bool: True X:iNf Y:-inF Z:nAN') +def _CreateConsumeLiteralToken(expected_literal): + def _Consume(tokenizer): + tokenizer.Consume(expected_literal) + return expected_literal + + return (_Consume, expected_literal) + + +class TokenizerTest(_parameterized.TestCase): + + @_parameterized.named_parameters([ + dict( + testcase_name='_string_double_quotes', + text='identifier1:"string1"\n', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'identifier1'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeString, 'string1'), + ], + ), + dict( + testcase_name='_integer', + text='identifier2 : \n \n123 ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'identifier2'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeInteger, 123), + ], + ), + dict( + testcase_name='_string_single_quotes', + text="\n identifier3:'string'\n", + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'identifier3'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeString, 'string'), + ], + ), + dict( + testcase_name='_float_exponent', + text='identifiER_4 : 1.1e+2 ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'identifiER_4'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeFloat, 1.1e2), + ], + ), + dict( + testcase_name='_float', + text='ID5:-0.23', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'ID5'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeFloat, -0.23), + ], + ), + dict( + testcase_name='_escape_single_quote', + text="ID6:'aaaa\\'bbbb'\n", + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'ID6'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeString, "aaaa'bbbb"), + ], + ), + dict( + testcase_name='_escape_double_quote', + text='ID7 : "aa\\"bb"\n\n\n\n ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'ID7'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeString, 'aa"bb'), + ], + ), + dict( + testcase_name='_submessage', + text='ID8: {A:inf B:-inf C:true D:false}\n', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'ID8'), + _CreateConsumeLiteralToken(':'), + _CreateConsumeLiteralToken('{'), + (text_format.Tokenizer.ConsumeIdentifier, 'A'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeFloat, float('inf')), + (text_format.Tokenizer.ConsumeIdentifier, 'B'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeFloat, float('-inf')), + (text_format.Tokenizer.ConsumeIdentifier, 'C'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeBool, True), + (text_format.Tokenizer.ConsumeIdentifier, 'D'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeBool, False), + _CreateConsumeLiteralToken('}'), + ], + ), + dict( + testcase_name='_large_negative_integer', + text='ID10: -111111111111111111 ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'ID10'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeInteger, -111111111111111111), + ], + ), + dict( + testcase_name='_negative_integer', + text='ID11: -22\n', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'ID11'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeInteger, -22), + ], + ), + dict( + testcase_name='_large_integer', + text='ID12: 2222222222222222222 ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'ID12'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeInteger, 2222222222222222222), + ], + ), + dict( + testcase_name='_float_suffix', + text='ID13: 1.23456f ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'ID13'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeFloat, 1.23456), + ], + ), + dict( + testcase_name='_float_exponent_suffix', + text='ID14: 1.2e+2f ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'ID14'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeFloat, 1.2e2), + ], + ), + dict( + testcase_name='_bool_zero', + text='false_bool: 0 ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'false_bool'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeBool, False), + ], + ), + dict( + testcase_name='_bool_t', + text='true_BOOL:t ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'true_BOOL'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeBool, True), + ], + ), + dict( + testcase_name='_bool_one', + text='true_bool1: 1 ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'true_bool1'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeBool, True), + ], + ), + dict( + testcase_name='_bool_f', + text='false_BOOL1:f ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'false_BOOL1'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeBool, False), + ], + ), + dict( + testcase_name='_bool_false', + text='False_bool: False ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'False_bool'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeBool, False), + ], + ), + dict( + testcase_name='_bool_true', + text='True_bool: True ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'True_bool'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeBool, True), + ], + ), + dict( + testcase_name='_float_inf', + text='X:iNf ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'X'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeFloat, float('inf')), + ], + ), + dict( + testcase_name='_float_negative_inf', + text='Y:-inF ', + expected=[ + (text_format.Tokenizer.ConsumeIdentifier, 'Y'), + _CreateConsumeLiteralToken(':'), + (text_format.Tokenizer.ConsumeFloat, float('-inf')), + ], + ), + ]) + def testSimpleTokenCases(self, text, expected): + consume_functions, expected_tokens = zip(*expected) tokenizer = text_format.Tokenizer(text.splitlines()) - methods = [(tokenizer.ConsumeIdentifier, 'identifier1'), ':', - (tokenizer.ConsumeString, 'string1'), - (tokenizer.ConsumeIdentifier, 'identifier2'), ':', - (tokenizer.ConsumeInteger, 123), - (tokenizer.ConsumeIdentifier, 'identifier3'), ':', - (tokenizer.ConsumeString, 'string'), - (tokenizer.ConsumeIdentifier, 'identifiER_4'), ':', - (tokenizer.ConsumeFloat, 1.1e+2), - (tokenizer.ConsumeIdentifier, 'ID5'), ':', - (tokenizer.ConsumeFloat, -0.23), - (tokenizer.ConsumeIdentifier, 'ID6'), ':', - (tokenizer.ConsumeString, 'aaaa\'bbbb'), - (tokenizer.ConsumeIdentifier, 'ID7'), ':', - (tokenizer.ConsumeString, 'aa\"bb'), - (tokenizer.ConsumeIdentifier, 'ID8'), ':', '{', - (tokenizer.ConsumeIdentifier, 'A'), ':', - (tokenizer.ConsumeFloat, float('inf')), - (tokenizer.ConsumeIdentifier, 'B'), ':', - (tokenizer.ConsumeFloat, -float('inf')), - (tokenizer.ConsumeIdentifier, 'C'), ':', - (tokenizer.ConsumeBool, True), - (tokenizer.ConsumeIdentifier, 'D'), ':', - (tokenizer.ConsumeBool, False), '}', - (tokenizer.ConsumeIdentifier, 'ID9'), ':', - (tokenizer.ConsumeInteger, 22), - (tokenizer.ConsumeIdentifier, 'ID10'), ':', - (tokenizer.ConsumeInteger, -111111111111111111), - (tokenizer.ConsumeIdentifier, 'ID11'), ':', - (tokenizer.ConsumeInteger, -22), - (tokenizer.ConsumeIdentifier, 'ID12'), ':', - (tokenizer.ConsumeInteger, 2222222222222222222), - (tokenizer.ConsumeIdentifier, 'ID13'), ':', - (tokenizer.ConsumeFloat, 1.23456), - (tokenizer.ConsumeIdentifier, 'ID14'), ':', - (tokenizer.ConsumeFloat, 1.2e+2), - (tokenizer.ConsumeIdentifier, 'false_bool'), ':', - (tokenizer.ConsumeBool, False), - (tokenizer.ConsumeIdentifier, 'true_BOOL'), ':', - (tokenizer.ConsumeBool, True), - (tokenizer.ConsumeIdentifier, 'true_bool1'), ':', - (tokenizer.ConsumeBool, True), - (tokenizer.ConsumeIdentifier, 'false_BOOL1'), ':', - (tokenizer.ConsumeBool, False), - (tokenizer.ConsumeIdentifier, 'False_bool'), ':', - (tokenizer.ConsumeBool, False), - (tokenizer.ConsumeIdentifier, 'True_bool'), ':', - (tokenizer.ConsumeBool, True), - (tokenizer.ConsumeIdentifier, 'X'), ':', - (tokenizer.ConsumeFloat, float('inf')), - (tokenizer.ConsumeIdentifier, 'Y'), ':', - (tokenizer.ConsumeFloat, float('-inf')), - (tokenizer.ConsumeIdentifier, 'Z'), ':', - (tokenizer.ConsumeFloat, float('nan'))] - - i = 0 - while not tokenizer.AtEnd(): - m = methods[i] - if isinstance(m, str): - token = tokenizer.token - self.assertEqual(token, m) - tokenizer.NextToken() - elif isinstance(m[1], float) and math.isnan(m[1]): - self.assertTrue(math.isnan(m[0]())) - else: - self.assertEqual(m[1], m[0]()) - i += 1 + tokens = [consume(tokenizer) for consume in consume_functions] + + self.assertTrue(tokenizer.AtEnd()) + self.assertEqual(tokens, [token for token in expected_tokens]) + + def testConsumeNan(self): + tokenizer = text_format.Tokenizer(['nAN']) + token = tokenizer.ConsumeFloat() + self.assertTrue(math.isnan(token), 'Expected NaN, got %s' % token) def testConsumeAbstractIntegers(self): # This test only tests the failures in the integer parsing methods as well