From cc30a3c15ef098860157788ed14e3a39f5c9b9e5 Mon Sep 17 00:00:00 2001 From: Mathis Chenuet <9201969+artemisart@users.noreply.github.com> Date: Tue, 3 Sep 2024 18:17:18 +0200 Subject: [PATCH 01/11] Fix _dict_from_slots, solves Path comparison --- deepdiff/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 4dfec50c..44277f59 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -421,7 +421,7 @@ def unmangle(attribute): else: all_slots.extend(slots) - return {i: getattr(object, unmangle(i)) for i in all_slots} + return {i: getattr(object, unmangle(i), None) for i in all_slots} def _diff_enum(self, level, parents_ids=frozenset(), local_tree=None): t1 = detailed__dict__(level.t1, include_keys=ENUM_INCLUDE_KEYS) From c1161b348e2c89335015b94064aa1effffb84db4 Mon Sep 17 00:00:00 2001 From: Mathis Chenuet <9201969+artemisart@users.noreply.github.com> Date: Tue, 3 Sep 2024 17:58:50 +0000 Subject: [PATCH 02/11] use hasattr instead of getattr None --- deepdiff/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 44277f59..6437fa37 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -421,7 +421,7 @@ def unmangle(attribute): else: all_slots.extend(slots) - return {i: getattr(object, unmangle(i), None) for i in all_slots} + return {i: getattr(object, key) for i in all_slots if hasattr(object, key := unmangle(i))} def _diff_enum(self, level, parents_ids=frozenset(), local_tree=None): t1 = detailed__dict__(level.t1, include_keys=ENUM_INCLUDE_KEYS) From 38ac719b33855ae3c859da7e107984f33045e236 Mon Sep 17 00:00:00 2001 From: Mathis Chenuet <9201969+artemisart@users.noreply.github.com> Date: Thu, 12 Sep 2024 21:24:41 +0000 Subject: [PATCH 03/11] no diff anymore --- tests/test_diff_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_diff_text.py b/tests/test_diff_text.py index ec6f66b4..3e5fcc8e 100755 --- a/tests/test_diff_text.py +++ b/tests/test_diff_text.py @@ -1713,7 +1713,7 @@ def __str__(self): t2 = Bad() ddiff = DeepDiff(t1, t2) - result = {'unprocessed': ['root: Bad Object and Bad Object']} + result = {} assert result == ddiff def test_dict_none_item_removed(self): From ce1c8fb389f627b55e007c3c9a3640ab59d5981d Mon Sep 17 00:00:00 2001 From: Mathis Chenuet <9201969+artemisart@users.noreply.github.com> Date: Thu, 12 Sep 2024 21:29:55 +0000 Subject: [PATCH 04/11] add author --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index cd3db130..32ae5fcc 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -63,3 +63,4 @@ Authors in order of the timeline of their contributions: - [sf-tcalhoun](https://github.com/sf-tcalhoun) for fixing "Instantiating a Delta with a flat_dict_list unexpectedly mutates the flat_dict_list" - [dtorres-sf](https://github.com/dtorres-sf) for fixing iterable moved items when iterable_compare_func is used. - [Florian Finkernagel](https://github.com/TyberiusPrime) for pandas and polars support. +- Mathis Chenuet [artemisart](https://github.com/artemisart) for fixing slots classes comparison. From 579784145b3cc289baa19a6857c4b8659d057c4c Mon Sep 17 00:00:00 2001 From: David Hotham Date: Sun, 15 Sep 2024 12:33:38 +0100 Subject: [PATCH 05/11] relax orderly-set dependency --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 28bbd74e..640cf147 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -orderly-set==5.2.2 +orderly-set>=5.2.2,<6 From 5f22bd27ad73b62183cc85dd550d669ddb9706e2 Mon Sep 17 00:00:00 2001 From: "Aaron D. Marasco" Date: Wed, 9 Oct 2024 21:40:40 -0400 Subject: [PATCH 06/11] Add print() option --- AUTHORS.md | 1 + CHANGELOG.md | 1 + deepdiff/serialization.py | 8 +++++-- docs/view.rst | 23 ++++++++++++++++++++ tests/test_serialization.py | 43 +++++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 2 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index 32ae5fcc..79d9edbf 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -64,3 +64,4 @@ Authors in order of the timeline of their contributions: - [dtorres-sf](https://github.com/dtorres-sf) for fixing iterable moved items when iterable_compare_func is used. - [Florian Finkernagel](https://github.com/TyberiusPrime) for pandas and polars support. - Mathis Chenuet [artemisart](https://github.com/artemisart) for fixing slots classes comparison. +- [Aaron D. Marasco](https://github.com/AaronDMarasco) added `prefix` option to `pretty()` diff --git a/CHANGELOG.md b/CHANGELOG.md index 95cd2c74..12da1c20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - v8-0-1 - Bugfix. Numpy should be optional. + - Added `prefix` option to `pretty()` - v8-0-0 diff --git a/deepdiff/serialization.py b/deepdiff/serialization.py index 5b4075e2..e350b3cf 100644 --- a/deepdiff/serialization.py +++ b/deepdiff/serialization.py @@ -296,7 +296,7 @@ def _to_delta_dict(self, directed=True, report_repetition_required=True, always_ return deepcopy(dict(result)) - def pretty(self): + def pretty(self, prefix=None): """ The pretty human readable string output for the diff object regardless of what view was used to generate the diff. @@ -310,12 +310,16 @@ def pretty(self): Item root[1] removed from set. """ result = [] + if prefix is None: + prefix = '' keys = sorted(self.tree.keys()) # sorting keys to guarantee constant order across python versions. for key in keys: for item_key in self.tree[key]: result += [pretty_print_diff(item_key)] - return '\n'.join(result) + if callable(prefix): + return "\n".join(f"{prefix(diff=self)}{r}" for r in result) + return "\n".join(f"{prefix}{r}" for r in result) class _RestrictedUnpickler(pickle.Unpickler): diff --git a/docs/view.rst b/docs/view.rst index f50fc9f1..6343590f 100644 --- a/docs/view.rst +++ b/docs/view.rst @@ -299,6 +299,29 @@ Use the pretty method for human readable output. This is regardless of what view Item root[4] removed from set. Item root[1] removed from set. +The pretty method has an optional parameter ``prefix`` that allows a prefix string before every output line (*e.g.* for logging): + >>> from deepdiff import DeepDiff + >>> t1={1,2,4} + >>> t2={2,3} + >>> print(DeepDiff(t1, t2).pretty(prefix='Diff: ')) + Diff: Item root[3] added to set. + Diff: Item root[4] removed from set. + Diff: Item root[1] removed from set. + +The ``prefix`` may also be a callable function. This function must accept ``**kwargs``; as of this version, the only parameter is ``diff`` but the signature allows for future expansion. +The ``diff`` given will be the ``DeepDiff`` that ``pretty`` was called on; this allows interesting capabilities such as: + >>> from deepdiff import DeepDiff + >>> t1={1,2,4} + >>> t2={2,3} + >>> def callback(**kwargs): + ... """Helper function using a hidden variable on the diff that tracks which count prints next""" + ... kwargs['diff']._diff_count = 1 + getattr(kwargs['diff'], '_diff_count', 0) + ... return f"Diff #{kwargs['diff']._diff_count}: " + ... + >>> print(DeepDiff(t1, t2).pretty(prefix=callback)) + Diff #1: Item root[3] added to set. + Diff #2: Item root[4] removed from set. + Diff #3: Item root[1] removed from set. Text view vs. Tree view vs. vs. pretty() method diff --git a/tests/test_serialization.py b/tests/test_serialization.py index facda246..d578e53a 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -330,6 +330,49 @@ def test_pretty_form_method(self, expected, verbose_level): result = ddiff.pretty() assert result == expected + @pytest.mark.parametrize("expected, verbose_level", + ( + ('\t\tItem root[5] added to dictionary.' + '\n\t\tItem root[3] removed from dictionary.' + '\n\t\tType of root[2] changed from int to str and value changed from 2 to "b".' + '\n\t\tValue of root[4] changed from 4 to 5.', 0), + ('\t\tItem root[5] (5) added to dictionary.' + '\n\t\tItem root[3] (3) removed from dictionary.' + '\n\t\tType of root[2] changed from int to str and value changed from 2 to "b".' + '\n\t\tValue of root[4] changed from 4 to 5.', 2), + ), ids=("verbose=0", "verbose=2") + ) + def test_pretty_form_method_prefixed_simple(self, expected, verbose_level): + t1 = {2: 2, 3: 3, 4: 4} + t2 = {2: 'b', 4: 5, 5: 5} + ddiff = DeepDiff(t1, t2, verbose_level=verbose_level) + result = ddiff.pretty(prefix="\t\t") + assert result == expected + + @pytest.mark.parametrize("expected, verbose_level", + ( + ('Diff #1: Item root[5] added to dictionary.' + '\nDiff #2: Item root[3] removed from dictionary.' + '\nDiff #3: Type of root[2] changed from int to str and value changed from 2 to "b".' + '\nDiff #4: Value of root[4] changed from 4 to 5.', 0), + ('Diff #1: Item root[5] (5) added to dictionary.' + '\nDiff #2: Item root[3] (3) removed from dictionary.' + '\nDiff #3: Type of root[2] changed from int to str and value changed from 2 to "b".' + '\nDiff #4: Value of root[4] changed from 4 to 5.', 2), + ), ids=("verbose=0", "verbose=2") + ) + def test_pretty_form_method_prefixed_callback(self, expected, verbose_level): + def prefix_callback(**kwargs): + """Helper function using a hidden variable on the diff that tracks which count prints next""" + kwargs['diff']._diff_count = 1 + getattr(kwargs['diff'], '_diff_count', 0) + return f"Diff #{kwargs['diff']._diff_count}: " + + t1 = {2: 2, 3: 3, 4: 4} + t2 = {2: 'b', 4: 5, 5: 5} + ddiff = DeepDiff(t1, t2, verbose_level=verbose_level) + result = ddiff.pretty(prefix=prefix_callback) + assert result == expected + @pytest.mark.parametrize('test_num, value, func_to_convert_back', [ (1, {'10': None}, None), (2, {"type_changes": {"root": {"old_type": None, "new_type": list, "new_value": ["你好", 2, 3, 5]}}}, None), From 32df472cdb4c5c5eba6d8bfe8e6f1429649f6460 Mon Sep 17 00:00:00 2001 From: Doron Behar Date: Sun, 20 Oct 2024 02:55:11 +0300 Subject: [PATCH 07/11] DeepHash: check numpy booleans like native booleans Fixes #494 --- deepdiff/deephash.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deepdiff/deephash.py b/deepdiff/deephash.py index 32fee9c3..7c2e2b47 100644 --- a/deepdiff/deephash.py +++ b/deepdiff/deephash.py @@ -24,6 +24,11 @@ import polars except ImportError: polars = False +try: + import numpy as np + booleanTypes = (bool, np.bool_) +except ImportError: + booleanTypes = bool logger = logging.getLogger(__name__) @@ -492,7 +497,7 @@ def _hash(self, obj, parent, parents_ids=EMPTY_FROZENSET): """The main hash method""" counts = 1 - if isinstance(obj, bool): + if isinstance(obj, booleanTypes): obj = self._prep_bool(obj) result = None elif self.use_enum_value and isinstance(obj, Enum): From cee3d41868a9c973c48471f020f63380c271fad0 Mon Sep 17 00:00:00 2001 From: Doron Behar Date: Sun, 20 Oct 2024 20:05:48 +0300 Subject: [PATCH 08/11] TestDeepHash: test numpy booleans --- tests/test_hash.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_hash.py b/tests/test_hash.py index 52637577..22a86e24 100755 --- a/tests/test_hash.py +++ b/tests/test_hash.py @@ -187,6 +187,12 @@ def test_re(self): a_hash = DeepHash(a)[a] assert not( a_hash is unprocessed) + # https://github.com/seperman/deepdiff/issues/494 + def test_numpy_bool(self): + a = {'b': np.array([True], dtype='bool')} + a_hash = DeepHash(a)[a] + assert not( a_hash is unprocessed) + class TestDeepHashPrep: """DeepHashPrep Tests covering object serialization.""" From 7bb48a13636df3ec9e5a7463a31f8f318ea3e86f Mon Sep 17 00:00:00 2001 From: Joachim Langenbach Date: Sat, 26 Oct 2024 10:31:36 +0200 Subject: [PATCH 09/11] Added missing suffix of tests/test_diff_include_paths_root.py --- tests/{test_diff_include_paths => test_diff_include_paths.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_diff_include_paths => test_diff_include_paths.py} (100%) diff --git a/tests/test_diff_include_paths b/tests/test_diff_include_paths.py similarity index 100% rename from tests/test_diff_include_paths rename to tests/test_diff_include_paths.py From 916f02f6a10f3338219fe3d9b1ae9658ea74c5ce Mon Sep 17 00:00:00 2001 From: Joachim Langenbach Date: Sat, 26 Oct 2024 10:32:33 +0200 Subject: [PATCH 10/11] Added tests for wrong diff result with include_paths and changed number of attributes in dict --- tests/test_diff_include_paths_count.py | 160 +++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 tests/test_diff_include_paths_count.py diff --git a/tests/test_diff_include_paths_count.py b/tests/test_diff_include_paths_count.py new file mode 100644 index 00000000..ccb195ce --- /dev/null +++ b/tests/test_diff_include_paths_count.py @@ -0,0 +1,160 @@ +import pytest +from deepdiff import DeepDiff + +@pytest.mark.parametrize( + "data, result", + [ + ( + { + "old": { + 'name': 'Testname Old', + 'desciption': 'Desc Old', + 'sub_path': { + 'name': 'Testname Subpath old', + 'desciption': 'Desc Subpath old', + }, + }, + "new": { + 'name': 'Testname New', + 'desciption': 'Desc New', + 'new_attribute': 'New Value', + 'sub_path': { + 'name': 'Testname Subpath old', + 'desciption': 'Desc Subpath old', + }, + }, + "include_paths": "root['sub_path']", + }, + {} + ), + ( + { + "old": { + 'name': 'Testname Old', + 'desciption': 'Desc Old', + 'sub_path': { + 'name': 'Testname Subpath old', + 'desciption': 'Desc Subpath old', + }, + }, + "new": { + 'name': 'Testname New', + 'desciption': 'Desc New', + 'new_attribute': 'New Value', + 'sub_path': { + 'name': 'Testname Subpath New', + 'desciption': 'Desc Subpath old', + }, + }, + "include_paths": "root['sub_path']", + }, + {"values_changed": {"root['sub_path']['name']": {"old_value": "Testname Subpath old", "new_value": "Testname Subpath New"}}} + ), + ( + { + "old": { + 'name': 'Testname Old', + 'desciption': 'Desc Old', + 'sub_path': { + 'name': 'Testname Subpath old', + 'desciption': 'Desc Subpath old', + 'old_attr': 'old attr value', + }, + }, + "new": { + 'name': 'Testname New', + 'desciption': 'Desc New', + 'new_attribute': 'New Value', + 'sub_path': { + 'name': 'Testname Subpath old', + 'desciption': 'Desc Subpath New', + 'new_sub_path_attr': 'new sub path attr value', + }, + }, + "include_paths": "root['sub_path']['name']", + }, + {} + ), + ( + { + "old": { + 'name': 'Testname old', + 'desciption': 'Desc old', + 'new_attribute': 'old Value', + 'sub_path': { + 'name': 'Testname', + 'removed_attr': 'revemod attr value', + }, + }, + "new": { + 'name': 'Testname new', + 'desciption': 'Desc new', + 'new_attribute': 'new Value', + 'sub_path': { + 'added_attr': 'Added Attr Value', + 'name': 'Testname', + }, + }, + "include_paths": "root['sub_path']['name']", + }, + {} + ), + ( + { + "old": { + 'name': 'Testname', + 'removed_attr': 'revemod attr value', + }, + "new": { + 'added_attr': 'Added Attr Value', + 'name': 'Testname', + }, + "include_paths": "root['name']", + }, + {} + ), + ( + { + "old": { + 'name': 'Testname', + 'removed_attr': 'revemod attr value', + 'removed_attr_2': 'revemod attr value', + }, + "new": { + 'added_attr': 'Added Attr Value', + 'name': 'Testname', + }, + "include_paths": "root['name']", + }, + {} + ), + ( + { + "old": { + 'name': 'Testname old', + 'desciption': 'Desc old', + 'new_attribute': 'old Value', + 'sub_path': { + 'name': 'Testname', + 'removed_attr': 'revemod attr value', + 'removed_attr_2': 'blu', + }, + }, + "new": { + 'name': 'Testname new', + 'desciption': 'Desc new', + 'new_attribute': 'new Value', + 'sub_path': { + 'added_attr': 'Added Attr Value', + 'name': 'Testname', + }, + }, + "include_paths": "root['sub_path']['name']", + }, + {} + ), + ] +) +def test_diff_include_paths_root(data, result): + diff = DeepDiff(data["old"], data["new"], include_paths=data["include_paths"]) + assert diff == result From fc8baaafc7077ca86c5d258e3aa1bb503b335db2 Mon Sep 17 00:00:00 2001 From: Joachim Langenbach Date: Sat, 26 Oct 2024 12:08:24 +0200 Subject: [PATCH 11/11] Fixed include_paths fault, if only certain keys of a path are included --- deepdiff/diff.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/deepdiff/diff.py b/deepdiff/diff.py index 4dfec50c..61284af8 100755 --- a/deepdiff/diff.py +++ b/deepdiff/diff.py @@ -510,6 +510,32 @@ def _skip_this(self, level): return skip + def _skip_this_key(self, level, key): + # if include_paths is not set, than treet every path as included + if self.include_paths is None: + return False + if "{}['{}']".format(level.path(), key) in self.include_paths: + return False + if level.path() in self.include_paths: + # matches e.g. level+key root['foo']['bar']['veg'] include_paths ["root['foo']['bar']"] + return False + for prefix in self.include_paths: + if "{}['{}']".format(level.path(), key) in prefix: + # matches as long the prefix is longer than this object key + # eg.: level+key root['foo']['bar'] matches prefix root['foo']['bar'] from include paths + # level+key root['foo'] matches prefix root['foo']['bar'] from include_paths + # level+key root['foo']['bar'] DOES NOT match root['foo'] from include_paths This needs to be handled afterwards + return False + # check if a higher level is included as a whole (=without any sublevels specified) + # matches e.g. level+key root['foo']['bar']['veg'] include_paths ["root['foo']"] + # but does not match, if it is level+key root['foo']['bar']['veg'] include_paths ["root['foo']['bar']['fruits']"] + up = level.up + while up is not None: + if up.path() in self.include_paths: + return False + up = up.up + return True + def _get_clean_to_keys_mapping(self, keys, level): """ Get a dictionary of cleaned value of keys to the keys themselves. @@ -570,11 +596,11 @@ def _diff_dict( rel_class = DictRelationship if self.ignore_private_variables: - t1_keys = SetOrdered([key for key in t1 if not(isinstance(key, str) and key.startswith('__'))]) - t2_keys = SetOrdered([key for key in t2 if not(isinstance(key, str) and key.startswith('__'))]) + t1_keys = SetOrdered([key for key in t1 if not(isinstance(key, str) and key.startswith('__')) and not self._skip_this_key(level, key)]) + t2_keys = SetOrdered([key for key in t2 if not(isinstance(key, str) and key.startswith('__')) and not self._skip_this_key(level, key)]) else: - t1_keys = SetOrdered(t1.keys()) - t2_keys = SetOrdered(t2.keys()) + t1_keys = SetOrdered([key for key in t1 if not self._skip_this_key(level, key)]) + t2_keys = SetOrdered([key for key in t2 if not self._skip_this_key(level, key)]) if self.ignore_string_type_changes or self.ignore_numeric_type_changes or self.ignore_string_case: t1_clean_to_keys = self._get_clean_to_keys_mapping(keys=t1_keys, level=level) t2_clean_to_keys = self._get_clean_to_keys_mapping(keys=t2_keys, level=level)