Merge pull request #174 from jacebrowning/release/v0.9

Release v0.9

.pylint.ini

@@ -26,11 +26,10 @@ disable=
     protected-access,
     # Disabling these helps to write more expressive tests:
     expression-not-assigned,
-    singleton-comparison,
-    bad-continuation,
     blacklisted-name,
     # Handled by automatic formatting:
     line-too-long,
+    bad-continuation,
     # False positives:
     no-member,
     unsupported-membership-test,

.travis.yml

@@ -21,6 +21,7 @@ install:
 script:
   - make test-repeat
   - make check
+  - make notebooks
   - make mkdocs
 
 after_success:

CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.9 (2020-04-13)
+
+- Fixed serialization of optional nested dataclasses with a value of `None`.
+- Fixed preservation of comments on nested dataclass attributes.
+- Added support for using `enum.Enum` subclasses as type annotations.
+
 # 0.8.1 (2020-03-30)
 
 - Fixed loading of `Missing` nested dataclasses attributes.

Makefile

@@ -89,7 +89,7 @@ test-profile: install
 MKDOCS_INDEX := site/index.html
 
 .PHONY: docs
-docs: mkdocs uml papermill  ## Generate documentation and UML
+docs: mkdocs uml notebooks  ## Generate documentation and UML
 
 .PHONY: mkdocs
 mkdocs: install $(MKDOCS_INDEX)
@@ -116,12 +116,12 @@ docs/*.png: $(MODULES)
 	- mv -f classes_$(PACKAGE).png docs/classes.png
 	- mv -f packages_$(PACKAGE).png docs/packages.png
 
-.PHONY: papermill
-papermill: install
+.PHONY: notebooks
+notebooks: install
 	@ cd notebooks; for filename in *.ipynb; do \
 	  poetry run papermill $$filename $$filename; \
 	done
-	git config filter.nbstripout.extrakeys 'metadata.papermill cell.metadata.papermill'
+	git config filter.nbstripout.extrakeys 'cell.metadata.execution cell.metadata.papermill metadata.papermill'
 	poetry run nbstripout --keep-output notebooks/*.ipynb
 
 # RELEASE #####################################################################

datafiles/converters/__init__.py

@@ -1,4 +1,5 @@
 import dataclasses
+from enum import Enum
 from inspect import isclass
 from typing import Any, Dict, Optional, Union
 
@@ -9,6 +10,7 @@
 from ._bases import Converter
 from .builtins import Boolean, Float, Integer, String
 from .containers import Dataclass, Dictionary, List
+from .enumerations import Enumeration
 from .extensions import *  # pylint: disable=unused-wildcard-import
 
 
@@ -46,7 +48,7 @@ def map_type(cls, *, name: str = '', item_cls: Optional[type] = None):
         converters = {}
         for field in dataclasses.fields(cls):
             converters[field.name] = map_type(field.type, name=field.name)
-        converter = Dataclass.subclass(cls, converters)
+        converter = Dataclass.of_mappings(cls, converters)
         log.debug(f'Mapped {cls!r} to new converter: {converter}')
         return converter
 
@@ -57,11 +59,10 @@ def map_type(cls, *, name: str = '', item_cls: Optional[type] = None):
             try:
                 converter = map_type(item_cls or cls.__args__[0])
             except TypeError as e:
-                if '~T' in str(e):
-                    e = TypeError(f"Type is required with 'List' annotation")
-                raise e from None
+                assert '~T' in str(e), f'Unhandled error: ' + str(e)
+                raise TypeError("Type is required with 'List' annotation") from None
             else:
-                converter = List.subclass(converter)
+                converter = List.of_type(converter)
 
         elif cls.__origin__ == dict:
             if item_cls:
@@ -72,7 +73,7 @@ def map_type(cls, *, name: str = '', item_cls: Optional[type] = None):
                 key = map_type(cls.__args__[0])
                 value = map_type(cls.__args__[1])
 
-            converter = Dictionary.subclass(key, value)
+            converter = Dictionary.of_mapping(key, value)
 
         elif cls.__origin__ == Union:
             converter = map_type(cls.__args__[0])
@@ -101,4 +102,7 @@ def map_type(cls, *, name: str = '', item_cls: Optional[type] = None):
         log.debug(f'Mapped {cls!r} to existing converter (itself)')
         return cls
 
+    if issubclass(cls, Enum):
+        return Enumeration.of_type(cls)
+
     raise TypeError(f'Could not map type: {cls}')

datafiles/converters/_bases.py

@@ -7,7 +7,7 @@ class Converter:
     """Base class for immutable attribute conversion."""
 
     TYPE: type = object
-    DEFAULT: Any = None
+    DEFAULT: Any = NotImplemented
 
     @classmethod
     def as_optional(cls):

datafiles/converters/containers.py

@@ -11,10 +11,10 @@
 class List(Converter):
     """Base converter for homogeneous lists of another converter."""
 
-    CONVERTER: Converter = None  # type: ignore
+    CONVERTER: Converter = NotImplemented
 
     @classmethod
-    def subclass(cls, converter: type):
+    def of_type(cls, converter: type):
         name = f'{converter.__name__}List'
         bases = (cls,)
         attributes = {'CONVERTER': converter}
@@ -87,7 +87,7 @@ class Dictionary(Converter):
     """Base converter for raw dictionaries."""
 
     @classmethod
-    def subclass(cls, key: type, value: type):
+    def of_mapping(cls, key: type, value: type):
         name = f'{key.__name__}{value.__name__}Dict'
         bases = (cls,)
         return type(name, bases, {})
@@ -121,11 +121,11 @@ def to_preserialization_data(cls, python_value, *, default_to_skip=None):
 class Dataclass(Converter):
     """Base converter for dataclasses."""
 
-    DATACLASS: Callable = None  # type: ignore
-    CONVERTERS: Dict = None  # type: ignore
+    DATACLASS: Callable = NotImplemented
+    CONVERTERS: Dict = NotImplemented
 
     @classmethod
-    def subclass(cls, dataclass, converters: Dict[str, type]):
+    def of_mappings(cls, dataclass, converters: Dict[str, type]):
         name = f'{dataclass.__name__}Converter'
         bases = (cls,)
         attributes = {'DATACLASS': dataclass, 'CONVERTERS': converters}
@@ -138,6 +138,9 @@ def to_python_value(cls, deserialized_data, *, target_object):
         else:
             data = {}
 
+        if deserialized_data is None and cls.DEFAULT is None:
+            return None
+
         for name, value in list(data.items()):
             if name not in cls.CONVERTERS:
                 log.debug(f'Removed unmapped nested file attribute: {name}')
@@ -171,19 +174,22 @@ def to_python_value(cls, deserialized_data, *, target_object):
     def to_preserialization_data(cls, python_value, *, default_to_skip=None):
         data = {}
 
+        if python_value is None and cls.DEFAULT is None:
+            return None
+
         for name, converter in cls.CONVERTERS.items():
 
             if isinstance(python_value, dict):
                 try:
                     value = python_value[name]
-                except KeyError as e:
-                    log.debug(e)
+                except KeyError:
+                    log.debug(f'Added missing nested attribute: {name}')
                     value = None
             else:
                 try:
                     value = getattr(python_value, name)
-                except AttributeError as e:
-                    log.debug(e)
+                except AttributeError:
+                    log.debug(f'Added missing nested attribute: {name}')
                     value = None
 
             with suppress(AttributeError):

datafiles/converters/enumerations.py

@@ -0,0 +1,23 @@
+# pylint: disable=unused-argument,not-callable
+
+from ._bases import Converter
+
+
+class Enumeration(Converter):
+
+    ENUM: type = NotImplemented
+
+    @classmethod
+    def of_type(cls, enum: type):
+        name = f'{enum.__name__}Converter'
+        bases = (cls,)
+        attributes = {'ENUM': enum}
+        return type(name, bases, attributes)
+
+    @classmethod
+    def to_python_value(cls, deserialized_data, *, target_object=None):
+        return cls.ENUM(deserialized_data)
+
+    @classmethod
+    def to_preserialization_data(cls, python_value, *, default_to_skip=None):
+        return python_value.value

datafiles/formats.py

@@ -9,7 +9,7 @@
 
 import log
 
-from . import settings
+from . import settings, types
 
 
 _REGISTRY: Dict[str, type] = {}
@@ -87,15 +87,24 @@ def deserialize(cls, file_object):
         from ruamel import yaml
 
         try:
-            return yaml.round_trip_load(file_object, preserve_quotes=True) or {}
+            data = yaml.round_trip_load(file_object, preserve_quotes=True)
         except NotImplementedError as e:
             log.error(str(e))
             return {}
+        else:
+            return data or {}
 
     @classmethod
     def serialize(cls, data):
         from ruamel import yaml
 
+        yaml.representer.RoundTripRepresenter.add_representer(
+            types.List, yaml.representer.RoundTripRepresenter.represent_list
+        )
+        yaml.representer.RoundTripRepresenter.add_representer(
+            types.Dict, yaml.representer.RoundTripRepresenter.represent_dict
+        )
+
         if settings.INDENT_YAML_BLOCKS:
             f = StringIO()
             y = yaml.YAML()
@@ -104,8 +113,11 @@ def serialize(cls, data):
             text = f.getvalue().strip() + '\n'
         else:
             text = yaml.round_trip_dump(data) or ""
-            text = text.replace('- \n', '-\n')
-        return "" if text == "{}\n" else text
+
+        if text == "{}\n":
+            return ""
+
+        return text.replace('- \n', '-\n')
 
 
 class PyYAML(Formatter):
@@ -144,19 +156,21 @@ def increase_indent(self, flow=False, indentless=False):
         return text
 
 
-def deserialize(path: Path, extension: str) -> Dict:
-    formatter = _get_formatter(extension)
+def deserialize(path: Path, extension: str, *, formatter=None) -> Dict:
+    if formatter is None:
+        formatter = _get_formatter(extension)
     with path.open('r') as file_object:
         return formatter.deserialize(file_object)
 
 
-def serialize(data: Dict, extension: str = '.yml') -> str:
-    formatter = _get_formatter(extension)
+def serialize(data: Dict, extension: str = '.yml', *, formatter=None) -> str:
+    if formatter is None:
+        formatter = _get_formatter(extension)
     return formatter.serialize(data)
 
 
 def _get_formatter(extension: str):
-    if settings.YAML_LIBRARY == 'PyYAML':
+    if settings.YAML_LIBRARY == 'PyYAML':  # pragma: no cover
         register('.yml', PyYAML)
 
     with suppress(KeyError):

datafiles/hooks.py

@@ -4,7 +4,7 @@
 
 import log
 
-from . import settings
+from . import settings, types
 from .mapper import create_mapper
 
 
@@ -27,14 +27,6 @@
 FLAG = '_patched'
 
 
-class List(list):
-    """Patchable `list` type."""
-
-
-class Dict(dict):
-    """Patchable `dict` type."""
-
-
 def apply(instance, mapper):
     """Path methods that get or set attributes."""
     cls = instance.__class__
@@ -56,10 +48,10 @@ def apply(instance, mapper):
         for attr_name in instance.datafile.attrs:
             attr = getattr(instance, attr_name)
             if isinstance(attr, list):
-                attr = List(attr)
+                attr = types.List(attr)
                 setattr(instance, attr_name, attr)
             elif isinstance(attr, dict):
-                attr = Dict(attr)
+                attr = types.Dict(attr)
                 setattr(instance, attr_name, attr)
             elif not is_dataclass(attr):
                 continue