Migrate yaml parsing to ruamel.yaml

Migrate yaml parsing code to ruamel.yaml.
Update spec file and setup.py accordingly.
Remove a bunch of old hacks, adjust tests.

fmf.spec

@@ -28,7 +28,7 @@ Summary:        %{summary}
 BuildRequires: python%{python3_pkgversion}-devel
 BuildRequires: python%{python3_pkgversion}-setuptools
 BuildRequires: python%{python3_pkgversion}-pytest
-BuildRequires: python%{python3_pkgversion}-PyYAML
+BuildRequires: python%{python3_pkgversion}-ruamel-yaml
 BuildRequires: python%{python3_pkgversion}-filelock
 BuildRequires: git-core
 %{?python_provide:%python_provide python%{python3_pkgversion}-%{name}}

fmf/base.py

@@ -7,9 +7,9 @@
 from io import open
 from pprint import pformat as pretty
 
-import yaml
-import yaml.constructor
-import yaml.resolver
+from ruamel.yaml import YAML
+from ruamel.yaml.constructor import DuplicateKeyError
+from ruamel.yaml.error import YAMLError
 
 import fmf.context
 import fmf.utils as utils
@@ -23,44 +23,6 @@
 MAIN = "main" + SUFFIX
 IGNORED_DIRECTORIES = ['/dev', '/proc', '/sys']
 
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-#  YAML
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-# Handle both older and newer yaml loader
-# https://msg.pyyaml.org/load
-try:
-    from yaml import FullLoader as YamlLoader
-except ImportError:  # pragma: no cover
-    from yaml import SafeLoader as YamlLoader
-
-
-# Load all strings from YAML files as unicode
-# https://stackoverflow.com/questions/2890146/
-def construct_yaml_str(self, node):
-    return self.construct_scalar(node)
-
-
-# Raise an exception on duplicate keys
-# https://gist.github.com/pypt/94d747fe5180851196eb
-def unique_key_constructor(loader, node, deep=False):
-    """ YAML constructor that checks for duplicate keys """
-    mapping = {}
-    for key_node, value_node in node.value:
-        key = loader.construct_object(key_node, deep=deep)
-        value = loader.construct_object(value_node, deep=deep)
-        if key in mapping:
-            raise yaml.constructor.ConstructorError(
-                "Duplicate key '{}' detected.".format(key))
-        mapping[key] = value
-    return loader.construct_mapping(node, deep)
-
-
-YamlLoader.add_constructor(
-    yaml.resolver.BaseResolver.DEFAULT_SCALAR_TAG, construct_yaml_str)
-YamlLoader.add_constructor(
-    yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, unique_key_constructor)
-
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #  Metadata
@@ -465,10 +427,10 @@ def grow(self, path):
             log.info("Checking file {0}".format(fullpath))
             try:
                 with open(fullpath, encoding='utf-8') as datafile:
-                    data = yaml.load(datafile, Loader=YamlLoader)
-            except yaml.error.YAMLError as error:
-                raise(utils.FileError("Failed to parse '{0}'.\n{1}".format(
-                    fullpath, error)))
+                    data = YAML(typ="safe").load(datafile)
+            except (YAMLError, DuplicateKeyError) as error:
+                raise(utils.FileError(
+                    f"Failed to parse '{fullpath}'.\n{error}"))
             log.data(pretty(data))
             # Handle main.fmf as data for self
             if filename == MAIN:

fmf/utils.py

@@ -12,8 +12,9 @@
 from io import StringIO
 from pprint import pformat as pretty
 
-import yaml
 from filelock import FileLock, Timeout
+from ruamel.yaml import YAML, scalarstring
+from ruamel.yaml.comments import CommentedMap
 
 import fmf.base
 
@@ -782,37 +783,28 @@ def run(command, cwd=None, check_exit_code=True, env=None):
 #  Convert dict to yaml
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-# Special hack to store multiline text with the '|' style
-# See https://stackoverflow.com/questions/45004464/
-yaml.SafeDumper.orig_represent_str = yaml.SafeDumper.represent_str
-
-
-def repr_str(dumper, data):
-    if '\n' in data:
-        return dumper.represent_scalar(
-            u'tag:yaml.org,2002:str', data, style='|')
-    return dumper.orig_represent_str(data)
+def dict_to_yaml(data, width=None, sort=False):
+    """ Convert dictionary into yaml """
+    output = StringIO()
 
+    # Set formatting options
+    yaml = YAML()
+    yaml.indent(mapping=4, sequence=4, offset=2)
+    yaml.default_flow_style = False
+    yaml.allow_unicode = True
+    yaml.encoding = 'utf-8'
+    yaml.width = width
 
-yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)
+    # Make sure that multiline strings keep the formatting
+    data = copy.deepcopy(data)
+    scalarstring.walk_tree(data)
 
+    # Sort the data https://stackoverflow.com/a/40227545
+    if sort:
+        sorted_data = CommentedMap()
+        for key in sorted(data):
+            sorted_data[key] = data[key]
+        data = sorted_data
 
-def dict_to_yaml(data, width=None, sort=False):
-    """ Convert dictionary into yaml """
-    output = StringIO()
-    try:
-        yaml.safe_dump(
-            data, output, sort_keys=sort,
-            encoding='utf-8', allow_unicode=True,
-            width=width, indent=4, default_flow_style=False)
-    except TypeError:  # pragma: no cover
-        # FIXME: Temporary workaround for rhel-8 to disable key sorting
-        # https://stackoverflow.com/questions/31605131/
-        # https://github.com/psss/tmt/issues/207
-        def representer(self, data): return self.represent_mapping(
-            'tag:yaml.org,2002:map', data.items())
-        yaml.add_representer(dict, representer, Dumper=yaml.SafeDumper)
-        yaml.safe_dump(
-            data, output, encoding='utf-8', allow_unicode=True,
-            width=width, indent=4, default_flow_style=False)
+    yaml.dump(data, output)
     return output.getvalue()

setup.py

@@ -21,7 +21,7 @@
 
 # Prepare install requires and extra requires
 install_requires = [
-    'PyYAML',
+    'ruamel.yaml',
     'filelock'
     ]
 extras_require = {

tests/unit/test_adjust.py

@@ -1,7 +1,7 @@
 import copy
 
 import pytest
-import yaml
+from ruamel.yaml import YAML
 
 import fmf
 from fmf.context import CannotDecide, Context
@@ -29,7 +29,8 @@ def mini():
             enabled: false
             when: distro = centos
         """
-    return fmf.Tree(yaml.safe_load(data))
+    yaml = YAML(typ="safe")
+    return fmf.Tree(yaml.load(data))
 
 
 @pytest.fixture
@@ -66,7 +67,8 @@ def full():
               - require+: [three]
                 when: distro = fedora
         """
-    return fmf.Tree(yaml.safe_load(data))
+    yaml = YAML(typ="safe")
+    return fmf.Tree(yaml.load(data))
 
 
 class TestInvalid:

tests/unit/test_utils.py

@@ -6,6 +6,7 @@
 
 import pytest
 
+import fmf
 import fmf.utils as utils
 from fmf.utils import filter, listed, run
 
@@ -415,3 +416,13 @@ def test_force_cache_fetch(self, monkeypatch, tmpdir):
         assert os.path.isfile(fetch_head)
         utils.invalidate_cache()
         assert not os.path.isfile(fetch_head)
+
+
+class TestDictToYaml:
+    """ Verify dictionary to yaml format conversion """
+
+    def test_sort(self):
+        """ Verify key sorting """
+        data = dict(y=2, x=1)
+        assert fmf.utils.dict_to_yaml(data) == "y: 2\nx: 1\n"
+        assert fmf.utils.dict_to_yaml(data, sort=True) == "x: 1\ny: 2\n"