Added QCOW differential image support #366 (#520)

config/dpkg/changelog

@@ -1,5 +1,5 @@
-dfvfs (20201202-1) unstable; urgency=low
+dfvfs (20201219-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Log2Timeline maintainers <[email protected]>  Wed, 02 Dec 2020 08:10:45 +0100
+ -- Log2Timeline maintainers <[email protected]>  Sat, 19 Dec 2020 08:46:12 +0100

config/dpkg/control

@@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/dfvfs
 
 Package: python3-dfvfs
 Architecture: all
-Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20200819), libfshfs-python3 (>= 20201103), libfsntfs-python3 (>= 20200921), libfsxfs-python3 (>= 20201114), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libluksde-python3 (>= 20200101), libqcow-python3 (>= 20131204), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20200809), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends}
+Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20200819), libfshfs-python3 (>= 20201103), libfsntfs-python3 (>= 20200921), libfsxfs-python3 (>= 20201114), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libluksde-python3 (>= 20200101), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20200809), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends}
 Description: Python 3 module of dfVFS
  dfVFS, or Digital Forensics Virtual File System, provides read-only access to
  file-system objects from various storage media types and file formats. The goal

config/linux/gift_copr_install.sh

@@ -81,18 +81,18 @@ DEBUG_DEPENDENCIES="libbde-debuginfo
                     libvslvm-debuginfo
                     libvslvm-python3-debuginfo";
 
-sudo dnf install dnf-plugins-core
-sudo dnf copr -y enable @gift/dev
-sudo dnf install -y ${PYTHON3_DEPENDENCIES}
+sudo dnf install -q dnf-plugins-core
+sudo dnf copr -q -y enable @gift/dev
+sudo dnf install -q -y ${PYTHON3_DEPENDENCIES}
 
 if [[ "$*" =~ "include-debug" ]]; then
-    sudo dnf install -y ${DEBUG_DEPENDENCIES}
+    sudo dnf install -q -y ${DEBUG_DEPENDENCIES}
 fi
 
 if [[ "$*" =~ "include-development" ]]; then
-    sudo dnf install -y ${DEVELOPMENT_DEPENDENCIES}
+    sudo dnf install -q -y ${DEVELOPMENT_DEPENDENCIES}
 fi
 
 if [[ "$*" =~ "include-test" ]]; then
-    sudo dnf install -y ${TEST_DEPENDENCIES}
+    sudo dnf install -q -y ${TEST_DEPENDENCIES}
 fi

config/linux/ubuntu_install_dfvfs.sh

@@ -91,19 +91,19 @@ DEBUG_DEPENDENCIES="libbde-dbg
 
 sudo add-apt-repository ppa:gift/${GIFT_PPA_TRACK} -y
 sudo apt-get update -q
-sudo apt-get install -y ${PYTHON_DEPENDENCIES}
+sudo apt-get install -q -y ${PYTHON_DEPENDENCIES}
 
 if [[ "$*" =~ "include-debug" ]];
 then
-	sudo apt-get install -y ${DEBUG_DEPENDENCIES}
+	sudo apt-get install -q -y ${DEBUG_DEPENDENCIES}
 fi
 
 if [[ "$*" =~ "include-development" ]];
 then
-	sudo apt-get install -y ${DEVELOPMENT_DEPENDENCIES}
+	sudo apt-get install -q -y ${DEVELOPMENT_DEPENDENCIES}
 fi
 
 if [[ "$*" =~ "include-test" ]];
 then
-	sudo apt-get install -y ${TEST_DEPENDENCIES}
+	sudo apt-get install -q -y ${TEST_DEPENDENCIES}
 fi

dependencies.ini

@@ -112,7 +112,7 @@ version_property: get_version()
 [pyqcow]
 dpkg_name: libqcow-python3
 l2tbinaries_name: libqcow
-minimum_version: 20131204
+minimum_version: 20201213
 pypi_name: libqcow-python
 rpm_name: libqcow-python3
 version_property: get_version()

dfvfs/__init__.py

@@ -6,4 +6,4 @@
 storage media types and file formats.
 """
 
-__version__ = '20201202'
+__version__ = '20201219'

dfvfs/file_io/qcow_file_io.py

@@ -7,12 +7,37 @@
 
 from dfvfs.file_io import file_object_io
 from dfvfs.lib import errors
+from dfvfs.path import factory as path_spec_factory
 from dfvfs.resolver import resolver
 
 
 class QCOWFile(file_object_io.FileObjectIO):
   """File-like object using pyqcow."""
 
+  def __init__(self, resolver_context, file_object=None):
+    """Initializes a file-like object.
+
+    Args:
+      resolver_context (Context): resolver context.
+      file_object (Optional[FileIO]): file-like object.
+    """
+    super(QCOWFile, self).__init__(resolver_context, file_object=file_object)
+    self._parent_qcow_files = []
+    self._sub_file_objects = []
+
+  def _Close(self):
+    """Closes the file-like object."""
+    super(QCOWFile, self)._Close()
+
+    for qcow_file in self._parent_qcow_files:
+      qcow_file.close()
+
+    for file_object in self._sub_file_objects:
+      file_object.close()
+
+    self._parent_qcow_files = []
+    self._sub_file_objects = []
+
   def _OpenFileObject(self, path_spec):
     """Opens the file-like object defined by path specification.
 
@@ -33,8 +58,70 @@ def _OpenFileObject(self, path_spec):
         path_spec.parent, resolver_context=self._resolver_context)
     qcow_file = pyqcow.file()
     qcow_file.open_file_object(file_object)
+
+    if qcow_file.backing_filename:  # pylint: disable=using-constant-test
+      file_system = resolver.Resolver.OpenFileSystem(
+          path_spec.parent, resolver_context=self._resolver_context)
+
+      try:
+        self._OpenParentFile(file_system, path_spec.parent, qcow_file)
+      finally:
+        file_system.Close()
+
     return qcow_file
 
+  def _OpenParentFile(self, file_system, path_spec, qcow_file):
+    """Opens the parent file.
+
+    Args:
+      file_system (FileSystem): file system of the QCOW file.
+      path_spec (PathSpec): path specification of the QCOW file.
+      qcow_file (pyqcow.file): QCOW file.
+
+    Raises:
+      PathSpecError: if the path specification is incorrect.
+    """
+    location = getattr(path_spec, 'location', None)
+    if not location:
+      raise errors.PathSpecError(
+          'Unsupported path specification without location.')
+
+    location_path_segments = file_system.SplitPath(location)
+
+    location_path_segments.pop()
+    location_path_segments.append(qcow_file.backing_filename)
+    parent_file_location = file_system.JoinPath(location_path_segments)
+
+    # Note that we don't want to set the keyword arguments when not used
+    # because the path specification base class will check for unused
+    # keyword arguments and raise.
+    kwargs = path_spec_factory.Factory.GetProperties(path_spec)
+
+    kwargs['location'] = parent_file_location
+    if path_spec.parent is not None:
+      kwargs['parent'] = path_spec.parent
+
+    parent_file_path_spec = path_spec_factory.Factory.NewPathSpec(
+        path_spec.type_indicator, **kwargs)
+
+    if not file_system.FileEntryExistsByPathSpec(parent_file_path_spec):
+      return
+
+    file_object = resolver.Resolver.OpenFileObject(
+        parent_file_path_spec, resolver_context=self._resolver_context)
+
+    qcow_parent_file = pyqcow.file()
+    qcow_parent_file.open_file_object(file_object)
+
+    if qcow_parent_file.backing_filename:  # pylint: disable=using-constant-test
+      self._OpenParentFile(
+          file_system, parent_file_path_spec, qcow_parent_file)
+
+    qcow_file.set_parent(qcow_parent_file)
+
+    self._parent_qcow_files.append(qcow_parent_file)
+    self._sub_file_objects.append(file_object)
+
   def get_size(self):
     """Retrieves the size of the file-like object.
 

docs/conf.py

@@ -42,7 +42,8 @@
 pip_installed_modules = set(['six'])
 
 dependency_helper = utils.dependencies.DependencyHelper(
-    configuration_file=os.path.join('..', 'dependencies.ini'))
+    dependencies_file=os.path.join('..', 'dependencies.ini'),
+    test_dependencies_file=os.path.join('..', 'test_dependencies.ini'))
 modules_to_mock = set(dependency_helper.dependencies.keys())
 modules_to_mock = modules_to_mock.difference(pip_installed_modules)
 

requirements.txt

@@ -15,7 +15,7 @@ libfsxfs-python >= 20201114
 libfvde-python >= 20160719
 libfwnt-python >= 20160418
 libluksde-python >= 20200101
-libqcow-python >= 20131204
+libqcow-python >= 20201213
 libsigscan-python >= 20191221
 libsmdev-python >= 20140529
 libsmraw-python >= 20140612

setup.cfg

@@ -27,7 +27,7 @@ requires = libbde-python3 >= 20140531
            libfvde-python3 >= 20160719
            libfwnt-python3 >= 20160418
            libluksde-python3 >= 20200101
-           libqcow-python3 >= 20131204
+           libqcow-python3 >= 20201213
            libsigscan-python3 >= 20191221
            libsmdev-python3 >= 20140529
            libsmraw-python3 >= 20140612

utils/dependencies.py

@@ -5,6 +5,7 @@
 from __future__ import unicode_literals
 
 import configparser
+import os
 import re
 
 
@@ -14,8 +15,6 @@ class DependencyDefinition(object):
   Attributes:
     dpkg_name (str): name of the dpkg package that provides the dependency.
     is_optional (bool): True if the dependency is optional.
-    l2tbinaries_macos_name (str): name of the l2tbinaries macos package that
-        provides the dependency.
     l2tbinaries_name (str): name of the l2tbinaries package that provides
         the dependency.
     maximum_version (str): maximum supported version, a greater or equal
@@ -41,7 +40,6 @@ def __init__(self, name):
     super(DependencyDefinition, self).__init__()
     self.dpkg_name = None
     self.is_optional = False
-    self.l2tbinaries_macos_name = None
     self.l2tbinaries_name = None
     self.maximum_version = None
     self.minimum_version = None
@@ -60,7 +58,6 @@ class DependencyDefinitionReader(object):
   _VALUE_NAMES = frozenset([
       'dpkg_name',
       'is_optional',
-      'l2tbinaries_macos_name',
       'l2tbinaries_name',
       'maximum_version',
       'minimum_version',
@@ -118,11 +115,15 @@ class DependencyHelper(object):
   _VERSION_NUMBERS_REGEX = re.compile(r'[0-9.]+')
   _VERSION_SPLIT_REGEX = re.compile(r'\.|\-')
 
-  def __init__(self, configuration_file='dependencies.ini'):
+  def __init__(
+      self, dependencies_file='dependencies.ini',
+      test_dependencies_file='test_dependencies.ini'):
     """Initializes a dependency helper.
 
     Args:
-      configuration_file (Optional[str]): path to the dependencies
+      dependencies_file (Optional[str]): path to the dependencies configuration
+          file.
+      test_dependencies_file (Optional[str]): path to the test dependencies
           configuration file.
     """
     super(DependencyHelper, self).__init__()
@@ -131,14 +132,14 @@ def __init__(self, configuration_file='dependencies.ini'):
 
     dependency_reader = DependencyDefinitionReader()
 
-    with open(configuration_file, 'r') as file_object:
+    with open(dependencies_file, 'r') as file_object:
       for dependency in dependency_reader.Read(file_object):
         self.dependencies[dependency.name] = dependency
 
-    dependency = DependencyDefinition('mock')
-    dependency.minimum_version = '0.7.1'
-    dependency.version_property = '__version__'
-    self._test_dependencies['mock'] = dependency
+    if os.path.exists(test_dependencies_file):
+      with open(test_dependencies_file, 'r') as file_object:
+        for dependency in dependency_reader.Read(file_object):
+          self._test_dependencies[dependency.name] = dependency
 
   def _CheckPythonModule(self, dependency):
     """Checks the availability of a Python module.