WIP: Refactor images store

lago.spec.in

@@ -61,6 +61,7 @@ BuildRequires: python-scp
 BuildRequires: python2-rpm-macros
 BuildRequires: python-wrapt
 BuildRequires: python2-future
+BuildRequires: sqlite-3
 %if 0%{?fedora} >= 24
 BuildRequires: python2-configparser
 BuildRequires: python2-paramiko >= 2.1.1
@@ -88,6 +89,7 @@ Requires: libguestfs-tools >= 1.30
 Requires: libguestfs-devel >= 1.30
 Requires: libvirt >= 1.2.8
 Requires: libvirt-python
+Requires: sqlite-3
 Requires: python-libguestfs >= 1.30
 Requires: python-lxml
 Requires: python-lockfile

lago/db_utils.py

@@ -0,0 +1,37 @@
+from contextlib import contextmanager
+from collections import namedtuple
+from sqlalchemy.sql import func
+from sqlalchemy import (Column, DateTime, Integer)
+
+
+def autorepr(self):
+    cols = (str(col.key) for col in self.__table__.columns)
+    key_values = ('{0}="{1}"'.format(col, getattr(self, col)) for col in cols)
+    return '<{0}({1})>'.format(self.__class__.name, ','.join(key_values))
+
+
+@contextmanager
+def autocommit_safe(session):
+    try:
+        yield session
+        session.commit()
+    except:
+        session.rollback()
+        raise
+
+
+def namedtuple_serialize(self):
+    cols = ','.join([str(col.key) for col in self.__table__.columns])
+    record = namedtuple(self.__class__.__name__, cols)
+    return record._make(
+        [getattr(self, col.key) for col in self.__table__.columns]
+    )
+
+
+class BaseMixin(object):
+    id = Column(Integer, primary_key=True)
+    add_date = Column(DateTime, server_default=func.now())
+    __repr__ = autorepr
+
+    def serialize(self):
+        return namedtuple_serialize(self)

lago/prefix.py

@@ -891,32 +891,24 @@ def _create_link_to_parent(self, base, link_name):
     def _handle_lago_template(
         self, disk_path, template_spec, template_store, template_repo
     ):
-        disk_metadata = template_spec.get('metadata', {})
-        if template_store is None or template_repo is None:
-            raise RuntimeError('No templates directory provided')
-
-        template = template_repo.get_by_name(template_spec['template_name'])
-        template_version = template.get_version(
-            template_spec.get('template_version', None)
-        )
-        if template_version not in template_store:
-            LOGGER.info(
-                log_utils.log_always("Template %s not in cache, downloading") %
-                template_version.name,
-            )
-            template_store.download(template_version)
-
-        disk_metadata.update(
-            template_store.get_stored_metadata(
-                template_version,
-            ),
+        from templates import LagoImageProvider
+        from templates_store import ImagesStore
+        store = ImagesStore(root='/tmp/fancy_store_test')
+        provider = LagoImageProvider(
+            store=store,
+            config={
+                'url': 'http://templates.ovirt.org/repo/repo.metadata',
+                'name': 'us-lago-test'
+            }
         )
-        base = template_store.get_path(template_version)
+        image = provider.update(template_spec['template_name'])
+        disk_metadata = template_spec.get('metadata', {})
+        disk_metadata.update(store.get_metadata(image.hash))
         qemu_cmd = [
             'qemu-img', 'create', '-f', 'qcow2', '-o', 'lazy_refcounts=on',
-            '-b', base, disk_path
+            '-b', image.file, disk_path
         ]
-        return qemu_cmd, disk_metadata, base
+        return qemu_cmd, disk_metadata, image.file
 
     def _ova_to_spec(self, filename):
         """

lago/qemuimg.py

@@ -0,0 +1,16 @@
+from utils import run_command, run_command_with_validation
+
+
+def convert(src, dst, convert_format='raw'):
+    result = run_command_with_validation(
+        [
+            'qemu-img',
+            'convert',
+            '-O',
+            convert_format,
+            src,
+            dst,
+        ],
+        msg='qemu-img convert failed:'
+    )
+    return result

lago/templates.py

@@ -18,6 +18,7 @@
         having to change the template name everywhere
 
 """
+import qemuimg
 import errno
 import functools
 import json
@@ -27,15 +28,23 @@
 import shutil
 import urllib
 import sys
-
+from datetime import datetime
 import lockfile
-
+import tempfile
 import utils
 from . import log_utils
 from .config import config
-
+from collections import namedtuple
+from utils import LagoException
+from future.utils import raise_from
 LOGGER = logging.getLogger(__name__)
 
+ImageName = namedtuple('ImageName', 'name, hash')
+
+
+class LagoImageError(LagoException):
+    pass
+
 
 class FileSystemTemplateProvider:
     """
@@ -387,6 +396,11 @@ def get_by_name(self, name):
         Raises:
             KeyError: if no template is found
         """
+        if name not in self._dom['templates']:
+            raise LagoImageError(
+                'No image named {0} at {1}'.format(name, self.name)
+            )
+
         spec = self._dom.get('templates', {})[name]
         return Template(
             name=name,
@@ -421,6 +435,10 @@ def __init__(self, name, versions):
         self.name = name
         self._versions = versions
 
+    @property
+    def versions(self):
+        return self._versions
+
     def get_version(self, ver_name=None):
         """
         Get the given version for this template, or the latest
@@ -473,6 +491,12 @@ def __init__(self, name, source, handle, timestamp):
         self._hash = None
         self._metadata = None
 
+    def __repr__(self):
+        return (
+            '<TemplateVersion(name={0}, source={1}, handle={2}, '
+            'timestamp={3})>'
+        ).format(self.name, self._source, self._handle, self._timestamp)
+
     def timestamp(self):
         """
         Getter for the timestamp
@@ -515,17 +539,192 @@ def download(self, destination):
         self._source.download_image(self._handle, destination)
 
 
-def _locked(func):
-    """
-    Decorator that ensures that the decorated function has the lock of the
-    repo while running, meant to decorate only bound functions for classes that
-    have `lock_path` method.
-    """
+RemoteImage = namedtuple(
+    'RemoteImage', 'name, hash, creation_date,repo_name,tags,template_version'
+)
+
+
+class LagoImageProvider(object):
+    def __init__(self, config, store):
+        self._name = config['name']
+        self._url = config['url']
+        self.max_versions = config.get('max_versions', 5)
+        self._store = store
+        self._config = config
+        if not self._store.exists_repo(self.name):
+            store.add_repo(repo_name=self.name, repo_type='lago')
+
+    def update(self, raw_name, fail=False):
+        image_info = self._make_name(raw_name)
+        local_images = self.list_local_images(image_info)
+        remote_images = []
+        try:
+            remote_images = self.list_remote_images(image_info.name)
+        except LagoImageError:
+            if fail:
+                raise
+
+        if image_info.hash is None:
+            result = self._decide_by_name(
+                image_info.name, local_images, remote_images
+            )
+        else:
+            result = self._decide_by_hash(
+                image_info.hash, local_images, remote_images
+            )
+
+        if isinstance(result, RemoteImage):
+            image = self._add_from_remote(result)
+            if len(local_images) > self.max_versions:
+                LOGGER.debug(
+                    'more than %s images per name, deleting %s',
+                    self.max_versions, local_images[0].hash
+                )
+                self.store.delete_image(local_images[0].hash)
+        else:
+            image = result
+        return image
+
+    def list_local_images(self, image_info):
+        if image_info.hash is None:
+            return self._store.search(image_info.name, self.name)
+        else:
+            return [self._store.get_image(image_info.hash)]
+
+    def list_remote_images(self, name):
+        try:
+            remote_repo = TemplateRepository.from_url(self.url)
+        except RuntimeError as exc:
+            raise_from(
+                exc,
+                LagoImageError(
+                    'Unable to fetch Lago images '
+                    'repository from '
+                    '{0}'.format(self._url)
+                )
+            )
+        candidates = remote_repo.get_by_name(name)
+        remote_images = []
+        for ver_name, ver in candidates.versions.viewitems():
+            try:
+                sha1 = 'sha1:' + ver.get_metadata()['sha1']
+            except KeyError:
+                LOGGER.warning(
+                    (
+                        'Image without hash found at {0}, ignoring '
+                        'image: {1}'
+                    ).format(self.url, ver.name)
+                )
+                continue
+
+            remote_images.append(
+                RemoteImage(
+                    name=name,
+                    repo_name=self.name,
+                    hash=sha1,
+                    creation_date=datetime.fromtimestamp(ver.timestamp()),
+                    tags=[ver_name],
+                    template_version=ver
+                )
+            )
+            if remote_images != []:
+                remote_images.sort(key=lambda image: image.creation_date)
 
-    @functools.wraps(func)
-    def wrapper(self, *args, **kwargs):
-        with lockfile.LockFile(self.lock_path()):
-            return func(self, *args, **kwargs)
+        return remote_images
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def url(self):
+        return self._url
+
+    def _add_from_remote(self, remote_image):
+        tmp_dir = self._store.tmp_dir
+        _, tmp_dest = tempfile.mkstemp(dir=tmp_dir)
+        try:
+            remote_image.template_version.download(tmp_dest)
+            result = utils.verify_hash(
+                tmp_dest,
+                remote_image.hash.split(':')[-1],
+                hash_algo=remote_image.hash.split(':')[0]
+            )
+            if result is False:
+                raise LagoImageError(
+                    (
+                        'Failed verifying hash for image: '
+                        '{0}.'.format(remote_image)
+                    )
+                )
+
+            image = self._store.add_image(
+                name=remote_image.name,
+                repo_name=remote_image.repo_name,
+                hash=remote_image.hash,
+                image_file=tmp_dest,
+                creation_date=remote_image.creation_date,
+                metadata=remote_image.template_version.get_metadata(),
+                tags=remote_image.tags,
+                transfer_function=qemuimg.convert
+            )
+            return image
+        finally:
+            os.unlink(tmp_dest)
+
+    def _decide_by_hash(self, hash, local_images, remote_images):
+        raise LagoException('fetching by hash not implemented yet')
+
+    def _decide_by_name(self, name, local_images, remote_images):
+        if not local_images and not remote_images:
+            raise LagoImageError(
+                (
+                    'Unable to list remote images, and no '
+                    'local image {0} found.'
+                ).format(name)
+            )
+
+        elif local_images and not remote_images:
+            LOGGER.debug(
+                'no remote image was found with name %s, using local '
+                'image: %s', name, local_images[-1]
+            )
+            return local_images[-1]
+
+        elif not local_images and remote_images:
+            LOGGER.debug(
+                'no local image %s, acquiring remote: %s', name,
+                remote_images[-1]
+            )
+            return remote_images[-1]
+
+        else:
+            head_remote = remote_images[-1]
+            head_local = local_images[-1]
+            if head_remote.hash != head_local.hash and head_remote.creation_date > head_local.creation_date:
+                LOGGER.debug(
+                    (
+                        'found newer version for image name %s '
+                        'remote: %s, local: %s'
+                    ), name, head_remote, head_local
+                )
+                return head_remote
+            else:
+                return head_local
+
+    def _make_name(self, name):
+        components = name.split(':')
+        if len(components) == 1:
+            return ImageName(components[0], None)
+        elif len(components) == 2:
+            return ImageName(components[0], 'sha1:' + components[1])
+        else:
+            raise LagoImageError(
+                (
+                    'Illegal image name name, should be '
+                    'name[:SHA1]: {0}'.format(name)
+                )
+            )
 
 
 class TemplateStore: