Adds Debian osmorphing

coriolis/osmorphing/base.py

@@ -0,0 +1,26 @@
+import abc
+
+
+class BaseOSMorphingTools(object):
+    def __init__(self, os_root_dir):
+        self._os_root_dir = os_root_dir
+
+    @staticmethod
+    def check_os(ssh, os_root_dir):
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def set_dhcp(self, ssh):
+        pass
+
+    def get_packages(self, hypervisor, platform):
+        pass
+
+    def update_packages_list(self, ssh):
+        pass
+
+    def install_packages(self, ssh, package_names):
+        pass
+
+    def uninstall_packages(self, ssh, package_names):
+        pass

coriolis/osmorphing/debian.py

@@ -0,0 +1,65 @@
+import itertools
+import os
+import re
+
+from coriolis import constants
+from coriolis.osmorphing import base
+from coriolis import utils
+
+
+class DebianOSMorphingTools(base.BaseOSMorphingTools):
+    _packages = {
+        (constants.HYPERVISOR_VMWARE, None): [("open-vm-tools", True)],
+        # TODO: add cloud-initramfs-growroot
+        (None, constants.PLATFORM_OPENSTACK): [("cloud-init", True)],
+    }
+
+    @staticmethod
+    def check_os(ssh, os_root_dir):
+        lsb_release_path = os.path.join(os_root_dir, "etc/lsb-release")
+        debian_version_path = os.path.join(os_root_dir, "etc/debian_version")
+        if utils.test_ssh_path(ssh, lsb_release_path):
+            out = utils.exec_ssh_cmd(
+                ssh, "cat %s" % lsb_release_path).decode()
+            dist_id = re.findall('^DISTRIB_ID=(.*)$', out, re.MULTILINE)
+            release = re.findall('^DISTRIB_RELEASE=(.*)$', out, re.MULTILINE)
+            if 'Debian' in dist_id:
+                return (dist_id, release)
+        elif utils.test_ssh_path(ssh, debian_version_path):
+            release = utils.exec_ssh_cmd(
+                ssh, "cat %s" % debian_version_path).decode()
+            return ('Debian', release)
+
+    def set_dhcp(self, ssh):
+        interfaces_path = os.path.join(
+            self._os_root_dir, "etc/network/interfaces")
+        utils.exec_ssh_cmd(
+            ssh, 'sudo sed -i.bak "s/static/dhcp/g" %s' % interfaces_path)
+
+    def get_packages(self, hypervisor, platform):
+        k_add = [(h, p) for (h, p) in self._packages.keys() if
+                 (h is None or h == hypervisor) and
+                 (p is None or p == platform)]
+
+        add = [p[0] for p in itertools.chain.from_iterable(
+               [l for k, l in self._packages.items() if k in k_add])]
+
+        k_remove = set(self._packages.keys()) - set(k_add)
+        remove = [p[0] for p in itertools.chain.from_iterable(
+                  [l for k, l in self._packages.items() if k in k_remove])
+                  if p[1]]
+
+        return add, remove
+
+    def update_packages_list(self, ssh):
+        apt_get_cmd = 'apt-get update -y'
+        utils.exec_ssh_cmd_chroot(ssh, self._os_root_dir, apt_get_cmd)
+
+    def install_packages(self, ssh, package_names):
+        apt_get_cmd = 'apt-get install %s -y' % " ".join(package_names)
+        utils.exec_ssh_cmd_chroot(ssh, self._os_root_dir, apt_get_cmd)
+
+    def uninstall_packages(self, ssh, package_names):
+        for package_name in package_names:
+            apt_get_cmd = 'apt-get remove %s -y || true' % package_name
+            utils.exec_ssh_cmd_chroot(ssh, self._os_root_dir, apt_get_cmd)

coriolis/osmorphing/factory.py

@@ -1,10 +1,13 @@
+from coriolis.osmorphing import debian
 from coriolis.osmorphing import ubuntu
 
 
 def get_os_morphing_tools(ssh, os_root_dir):
-    os_morphing_tools_clss = [ubuntu.UbuntuOSMorphingTools]
+    os_morphing_tools_clss = [debian.DebianOSMorphingTools,
+                              ubuntu.UbuntuOSMorphingTools]
 
     for cls in os_morphing_tools_clss:
-        if cls.check_os(ssh, os_root_dir):
-            return cls(os_root_dir)
+        os_info = cls.check_os(ssh, os_root_dir)
+        if os_info:
+            return cls(os_root_dir), os_info
     raise Exception("Cannot find the morphing tools for this OS image")

coriolis/osmorphing/manager.py

@@ -22,9 +22,11 @@ def morph_image(connection_info, target_hypervisor, target_platform,
 
     os_mount_tools = osmount_factory.get_os_mount_tools(ssh)
     os_root_dir = os_mount_tools.mount_os(ssh, volume_devs)
-    os_morphing_tools = osmorphing_factory.get_os_morphing_tools(
+    os_morphing_tools, os_info = osmorphing_factory.get_os_morphing_tools(
         ssh, os_root_dir)
 
+    LOG.info('OS being migrated: %s', str(os_info))
+
     os_morphing_tools.set_dhcp(ssh)
 
     (packages_add,

coriolis/osmorphing/ubuntu.py

@@ -1,65 +1,29 @@
-import itertools
 import os
 import re
 
 from coriolis import constants
+from coriolis.osmorphing import debian
 from coriolis import utils
 
 
-class UbuntuOSMorphingTools(object):
+class UbuntuOSMorphingTools(debian.DebianOSMorphingTools):
     _packages = {
         (constants.HYPERVISOR_VMWARE, None): [("open-vm-tools", True)],
         # TODO: sudo agt-get install linux-tool-<kernel release>
         # linux-cloud-tools-<kernel release> -y
         (constants.HYPERVISOR_HYPERV, None): [("hv-kvp-daemon-init", True)],
+        # TODO: add cloud-initramfs-growroot
         (None, constants.PLATFORM_OPENSTACK): [("cloud-init", True)],
     }
 
-    def __init__(self, os_root_dir):
-        self._os_root_dir = os_root_dir
-
     @staticmethod
     def check_os(ssh, os_root_dir):
         lsb_release_path = os.path.join(os_root_dir, "etc/lsb-release")
-        out = utils.exec_ssh_cmd(
-            ssh, "cat %s || true" % lsb_release_path).decode()
-
-        dist_id = re.findall('^DISTRIB_ID=(.*)$', out, re.MULTILINE)
-        release = re.findall('^DISTRIB_RELEASE=(.*)$', out, re.MULTILINE)
-        # TODO: validate release as well
-        if 'Ubuntu' in dist_id:
-            return True
-
-    def set_dhcp(self, ssh):
-        interfaces_path = os.path.join(
-            self._os_root_dir, "etc/network/interfaces")
-        utils.exec_ssh_cmd(
-            ssh, 'sudo sed -i.bak "s/static/dhcp/g" %s' % interfaces_path)
-
-    def get_packages(self, hypervisor, platform):
-        k_add = [(h, p) for (h, p) in self._packages.keys() if
-                 (h is None or h == hypervisor) and
-                 (p is None or p == platform)]
-
-        add = [p[0] for p in itertools.chain.from_iterable(
-               [l for k, l in self._packages.items() if k in k_add])]
-
-        k_remove = set(self._packages.keys()) - set(k_add)
-        remove = [p[0] for p in itertools.chain.from_iterable(
-                  [l for k, l in self._packages.items() if k in k_remove])
-                  if p[1]]
-
-        return add, remove
-
-    def update_packages_list(self, ssh):
-        apt_get_cmd = 'sudo apt-get update -y'
-        utils.exec_ssh_cmd_chroot(ssh, self._os_root_dir, apt_get_cmd)
-
-    def install_packages(self, ssh, package_names):
-        apt_get_cmd = 'sudo apt-get install %s -y' % " ".join(package_names)
-        utils.exec_ssh_cmd_chroot(ssh, self._os_root_dir, apt_get_cmd)
-
-    def uninstall_packages(self, ssh, package_names):
-        for package_name in package_names:
-            apt_get_cmd = 'sudo apt-get remove %s -y || true' % package_name
-            utils.exec_ssh_cmd_chroot(ssh, self._os_root_dir, apt_get_cmd)
+        if utils.test_ssh_path(ssh, lsb_release_path):
+            out = utils.exec_ssh_cmd(
+                ssh, "cat %s" % lsb_release_path).decode()
+
+            dist_id = re.findall('^DISTRIB_ID=(.*)$', out, re.MULTILINE)
+            release = re.findall('^DISTRIB_RELEASE=(.*)$', out, re.MULTILINE)
+            if 'Ubuntu' in dist_id:
+                return (dist_id, release)

coriolis/providers/openstack/__init__.py

@@ -1,5 +1,4 @@
 import math
-import os
 import time
 import uuid
 
@@ -238,6 +237,10 @@ class ImportProvider(base.BaseExportProvider):
                 to_port=SSH_PORT)
             instance.add_security_group(sec_group.id)
 
+            LOG.info("Waiting for connectivity on host: %s:%s",
+                     (floating_ip.ip, SSH_PORT))
+            utils.wait_for_port_connectivity(floating_ip.ip, SSH_PORT)
+
             return _MigrationResources(nova, neutron, keypair, instance, port,
                                        floating_ip, sec_group, k)
         except:

coriolis/providers/vmware_vsphere/__init__.py

@@ -29,6 +29,7 @@ class ExportProvider(base.BaseExportProvider):
     def validate_connection_info(self, connection_info):
         return True
 
+    @utils.retry_on_error()
     def _convert_disk_type(self, disk_path, target_disk_path, target_type=0):
         utils.exec_process([CONF.vmware_vsphere.vdiskmanager_path, "-r",
                             disk_path, "-t", str(target_type),
@@ -41,30 +42,26 @@ class ExportProvider(base.BaseExportProvider):
         if task.info.state == vim.TaskInfo.State.error:
             raise Exception(task.info.error.msg)
 
-    def export_instance(self, connection_info, instance_name, export_path):
-        host = connection_info["host"]
-        port = connection_info.get("port", 443)
-        username = connection_info["username"]
-        password = connection_info["password"]
-        allow_untrusted = connection_info.get("allow_untrusted", False)
-
-        # pyVmomi locks otherwise
-        sys.modules['socket'] = eventlet.patcher.original('socket')
-        ssl = eventlet.patcher.original('ssl')
-
-        context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
-        if allow_untrusted:
-            context.verify_mode = ssl.CERT_NONE
-
+    @utils.retry_on_error()
+    def _connect(self, host, username, password, port, context):
         LOG.info("Connecting to: %s:%s" % (host, port))
-
-        si = connect.SmartConnect(
+        return connect.SmartConnect(
             host=host,
             user=username,
             pwd=password,
             port=port,
             sslContext=context)
 
+    def _wait_for_vm_status(self, vm, status, max_wait=120):
+        i = 0
+        while i < max_wait and vm.runtime.powerState != status:
+            time.sleep(1)
+            i += 1
+        return i < max_wait
+
+    @utils.retry_on_error()
+    def _get_vm_info(self, si, instance_name):
+
         LOG.info("Retrieving data for VM: %s" % instance_name)
 
         # TODO: provide path selection
@@ -92,10 +89,15 @@ class ExportProvider(base.BaseExportProvider):
         LOG.info("vm info: %s" % str(vm_info))
 
         if vm.runtime.powerState != vim.VirtualMachinePowerState.poweredOff:
+            power_off = True
             if (vm.guest.toolsRunningStatus !=
                     vim.vm.GuestInfo.ToolsRunningStatus.guestToolsNotRunning):
                 vm.ShutdownGuest()
-            else:
+                if self._wait_for_vm_status(
+                        vm, vim.VirtualMachinePowerState.poweredOff):
+                    power_off = False
+
+            if power_off:
                 task = vm.PowerOff()
                 self._wait_for_task(task)
 
@@ -172,6 +174,19 @@ class ExportProvider(base.BaseExportProvider):
                             vim.vm.BootOptions.BootableFloppyDevice):
                 boot_order.append({"type": "floppy", "id": None})
 
+        vm_info["devices"] = {
+            "nics": nics,
+            "controllers": disk_ctrls,
+            "disks": disks,
+            "cdroms": cdroms,
+            "floppy": floppy,
+        }
+        vm_info["boot_order"] = boot_order
+
+        return vm_info, vm
+
+    @utils.retry_on_error()
+    def _export_disks(self, vm, export_path, context):
         disk_paths = []
         lease = vm.ExportVm()
         while True:
@@ -217,7 +232,7 @@ class ExportProvider(base.BaseExportProvider):
                                          1024)))
 
                     lease.HttpNfcLeaseComplete()
-                    break
+                    return disk_paths
                 except:
                     lease.HttpNfcLeaseAbort()
                     raise
@@ -226,7 +241,27 @@ class ExportProvider(base.BaseExportProvider):
             else:
                 time.sleep(.1)
 
-        connect.Disconnect(si)
+    def export_instance(self, connection_info, instance_name, export_path):
+        host = connection_info["host"]
+        port = connection_info.get("port", 443)
+        username = connection_info["username"]
+        password = connection_info["password"]
+        allow_untrusted = connection_info.get("allow_untrusted", False)
+
+        # pyVmomi locks otherwise
+        sys.modules['socket'] = eventlet.patcher.original('socket')
+        ssl = eventlet.patcher.original('ssl')
+
+        context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        if allow_untrusted:
+            context.verify_mode = ssl.CERT_NONE
+
+        si = self._connect(host, username, password, port, context)
+        try:
+            vm_info, vm = self._get_vm_info(si, instance_name)
+            disk_paths = self._export_disks(vm, export_path, context)
+        finally:
+            connect.Disconnect(si)
 
         for disk_path in disk_paths:
             path = disk_path["path"]
@@ -235,17 +270,10 @@ class ExportProvider(base.BaseExportProvider):
             self._convert_disk_type(path, tmp_path)
             os.remove(path)
             os.rename(tmp_path, path)
+
+            disks = vm_info["devices"]["disks"]
             disk_info = [d for d in disks if d["id"] == disk_path["id"]][0]
             disk_info["path"] = os.path.abspath(path)
             disk_info["format"] = constants.DISK_FORMAT_VMDK
 
-        vm_info["devices"] = {
-            "nics": nics,
-            "controllers": disk_ctrls,
-            "disks": disks,
-            "cdroms": cdroms,
-            "floppy": floppy,
-        }
-        vm_info["boot_order"] = boot_order
-
         return vm_info

coriolis/utils.py

@@ -23,7 +23,7 @@ CONF.register_opts(opts)
 LOG = logging.getLogger(__name__)
 
 
-def retry_on_error(max_attempts=5, sleep_seconds=1):
+def retry_on_error(max_attempts=5, sleep_seconds=0):
     def _retry_on_error(func):
         @functools.wraps(func)
         def _exec_retry(*args, **kwargs):
@@ -50,6 +50,14 @@ def get_linux_os_info(ssh):
         return (dist_id[0], release[0])
 
 
+def test_ssh_path(ssh, remote_path):
+    try:
+        exec_ssh_cmd(ssh, "test -f %s" % remote_path)
+        return True
+    except Exception:
+        return False
+
+
 @retry_on_error()
 def exec_ssh_cmd(ssh, cmd):
     stdin, stdout, stderr = ssh.exec_command(cmd)
@@ -73,20 +81,18 @@ def _check_port_open(host, port):
         s.settimeout(1)
         s.connect((host, port))
         return True
-    except ConnectionRefusedError:
-        return False
-    except socket.timeout:
+    except (ConnectionRefusedError, socket.timeout, OSError):
         return False
     finally:
         s.close()
 
 
-def wait_for_port_connectivity(address, port):
+def wait_for_port_connectivity(address, port, max_wait=300):
     i = 0
-    while not _check_port_open(address, port) and i < 120:
+    while not _check_port_open(address, port) and i < max_wait:
         time.sleep(1)
         i += 1
-    if i == 120:
+    if i == max_wait:
         raise Exception("Connection failed on port %s" % port)