RHEL/CentOS/Fedora support improvements

coriolis/osmorphing/base.py

@@ -1,26 +1,75 @@
 import abc
+import itertools
+import os
+
+from coriolis import utils
 
 
 class BaseOSMorphingTools(object):
-    def __init__(self, os_root_dir):
+    _packages = {}
+
+    def __init__(self, ssh, os_root_dir, hypervisor, platform):
+        self._ssh = ssh
         self._os_root_dir = os_root_dir
+        self._hypervisor = hypervisor
+        self._platform = platform
+
+    def _test_path(self, chroot_path):
+        path = os.path.join(self._os_root_dir, chroot_path)
+        return utils.test_ssh_path(self._ssh, path)
+
+    def _read_file(self, chroot_path):
+        path = os.path.join(self._os_root_dir, chroot_path)
+        return utils.read_ssh_file(self._ssh, path)
+
+    def _write_file(self, chroot_path, content):
+        path = os.path.join(self._os_root_dir, chroot_path)
+        utils.write_ssh_file(self._ssh, path, content)
+
+    def _exec_cmd(self, cmd):
+        return utils.exec_ssh_cmd(self._ssh, cmd)
+
+    def _exec_cmd_chroot(self, cmd):
+        return utils.exec_ssh_cmd_chroot(self._ssh, self._os_root_dir, cmd)
+
+    def _check_user_exists(self, username):
+        try:
+            self._exec_cmd_chroot("id -u %s" % username)
+            return True
+        except:
+            return False
+
+    @abc.abstractmethod
+    def check_os(self):
+        pass
+
+    def get_packages(self):
+        k_add = [(h, p) for (h, p) in self._packages.keys() if
+                 (h is None or h == self._hypervisor) and
+                 (p is None or p == self._platform)]
+
+        add = [p[0] for p in itertools.chain.from_iterable(
+               [l for k, l in self._packages.items() if k in k_add])]
+
+        k_remove = set(self._packages.keys()) - set(k_add)
+        remove = [p[0] for p in itertools.chain.from_iterable(
+                  [l for k, l in self._packages.items() if k in k_remove])
+                  if p[1]]
 
-    @staticmethod
-    def check_os(ssh, os_root_dir):
-        raise NotImplementedError()
+        return add, remove
 
     @abc.abstractmethod
-    def set_dhcp(self, ssh):
+    def set_dhcp(self):
         pass
 
-    def get_packages(self, hypervisor, platform):
+    def pre_packages_install(self):
         pass
 
-    def update_packages_list(self, ssh):
+    def install_packages(self, package_names):
         pass
 
-    def install_packages(self, ssh, package_names):
+    def uninstall_packages(self, package_names):
         pass
 
-    def uninstall_packages(self, ssh, package_names):
+    def post_packages_install(self):
         pass

coriolis/osmorphing/debian.py

@@ -1,65 +1,46 @@
-import itertools
 import os
 import re
 
 from coriolis import constants
 from coriolis.osmorphing import base
-from coriolis import utils
 
 
-class DebianOSMorphingTools(base.BaseOSMorphingTools):
+class DebianMorphingTools(base.BaseOSMorphingTools):
     _packages = {
         (constants.HYPERVISOR_VMWARE, None): [("open-vm-tools", True)],
         # TODO: add cloud-initramfs-growroot
         (None, constants.PLATFORM_OPENSTACK): [("cloud-init", True)],
     }
 
-    @staticmethod
-    def check_os(ssh, os_root_dir):
-        lsb_release_path = os.path.join(os_root_dir, "etc/lsb-release")
-        debian_version_path = os.path.join(os_root_dir, "etc/debian_version")
-        if utils.test_ssh_path(ssh, lsb_release_path):
-            out = utils.exec_ssh_cmd(
-                ssh, "cat %s" % lsb_release_path).decode()
+    def check_os(self):
+        lsb_release_path = "etc/lsb-release"
+        debian_version_path = "etc/debian_version"
+        if self._test_path(lsb_release_path):
+            out = self._read_file(lsb_release_path).decode()
             dist_id = re.findall('^DISTRIB_ID=(.*)$', out, re.MULTILINE)
             release = re.findall('^DISTRIB_RELEASE=(.*)$', out, re.MULTILINE)
             if 'Debian' in dist_id:
                 return (dist_id, release)
-        elif utils.test_ssh_path(ssh, debian_version_path):
-            release = utils.exec_ssh_cmd(
-                ssh, "cat %s" % debian_version_path).decode()
+        elif self._test_path(debian_version_path):
+            release = self._read_file(debian_version_path).decode()
             return ('Debian', release)
 
-    def set_dhcp(self, ssh):
+    def set_dhcp(self):
+        # NOTE: doesn't work with chroot
         interfaces_path = os.path.join(
             self._os_root_dir, "etc/network/interfaces")
-        utils.exec_ssh_cmd(
-            ssh, 'sudo sed -i.bak "s/static/dhcp/g" %s' % interfaces_path)
+        self._exec_cmd('sudo sed -i.bak "s/static/dhcp/g" %s' %
+                       interfaces_path)
 
-    def get_packages(self, hypervisor, platform):
-        k_add = [(h, p) for (h, p) in self._packages.keys() if
-                 (h is None or h == hypervisor) and
-                 (p is None or p == platform)]
-
-        add = [p[0] for p in itertools.chain.from_iterable(
-               [l for k, l in self._packages.items() if k in k_add])]
-
-        k_remove = set(self._packages.keys()) - set(k_add)
-        remove = [p[0] for p in itertools.chain.from_iterable(
-                  [l for k, l in self._packages.items() if k in k_remove])
-                  if p[1]]
-
-        return add, remove
-
-    def update_packages_list(self, ssh):
+    def pre_packages_install(self):
         apt_get_cmd = 'apt-get update -y'
-        utils.exec_ssh_cmd_chroot(ssh, self._os_root_dir, apt_get_cmd)
+        self._exec_cmd_chroot(apt_get_cmd)
 
-    def install_packages(self, ssh, package_names):
+    def install_packages(self, package_names):
         apt_get_cmd = 'apt-get install %s -y' % " ".join(package_names)
-        utils.exec_ssh_cmd_chroot(ssh, self._os_root_dir, apt_get_cmd)
+        self._exec_cmd_chroot(apt_get_cmd)
 
-    def uninstall_packages(self, ssh, package_names):
+    def uninstall_packages(self, package_names):
         for package_name in package_names:
             apt_get_cmd = 'apt-get remove %s -y || true' % package_name
-            utils.exec_ssh_cmd_chroot(ssh, self._os_root_dir, apt_get_cmd)
+            self._exec_cmd_chroot(apt_get_cmd)

coriolis/osmorphing/factory.py

@@ -1,13 +1,17 @@
 from coriolis.osmorphing import debian
+from coriolis.osmorphing import redhat
 from coriolis.osmorphing import ubuntu
 
 
-def get_os_morphing_tools(ssh, os_root_dir):
-    os_morphing_tools_clss = [debian.DebianOSMorphingTools,
-                              ubuntu.UbuntuOSMorphingTools]
+def get_os_morphing_tools(ssh, os_root_dir, target_hypervisor,
+                          target_platform):
+    os_morphing_tools_clss = [debian.DebianMorphingTools,
+                              ubuntu.UbuntuMorphingTools,
+                              redhat.RedHatMorphingTools]
 
     for cls in os_morphing_tools_clss:
-        os_info = cls.check_os(ssh, os_root_dir)
+        tools = cls(ssh, os_root_dir, target_hypervisor, target_platform)
+        os_info = tools.check_os()
         if os_info:
-            return cls(os_root_dir), os_info
+            return (tools, os_info)
     raise Exception("Cannot find the morphing tools for this OS image")

coriolis/osmorphing/manager.py

@@ -21,23 +21,28 @@ def morph_image(connection_info, target_hypervisor, target_platform,
     ssh.connect(hostname=ip, port=port, username=username, pkey=pkey)
 
     os_mount_tools = osmount_factory.get_os_mount_tools(ssh)
-    os_root_dir = os_mount_tools.mount_os(ssh, volume_devs)
+    os_root_dir, other_mounted_dirs = os_mount_tools.mount_os(ssh, volume_devs)
     os_morphing_tools, os_info = osmorphing_factory.get_os_morphing_tools(
-        ssh, os_root_dir)
+        ssh, os_root_dir, target_hypervisor, target_platform)
 
     LOG.info('OS being migrated: %s', str(os_info))
 
-    os_morphing_tools.set_dhcp(ssh)
+    os_morphing_tools.set_dhcp()
+    LOG.info("Pre packages")
+    os_morphing_tools.pre_packages_install()
 
     (packages_add,
-     packages_remove) = os_morphing_tools.get_packages(target_hypervisor,
-                                                       target_platform)
-    os_morphing_tools.update_packages_list(ssh)
+     packages_remove) = os_morphing_tools.get_packages()
 
     if packages_add:
         LOG.info("Adding packages: %s" % str(packages_add))
-        os_morphing_tools.install_packages(ssh, packages_add)
+        os_morphing_tools.install_packages(packages_add)
 
     if packages_remove:
-        LOG.info("Removing packages: %s" % str(packages_add))
-        os_morphing_tools.uninstall_packages(ssh, packages_remove)
+        LOG.info("Removing packages: %s" % str(packages_remove))
+        os_morphing_tools.uninstall_packages(packages_remove)
+
+    LOG.info("Post packages")
+    os_morphing_tools.post_packages_install()
+
+    os_mount_tools.dismount_os(ssh, other_mounted_dirs + [os_root_dir])

coriolis/osmorphing/osmount/ubuntu.py

@@ -1,3 +1,4 @@
+import os
 import re
 
 from coriolis import utils
@@ -24,6 +25,7 @@ class UbuntuOSMountTools(object):
 
     def mount_os(self, ssh, volume_devs):
         dev_paths = []
+        other_mounted_dirs = []
 
         for volume_dev in volume_devs:
             utils.exec_ssh_cmd(
@@ -41,33 +43,59 @@ class UbuntuOSMountTools(object):
                 ssh, "sudo ls /dev/%s/*" % vg_name).decode().split('\n')[:-1]
             dev_paths += lvm_dev_paths
 
+        valid_filesystems = ['ext2', 'ext3', 'ext4', 'xfs']
+
         dev_paths_to_mount = []
         for dev_path in dev_paths:
             fs_type = utils.exec_ssh_cmd(
                 ssh, "sudo blkid -o value -s TYPE %s || true" %
                 dev_path).decode().split('\n')[0]
-            if fs_type in ['ext2', 'ext3', 'ext4']:
+            if fs_type in valid_filesystems:
                 dev_paths_to_mount.append(dev_path)
 
         os_root_dir = None
+        boot_dev_path = None
         for dev_path in dev_paths_to_mount:
             tmp_dir = utils.exec_ssh_cmd(
                 ssh,  'mktemp -d').decode().split('\n')[0]
             utils.exec_ssh_cmd(ssh, 'sudo mount %s %s' % (dev_path, tmp_dir))
             dirs = utils.exec_ssh_cmd(
                 ssh, 'ls %s' % tmp_dir).decode().split('\n')
+
             # TODO: better ways to check for a linux root?
-            if 'etc' in dirs and 'bin' in dirs and 'sbin' in dirs:
-                for dir in set(dirs).intersection(['proc', 'sys', 'dev',
-                                                   'run']):
-                    utils.exec_ssh_cmd(
-                        ssh,
-                        'sudo mount -o bind /%(dir)s/ %(mount_dir)s/%(dir)s' %
-                        {'dir': dir, 'mount_dir': tmp_dir})
+            if (not os_root_dir and 'etc' in dirs and 'bin' in dirs and
+                    'sbin' in dirs):
                 os_root_dir = tmp_dir
+            # TODO: better ways to check for a linux boot dir?
+            else:
+                # TODO: better ways to check for a linux boot dir?
+                if not boot_dev_path and ('grub' in dirs or 'grub2' in dirs):
+                    # Needs to be remounted under os_root_dir
+                    boot_dev_path = dev_path
+
+                utils.exec_ssh_cmd(ssh, 'sudo umount %s' % tmp_dir)
+
+            if os_root_dir and boot_dev_path:
                 break
 
         if not os_root_dir:
             raise Exception("root partition not found")
 
-        return os_root_dir
+        for dir in set(dirs).intersection(['proc', 'sys', 'dev', 'run']):
+            mount_dir = os.path.join(os_root_dir, dir)
+            utils.exec_ssh_cmd(
+                ssh, 'sudo mount -o bind /%(dir)s/ %(mount_dir)s' %
+                {'dir': dir, 'mount_dir': mount_dir})
+            other_mounted_dirs.append(mount_dir)
+
+        if boot_dev_path:
+            boot_dir = os.path.join(os_root_dir, 'boot')
+            utils.exec_ssh_cmd(ssh, 'sudo mount %s %s' %
+                               (boot_dev_path, boot_dir))
+            other_mounted_dirs.append(boot_dir)
+
+        return os_root_dir, other_mounted_dirs
+
+    def dismount_os(self, ssh, dirs):
+        for dir in dirs:
+            utils.exec_ssh_cmd(ssh, 'sudo umount %s' % dir)

coriolis/osmorphing/redhat.py

@@ -0,0 +1,75 @@
+import re
+
+from oslo_log import log as logging
+
+from coriolis import constants
+from coriolis.osmorphing import base
+
+LOG = logging.getLogger(__name__)
+
+
+class RedHatMorphingTools(base.BaseOSMorphingTools):
+    _packages = {
+        (None, None): [("dracut-config-generic", False)],
+        (constants.HYPERVISOR_VMWARE, None): [("open-vm-tools", True)],
+        (constants.HYPERVISOR_HYPERV, None): [("hyperv-daemons", True)],
+        (None, constants.PLATFORM_OPENSTACK): [
+            ("cloud-init", True),
+            ("cloud-utils-growpart", False)],
+    }
+    _CLOUD_INIT_USER = 'centos'
+
+    def check_os(self):
+        redhat_release_path = "etc/redhat-release"
+        if self._test_path(redhat_release_path):
+            release_info = self._read_file(
+                redhat_release_path).decode().split('\n')[0].strip()
+            m = re.match(r"^(.*) release ([0-9].*) \((.*)\).*$", release_info)
+            if m:
+                distro, version, codename = m.groups()
+                return (distro, version)
+
+    def set_dhcp(self):
+        # TODO: set BOOTPROTO="dhcp" in all relevant ifcfg-* configurations
+        pass
+
+    def install_packages(self, package_names):
+        apt_get_cmd = 'yum install %s -y' % " ".join(package_names)
+        self._exec_cmd_chroot(apt_get_cmd)
+
+    def uninstall_packages(self, package_names):
+        for package_name in package_names:
+            apt_get_cmd = 'yum remove %s -y' % package_name
+            self._exec_cmd_chroot(apt_get_cmd)
+
+    def _run_dracut(self):
+        package_names = self._exec_cmd_chroot(
+            'rpm -q kernel').decode().split('\n')[:-1]
+        for package_name in package_names:
+            m = re.match('^kernel-(.*)$', package_name)
+            if m:
+                kernel_version = m.groups()[0]
+                LOG.info("Generating initrd for kernel: %s", kernel_version)
+                self._exec_cmd_chroot("dracut -f --kver %s" % kernel_version)
+
+    def _add_cloud_init_user(self):
+        if ("cloud-init" in self.get_packages()[0] and not
+                self._check_user_exists(self._CLOUD_INIT_USER)):
+            self._exec_cmd_chroot("useradd %s" % self._CLOUD_INIT_USER)
+
+    def _add_hyperv_ballooning_rule(self):
+        udev_file = "etc/udev/rules.d/100-balloon.rules"
+        content = 'SUBSYSTEM=="memory", ACTION=="add", ATTR{state}="online"\n'
+
+        if (self._hypervisor == constants.HYPERVISOR_HYPERV and
+                not self._test_path(udev_file)):
+            # NOTE: writes the file to a temp location due to permission issues
+            tmp_file = 'tmp/100-balloon.rules'
+            self._write_file(tmp_file, content)
+            self._exec_cmd_chroot("cp /%s /%s" % (tmp_file, udev_file))
+            self._exec_cmd_chroot("rm /%s" % tmp_file)
+
+    def post_packages_install(self):
+        self._run_dracut()
+        self._add_cloud_init_user()
+        self._add_hyperv_ballooning_rule()

coriolis/osmorphing/ubuntu.py

@@ -1,12 +1,10 @@
-import os
 import re
 
 from coriolis import constants
 from coriolis.osmorphing import debian
-from coriolis import utils
 
 
-class UbuntuOSMorphingTools(debian.DebianOSMorphingTools):
+class UbuntuMorphingTools(debian.DebianMorphingTools):
     _packages = {
         (constants.HYPERVISOR_VMWARE, None): [("open-vm-tools", True)],
         # TODO: sudo agt-get install linux-tool-<kernel release>
@@ -16,12 +14,10 @@ class UbuntuOSMorphingTools(debian.DebianOSMorphingTools):
         (None, constants.PLATFORM_OPENSTACK): [("cloud-init", True)],
     }
 
-    @staticmethod
-    def check_os(ssh, os_root_dir):
-        lsb_release_path = os.path.join(os_root_dir, "etc/lsb-release")
-        if utils.test_ssh_path(ssh, lsb_release_path):
-            out = utils.exec_ssh_cmd(
-                ssh, "cat %s" % lsb_release_path).decode()
+    def check_os(self):
+        lsb_release_path = "etc/lsb-release"
+        if self._test_path(lsb_release_path):
+            out = self._read_file(lsb_release_path).decode()
 
             dist_id = re.findall('^DISTRIB_ID=(.*)$', out, re.MULTILINE)
             release = re.findall('^DISTRIB_RELEASE=(.*)$', out, re.MULTILINE)

coriolis/providers/openstack/__init__.py

@@ -237,8 +237,8 @@ class ImportProvider(base.BaseExportProvider):
                 to_port=SSH_PORT)
             instance.add_security_group(sec_group.id)
 
-            LOG.info("Waiting for connectivity on host: %s:%s",
-                     (floating_ip.ip, SSH_PORT))
+            LOG.info("Waiting for connectivity on host: %(ip)s:%(port)s",
+                     {"ip": floating_ip.ip, "port": SSH_PORT})
             utils.wait_for_port_connectivity(floating_ip.ip, SSH_PORT)
 
             return _MigrationResources(nova, neutron, keypair, instance, port,

coriolis/utils.py

@@ -50,12 +50,28 @@ def get_linux_os_info(ssh):
         return (dist_id[0], release[0])
 
 
+@retry_on_error()
 def test_ssh_path(ssh, remote_path):
+    sftp = ssh.open_sftp()
     try:
-        exec_ssh_cmd(ssh, "test -f %s" % remote_path)
+        sftp.stat(remote_path)
         return True
-    except Exception:
-        return False
+    except IOError as ex:
+        if ex.args[0] == 2:
+            return False
+        raise
+
+
+@retry_on_error()
+def read_ssh_file(ssh, remote_path):
+    sftp = ssh.open_sftp()
+    return sftp.open(remote_path, 'rb').read()
+
+
+@retry_on_error()
+def write_ssh_file(ssh, remote_path, content):
+    sftp = ssh.open_sftp()
+    sftp.open(remote_path, 'wb').write(content)
 
 
 @retry_on_error()