Add osmorphing commands channel timeout

coriolis/exception.py

@@ -26,12 +26,10 @@ from webob.util import status_reasons
 
 from coriolis.i18n import _, _LE  # noqa
 
-
 LOG = logging.getLogger(__name__)
 
 CONF = cfg.CONF
 
-
 TASK_ALREADY_CANCELLING_EXCEPTION_FMT = (
     "Task %(task_id)s is in CANCELLING status.")
 
@@ -470,3 +468,31 @@ class FailedPackageUninstallationException(PackageManagerOperationException):
     message = (
         "Failed to remove unwanted packages (%(package_names)s) through "
         "%(package_manager)s. Error was: %(error)s")
+
+
+class MinionMachineCommandTimeout(CoriolisException):
+    pass
+
+
+class OSMorphingOperationTimeout(MinionMachineCommandTimeout):
+    pass
+
+
+class OSMorphingSSHOperationTimeout(OSMorphingOperationTimeout):
+    message = (
+        "Pending SSH command %(cmd)s timed out after %(timeout)s seconds. "
+        "Coriolis may have encountered connection issues to the minion machine"
+        " or the command execution time exceeds the timeout set. Try extending"
+        " the timeout by editing the 'default_osmorphing_operation_timeout' "
+        "in Coriolis' static configuration file."
+    )
+
+
+class OSMorphingWinRMOperationTimeout(OSMorphingOperationTimeout):
+    message = (
+        "Pending WinRM command %(cmd)s timed out after %(timeout)s seconds. "
+        "Coriolis may have encountered connection issues to the minion machine"
+        " or the command execution time exceeds the timeout set. Try extending"
+        " the timeout by editing the 'default_osmorphing_operation_timeout' "
+        "in Coriolis' static configuration file."
+    )

coriolis/osmorphing/base.py

@@ -28,7 +28,8 @@ class BaseOSMorphingTools(object, with_metaclass(abc.ABCMeta)):
 
     def __init__(
             self, conn, os_root_dir, os_root_device, hypervisor,
-            event_manager, detected_os_info, osmorphing_parameters):
+            event_manager, detected_os_info, osmorphing_parameters,
+            operation_timeout):
 
         self.check_detected_os_info_parameters(detected_os_info)
 
@@ -42,6 +43,7 @@ class BaseOSMorphingTools(object, with_metaclass(abc.ABCMeta)):
         self._detected_os_info = detected_os_info
         self._environment = {}
         self._osmorphing_parameters = osmorphing_parameters
+        self._osmorphing_operation_timeout = operation_timeout
 
     @abc.abstractclassmethod
     def get_required_detected_os_info_fields(cls):
@@ -120,10 +122,11 @@ class BaseLinuxOSMorphingTools(BaseOSMorphingTools):
     _packages = {}
 
     def __init__(self, conn, os_root_dir, os_root_dev, hypervisor,
-                 event_manager, detected_os_info, osmorphing_parameters):
+                 event_manager, detected_os_info, osmorphing_parameters,
+                 operation_timeout=None):
         super(BaseLinuxOSMorphingTools, self).__init__(
             conn, os_root_dir, os_root_dev, hypervisor, event_manager,
-            detected_os_info, osmorphing_parameters)
+            detected_os_info, osmorphing_parameters, operation_timeout)
         self._ssh = conn
 
     @classmethod
@@ -256,14 +259,27 @@ class BaseLinuxOSMorphingTools(BaseOSMorphingTools):
         path = os.path.join(self._os_root_dir, chroot_path)
         return utils.list_ssh_dir(self._ssh, path)
 
-    def _exec_cmd(self, cmd):
-        return utils.exec_ssh_cmd(
-            self._ssh, cmd, environment=self._environment, get_pty=True)
-
-    def _exec_cmd_chroot(self, cmd):
-        return utils.exec_ssh_cmd_chroot(
-            self._ssh, self._os_root_dir, cmd,
-            environment=self._environment, get_pty=True)
+    def _exec_cmd(self, cmd, timeout=None):
+        if not timeout:
+            timeout = self._osmorphing_operation_timeout
+        try:
+            return utils.exec_ssh_cmd(
+                self._ssh, cmd, environment=self._environment, get_pty=True,
+                timeout=timeout)
+        except exception.MinionMachineCommandTimeout as ex:
+            raise exception.OSMorphingSSHOperationTimeout(
+                cmd=cmd, timeout=timeout) from ex
+
+    def _exec_cmd_chroot(self, cmd, timeout=None):
+        if not timeout:
+            timeout = self._osmorphing_operation_timeout
+        try:
+            return utils.exec_ssh_cmd_chroot(
+                self._ssh, self._os_root_dir, cmd,
+                environment=self._environment, get_pty=True, timeout=timeout)
+        except exception.MinionMachineCommandTimeout as ex:
+            raise exception.OSMorphingSSHOperationTimeout(
+                cmd=cmd, timeout=timeout) from ex
 
     def _check_user_exists(self, username):
         try:

coriolis/osmorphing/manager.py

@@ -13,6 +13,13 @@ from coriolis.osmorphing import base as base_osmorphing
 from coriolis.osmorphing.osmount import factory as osmount_factory
 from coriolis.osmorphing.osdetect import manager as osdetect_manager
 
+
+opts = [
+    cfg.IntOpt('default_osmorphing_operation_timeout',
+               help='Number of seconds to wait for a pending SSH or WinRM '
+                    'command before the socket times out.')
+]
+
 proxy_opts = [
     cfg.StrOpt('url',
                default=None,
@@ -29,6 +36,7 @@ proxy_opts = [
 ]
 
 CONF = cfg.CONF
+CONF.register_opts(opts)
 CONF.register_opts(proxy_opts, 'proxy')
 
 LOG = logging.getLogger(__name__)
@@ -55,6 +63,7 @@ def run_os_detect(
 
     detected_info = osdetect_manager.detect_os(
         worker_connection, os_type, os_root_dir,
+        CONF.default_osmorphing_operation_timeout,
         tools_environment=tools_environment,
         custom_os_detect_tools=list(
             itertools.chain(
@@ -125,7 +134,8 @@ def morph_image(origin_provider, destination_provider, connection_info,
 
     # instantiate and run OSMount tools:
     os_mount_tools = osmount_factory.get_os_mount_tools(
-        os_type, connection_info, event_manager, ignore_devices)
+        os_type, connection_info, event_manager, ignore_devices,
+        CONF.default_osmorphing_operation_timeout)
 
     proxy_settings = _get_proxy_settings()
     os_mount_tools.set_proxy(proxy_settings)
@@ -172,7 +182,8 @@ def morph_image(origin_provider, destination_provider, connection_info,
                 type(origin_provider))
             export_os_morphing_tools = export_tools_cls(
                 conn, os_root_dir, os_root_dev, hypervisor_type,
-                event_manager, detected_os_info, osmorphing_parameters)
+                event_manager, detected_os_info, osmorphing_parameters,
+                CONF.default_osmorphing_operation_timeout)
             export_os_morphing_tools.set_environment(environment)
         else:
             LOG.debug(
@@ -195,7 +206,8 @@ def morph_image(origin_provider, destination_provider, connection_info,
 
     import_os_morphing_tools = import_os_morphing_tools_cls(
         conn, os_root_dir, os_root_dev, hypervisor_type,
-        event_manager, detected_os_info, osmorphing_parameters)
+        event_manager, detected_os_info, osmorphing_parameters,
+        CONF.default_osmorphing_operation_timeout)
     import_os_morphing_tools.set_environment(environment)
 
     if user_script:

coriolis/osmorphing/osdetect/base.py

@@ -7,9 +7,9 @@ import os
 
 from six import with_metaclass
 
+from coriolis import exception
 from coriolis import utils
 
-
 # Required OS release fields to be returned as declared in the
 # 'schemas.CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA' schema:
 REQUIRED_DETECTED_OS_FIELDS = [
@@ -18,10 +18,11 @@ REQUIRED_DETECTED_OS_FIELDS = [
 
 class BaseOSDetectTools(object, with_metaclass(abc.ABCMeta)):
 
-    def __init__(self, conn, os_root_dir):
+    def __init__(self, conn, os_root_dir, operation_timeout):
         self._conn = conn
         self._os_root_dir = os_root_dir
         self._environment = {}
+        self._osdetect_operation_timeout = operation_timeout
 
     @abc.abstractclassmethod
     def returned_detected_os_info_fields(cls):
@@ -74,11 +75,24 @@ class BaseLinuxOSDetectTools(BaseOSDetectTools):
         full_path = os.path.join(self._os_root_dir, chroot_path)
         return utils.test_ssh_path(self._conn, full_path)
 
-    def _exec_cmd(self, cmd):
-        return utils.exec_ssh_cmd(
-            self._conn, cmd, environment=self._environment, get_pty=True)
-
-    def _exec_cmd_chroot(self, cmd):
-        return utils.exec_ssh_cmd_chroot(
-            self._conn, self._os_root_dir, cmd,
-            environment=self._environment, get_pty=True)
+    def _exec_cmd(self, cmd, timeout=None):
+        if not timeout:
+            timeout = self._osdetect_operation_timeout
+        try:
+            return utils.exec_ssh_cmd(
+                self._conn, cmd, environment=self._environment, get_pty=True,
+                timeout=timeout)
+        except exception.MinionMachineCommandTimeout as ex:
+            raise exception.OSMorphingSSHOperationTimeout(
+                cmd=cmd, timeout=timeout) from ex
+
+    def _exec_cmd_chroot(self, cmd, timeout=None):
+        if not timeout:
+            timeout = self._osdetect_operation_timeout
+        try:
+            return utils.exec_ssh_cmd_chroot(
+                self._conn, self._os_root_dir, cmd,
+                environment=self._environment, get_pty=True, timeout=timeout)
+        except exception.MinionMachineCommandTimeout as ex:
+            raise exception.OSMorphingSSHOperationTimeout(
+                cmd=cmd, timeout=timeout) from ex

coriolis/osmorphing/osdetect/manager.py

@@ -48,7 +48,7 @@ def _check_custom_os_detect_tools(custom_os_detect_tools):
 
 
 def detect_os(
-        conn, os_type, os_root_dir, tools_environment=None,
+        conn, os_type, os_root_dir, operation_timeout, tools_environment=None,
         custom_os_detect_tools=None):
     """ Iterates through all of the OS detection tools until one successfully
     identifies the OS/release and returns the release info from it.
@@ -75,7 +75,7 @@ def detect_os(
     tools = None
     detected_info = {}
     for detectcls in detect_tools_classes:
-        tools = detectcls(conn, os_root_dir)
+        tools = detectcls(conn, os_root_dir, operation_timeout)
         tools.set_environment(tools_environment)
         LOG.debug(
             "Running OS detection tools class: %s" % detectcls.__name__)

coriolis/osmorphing/osmount/base.py

@@ -23,11 +23,13 @@ MAJOR_COLUMN_INDEX = 4
 
 class BaseOSMountTools(object, with_metaclass(abc.ABCMeta)):
 
-    def __init__(self, connection_info, event_manager, ignore_devices):
+    def __init__(self, connection_info, event_manager, ignore_devices,
+                 operation_timeout):
         self._event_manager = event_manager
         self._ignore_devices = ignore_devices
         self._environment = {}
         self._connection_info = connection_info
+        self._osmount_operation_timeout = operation_timeout
         self._connect()
 
     @abc.abstractmethod
@@ -94,9 +96,15 @@ class BaseSSHOSMountTools(BaseOSMountTools):
     def _allow_ssh_env_vars(self):
         pass
 
-    def _exec_cmd(self, cmd):
-        return utils.exec_ssh_cmd(self._ssh, cmd, self._environment,
-                                  get_pty=True)
+    def _exec_cmd(self, cmd, timeout=None):
+        if not timeout:
+            timeout = self._osmount_operation_timeout
+        try:
+            return utils.exec_ssh_cmd(self._ssh, cmd, self._environment,
+                                      get_pty=True, timeout=timeout)
+        except exception.MinionMachineCommandTimeout as ex:
+            raise exception.OSMorphingSSHOperationTimeout(
+                cmd=cmd, timeout=timeout) from ex
 
     def get_connection(self):
         return self._ssh

coriolis/osmorphing/osmount/factory.py

@@ -15,7 +15,7 @@ LOG = logging.getLogger(__name__)
 
 
 def get_os_mount_tools(os_type, connection_info, event_manager,
-                       ignore_devices):
+                       ignore_devices, operation_timeout):
     os_mount_tools = {constants.OS_TYPE_LINUX: [ubuntu.UbuntuOSMountTools,
                                                 redhat.RedHatOSMountTools],
                       constants.OS_TYPE_WINDOWS: [windows.WindowsMountTools]}
@@ -25,7 +25,8 @@ def get_os_mount_tools(os_type, connection_info, event_manager,
 
     for cls in os_mount_tools.get(os_type,
                                   itertools.chain(*os_mount_tools.values())):
-        tools = cls(connection_info, event_manager, ignore_devices)
+        tools = cls(connection_info, event_manager, ignore_devices,
+                    operation_timeout)
         LOG.debug("Testing OS mount tools: %s", cls.__name__)
         if tools.check_os():
             return tools

coriolis/osmorphing/osmount/windows.py

@@ -25,7 +25,7 @@ class WindowsMountTools(base.BaseOSMountTools):
             {"host": host, "port": port})
 
         self._conn = wsman.WSManConnection.from_connection_info(
-            connection_info)
+            connection_info, self._osmount_operation_timeout)
 
     def get_connection(self):
         return self._conn

coriolis/osmorphing/redhat.py

@@ -53,10 +53,11 @@ class BaseRedHatMorphingTools(base.BaseLinuxOSMorphingTools):
 
     def __init__(self, conn, os_root_dir, os_root_dev,
                  hypervisor, event_manager, detected_os_info,
-                 osmorphing_parameters):
+                 osmorphing_parameters, operation_timeout=None):
         super(BaseRedHatMorphingTools, self).__init__(
             conn, os_root_dir, os_root_dev,
-            hypervisor, event_manager, detected_os_info, osmorphing_parameters)
+            hypervisor, event_manager, detected_os_info, osmorphing_parameters,
+            operation_timeout)
         self._enable_repos = []
 
     def disable_predictable_nic_names(self):

coriolis/osmorphing/windows.py

@@ -168,10 +168,12 @@ class BaseWindowsMorphingTools(base.BaseOSMorphingTools):
 
     def __init__(
             self, conn, os_root_dir, os_root_device, hypervisor,
-            event_manager, detected_os_info, osmorphing_parameters):
+            event_manager, detected_os_info, osmorphing_parameters,
+            operation_timeout=None):
         super(BaseWindowsMorphingTools, self).__init__(
             conn, os_root_dir, os_root_device, hypervisor,
-            event_manager, detected_os_info, osmorphing_parameters)
+            event_manager, detected_os_info, osmorphing_parameters,
+            operation_timeout)
 
         self._version_number = detected_os_info['version_number']
         self._edition_id = detected_os_info['edition_id']

coriolis/utils.py

@@ -26,7 +26,6 @@ from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
 
-import __main__ as main
 import netifaces
 import paramiko
 # NOTE(gsamfira): I am aware that this is not ideal, but pip
@@ -292,9 +291,11 @@ def list_ssh_dir(ssh, remote_path):
     return sftp.listdir(remote_path)
 
 
-@retry_on_error()
-def exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False):
+@retry_on_error(terminal_exceptions=[exception.MinionMachineCommandTimeout])
+def exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False, timeout=None):
     remote_str = "<undeterminable>"
+    if timeout is not None:
+        timeout = float(timeout)
     try:
         remote_str = "%s:%s" % ssh.get_transport().sock.getpeername()
     except (ValueError, AttributeError, TypeError):
@@ -306,9 +307,12 @@ def exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False):
         "environment %s: '%s'", remote_str, environment, cmd)
 
     _, stdout, stderr = ssh.exec_command(
-        cmd, environment=environment, get_pty=get_pty)
-    std_out = stdout.read()
-    std_err = stderr.read()
+        cmd, environment=environment, get_pty=get_pty, timeout=timeout)
+    try:
+        std_out = stdout.read()
+        std_err = stderr.read()
+    except socket.timeout:
+        raise exception.MinionMachineCommandTimeout()
     exit_code = stdout.channel.recv_exit_status()
     if exit_code:
         raise exception.CoriolisException(
@@ -321,9 +325,11 @@ def exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False):
     return std_out.replace(b'\r\n', b'\n').replace(b'\n\r', b'\n')
 
 
-def exec_ssh_cmd_chroot(ssh, chroot_dir, cmd, environment=None, get_pty=False):
+def exec_ssh_cmd_chroot(ssh, chroot_dir, cmd, environment=None, get_pty=False,
+                        timeout=None):
     return exec_ssh_cmd(ssh, "sudo -E chroot %s %s" % (chroot_dir, cmd),
-                        environment=environment, get_pty=get_pty)
+                        environment=environment, get_pty=get_pty,
+                        timeout=timeout)
 
 
 def check_fs(ssh, fs_type, dev_path):

coriolis/wsman.py

@@ -4,6 +4,7 @@
 import base64
 
 from oslo_log import log as logging
+import requests
 from winrm import protocol
 from winrm import exceptions as winrm_exceptions
 
@@ -15,21 +16,21 @@ AUTH_KERBEROS = "kerberos"
 AUTH_CERTIFICATE = "certificate"
 
 CODEPAGE_UTF8 = 65001
+DEFAULT_TIMEOUT = 3600
 
 LOG = logging.getLogger(__name__)
 
 
 class WSManConnection(object):
-    def __init__(self):
+    def __init__(self, timeout=None):
         self._protocol = None
+        self._conn_timeout = int(timeout or DEFAULT_TIMEOUT)
 
     EOL = "\r\n"
 
     @utils.retry_on_error()
     def connect(self, url, username, auth=None, password=None,
                 cert_pem=None, cert_key_pem=None):
-        protocol.Protocol.DEFAULT_TIMEOUT = 3600
-
         if not auth:
             if cert_pem:
                 auth = AUTH_CERTIFICATE
@@ -49,7 +50,7 @@ class WSManConnection(object):
             cert_key_pem=cert_key_pem)
 
     @classmethod
-    def from_connection_info(cls, connection_info):
+    def from_connection_info(cls, connection_info, timeout=DEFAULT_TIMEOUT):
         """ Returns a wsman.WSManConnection object for the provided conn info. """
         if not isinstance(connection_info, dict):
             raise ValueError(
@@ -77,7 +78,7 @@ class WSManConnection(object):
                  {"host": host, "port": port})
         utils.wait_for_port_connectivity(host, port)
 
-        conn = cls()
+        conn = cls(timeout)
         conn.connect(url=url, username=username, password=password,
                      cert_pem=cert_pem, cert_key_pem=cert_key_pem)
 
@@ -86,9 +87,17 @@ class WSManConnection(object):
     def disconnect(self):
         self._protocol = None
 
+    def set_timeout(self, timeout):
+        if timeout:
+            self._protocol.timeout = timeout
+            self._protocol.transport.timeout = timeout
+
     @utils.retry_on_error(
-        terminal_exceptions=[winrm_exceptions.InvalidCredentialsError])
-    def _exec_command(self, cmd, args=[]):
+        terminal_exceptions=[winrm_exceptions.InvalidCredentialsError,
+                             exception.OSMorphingWinRMOperationTimeout])
+    def _exec_command(self, cmd, args=[], timeout=None):
+        timeout = int(timeout or self._conn_timeout)
+        self.set_timeout(timeout)
         shell_id = self._protocol.open_shell(codepage=CODEPAGE_UTF8)
         try:
             command_id = self._protocol.run_command(shell_id, cmd, args)
@@ -97,6 +106,9 @@ class WSManConnection(object):
                  std_err,
                  exit_code) = self._protocol.get_command_output(
                     shell_id, command_id)
+            except requests.exceptions.ReadTimeout:
+                raise exception.OSMorphingWinRMOperationTimeout(
+                    cmd=("%s %s" % (cmd, " ".join(args))), timeout=timeout)
             finally:
                 self._protocol.cleanup_command(shell_id, command_id)
 
@@ -104,9 +116,10 @@ class WSManConnection(object):
         finally:
             self._protocol.close_shell(shell_id)
 
-    def exec_command(self, cmd, args=[]):
+    def exec_command(self, cmd, args=[], timeout=None):
         LOG.debug("Executing WSMAN command: %s", str([cmd] + args))
-        std_out, std_err, exit_code = self._exec_command(cmd, args)
+        std_out, std_err, exit_code = self._exec_command(
+            cmd, args, timeout=timeout)
 
         if exit_code:
             raise exception.CoriolisException(
@@ -116,11 +129,12 @@ class WSManConnection(object):
 
         return std_out
 
-    def exec_ps_command(self, cmd, ignore_stdout=False):
+    def exec_ps_command(self, cmd, ignore_stdout=False, timeout=None):
         LOG.debug("Executing PS command: %s", cmd)
         base64_cmd = base64.b64encode(cmd.encode('utf-16le')).decode()
         return self.exec_command(
-            "powershell.exe", ["-EncodedCommand", base64_cmd])[:-2]
+            "powershell.exe", ["-EncodedCommand", base64_cmd],
+            timeout=timeout)[:-2]
 
     def test_path(self, remote_path):
         ret_val = self.exec_ps_command("Test-Path -Path \"%s\"" % remote_path)