Skips retrying on "command not found" errors

`exec_ssh_cmd` will retry the given cmd in case an error occurs. But if
the command does not exist, there's no need to retry.

coriolis/exception.py

@@ -503,6 +503,10 @@ class MinionMachineCommandTimeout(CoriolisException):
     pass
 
 
+class SSHCommandNotFoundException(CoriolisException):
+    pass
+
+
 class OSMorphingOperationTimeout(MinionMachineCommandTimeout):
     pass
 

coriolis/tests/test_utils.py

@@ -378,6 +378,7 @@ class UtilsTestCase(test_base.CoriolisBaseTestCase):
             "command", environment=None, get_pty=False, timeout=None)
 
     def test_exec_ssh_cmd_exception(self):
+        self.mock_stdout.read.return_value = b'some error output'
         self.mock_stdout.channel.recv_exit_status.return_value = 1
         self.mock_ssh.exec_command.return_value = (None, self.mock_stdout,
                                                    self.mock_stdout)
@@ -391,6 +392,30 @@ class UtilsTestCase(test_base.CoriolisBaseTestCase):
         self.mock_ssh.exec_command.assert_called_once_with(
             "command", environment=None, get_pty=False, timeout=None)
 
+    def test_exec_ssh_cmd_command_not_found_in_stdout(self):
+        self.mock_stdout.read.return_value = b'sudo: foo: command not found'
+        self.mock_stdout.channel.recv_exit_status.return_value = 1
+        self.mock_ssh.exec_command.return_value = (None, self.mock_stdout,
+                                                   self.mock_stdout)
+
+        original_exec_ssh_cmd = testutils.get_wrapped_function(
+            utils.exec_ssh_cmd)
+
+        self.assertRaises(exception.SSHCommandNotFoundException,
+                          original_exec_ssh_cmd, self.mock_ssh, "command")
+
+    def test_exec_ssh_cmd_exit_code_127(self):
+        self.mock_stdout.read.return_value = b''
+        self.mock_stdout.channel.recv_exit_status.return_value = 127
+        self.mock_ssh.exec_command.return_value = (None, self.mock_stdout,
+                                                   self.mock_stdout)
+
+        original_exec_ssh_cmd = testutils.get_wrapped_function(
+            utils.exec_ssh_cmd)
+
+        self.assertRaises(exception.SSHCommandNotFoundException,
+                          original_exec_ssh_cmd, self.mock_ssh, "command")
+
     def test_exec_ssh_cmd_chroot(self):
         self.mock_stdout.read.return_value = b'output\n'
         self.mock_stdout.channel.recv_exit_status.return_value = 0

coriolis/utils.py

@@ -311,7 +311,9 @@ def list_ssh_dir(ssh, remote_path):
         return [f for f in output.splitlines() if f.strip()]
 
 
-@retry_on_error(terminal_exceptions=[exception.MinionMachineCommandTimeout])
+@retry_on_error(terminal_exceptions=[
+    exception.MinionMachineCommandTimeout,
+    exception.SSHCommandNotFoundException])
 def exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False, timeout=None):
     remote_str = "<undeterminable>"
     if timeout is not None:
@@ -335,12 +337,17 @@ def exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False, timeout=None):
         raise exception.MinionMachineCommandTimeout()
     exit_code = stdout.channel.recv_exit_status()
     if exit_code:
-        raise exception.CoriolisException(
+        stdout_str = std_out.decode(errors='ignore')
+        stderr_str = std_err.decode(errors='ignore')
+        msg = (
             "Command \"%s\" failed on host '%s' with exit code: %s\n"
             "stdout: %s\nstd_err: %s" %
-            (cmd, remote_str, exit_code,
-             std_out.decode(errors='ignore'),
-             std_err.decode(errors='ignore')))
+            (cmd, remote_str, exit_code, stdout_str, stderr_str))
+        if (exit_code == 127 or
+                "command not found" in stdout_str or
+                "command not found" in stderr_str):
+            raise exception.SSHCommandNotFoundException(msg)
+        raise exception.CoriolisException(msg)
     # Most of the commands will use pseudo-terminal which unfortunately will
     # include a '\r' to every newline. This will affect all plugins too, so
     # best we can do now is replace them.