Handle worker failing to cancel tasks.

coriolis/conductor/rpc/server.py

@@ -1834,6 +1834,35 @@ class ConductorServerEndpoint(object):
                 exception_details=(
                     "This is a cleanup task so it was allowed to run to "
                     "completion after user-cancellation."))
+        elif task.status == constants.TASK_STATUS_CANCELLING:
+            LOG.error(
+                "Received confirmation that presumably cancelling task '%s' "
+                "(status '%s') has just completed successfully. "
+                "This should have never happened and indicates that its worker "
+                "host ('%s') has failed to cancel it properly. Please "
+                "check the worker logs for more details. "
+                "Marking as %s and processing its result as if it completed "
+                "successfully.",
+                task.id, task.status, task.host,
+                constants.TASK_STATUS_CANCELED_AFTER_COMPLETION)
+            db_api.set_task_status(
+                ctxt, task_id, constants.TASK_STATUS_CANCELED_AFTER_COMPLETION,
+                exception_details=(
+                    "The worker host for this task ('%s') has failed at "
+                    "cancelling it so this task was unintentionally run to "
+                    "completion. Please review the worker logs for "
+                    "more relevant details and contact support." % (
+                        task.host)))
+        elif task.status in constants.FINALIZED_TASK_STATUSES:
+            LOG.error(
+                "Received confirmation that presumably finalized task '%s' "
+                "(status '%s') has just completed successfully from worker "
+                "host '%s'. This should have never happened and indicates "
+                "that there is an inconsistency with the task scheduling. "
+                "Check the rest of the logs for further details. "
+                "The results of this task will NOT be processed.",
+                task.id, task.status, task.host)
+            return
         else:
             if task.status != constants.TASK_STATUS_RUNNING:
                 LOG.warn(
@@ -1944,6 +1973,22 @@ class ConductorServerEndpoint(object):
                 "Only just received error confirmation for force-cancelled "
                 "task '%s'. Leaving marked as force-cancelled.", task.id)
             final_status = constants.TASK_STATUS_FORCE_CANCELED
+            exception_details = (
+                "This task was force-cancelled. Confirmation of its "
+                "cancellation did eventually come in. The exception message "
+                "was: %s" % exception_details)
+        elif task.status in constants.FINALIZED_TASK_STATUSES:
+            LOG.error(
+                "Error confirmation on task '%s' arrived despite it being "
+                "in a terminal state ('%s'). This should never happen and "
+                "indicates an issue with its scheduling/handling. Error "
+                "was: %s", task.id, task.status, exception_details)
+            exception_details = (
+                "Error confirmation came in for this task despite it having "
+                "already been marked as %s. Please notify support of this "
+                "occurence and share the Conductor and Worker logs. "
+                "Error message in confirmation was: %s" % (
+                    task.status, exception_details))
         LOG.debug(
             "Transitioning errored task '%s' from '%s' to '%s'",
             task.id, task.status, final_status)