Merge pull request #160 from aznashwan/separate-minion-manager

Separate Minion Machine management logic into its own sevice.

coriolis/api/v1/migrations.py

@@ -61,9 +61,18 @@ class MigrationController(api_wsgi.Controller):
             'destination_minion_pool_id')
         instance_osmorphing_minion_pool_mappings = migration.get(
             'instance_osmorphing_minion_pool_mappings', {})
-        destination_environment = migration.get(
-            "destination_environment", {})
-        instances = migration["instances"]
+        instances = api_utils.validate_instances_list_for_transfer(
+            migration.get('instances'))
+        extras = [
+            instance
+            for instance in instance_osmorphing_minion_pool_mappings
+            if instance not in instances]
+        if extras:
+            raise ValueError(
+                "One or more instance OSMorphing pool mappings were "
+                "provided for instances (%s) which are not part of the "
+                "migration's declared instances (%s)" % (extras, instances))
+
         notes = migration.get("notes")
         skip_os_morphing = migration.get("skip_os_morphing", False)
         shutdown_instances = migration.get(
@@ -80,20 +89,22 @@ class MigrationController(api_wsgi.Controller):
 
         network_map = migration.get("network_map", {})
         api_utils.validate_network_map(network_map)
-        destination_environment['network_map'] = network_map
-
-        # NOTE(aznashwan): we validate the destination environment for the
-        # import provider before appending the 'storage_mappings' parameter
-        # for plugins with strict property name checks which do not yet
-        # support storage mapping features:
-        self._endpoints_api.validate_target_environment(
-            context, destination_endpoint_id, destination_environment)
 
         # TODO(aznashwan): until the provider plugin interface is updated
         # to have separate 'network_map' and 'storage_mappings' fields,
         # we add them as part of the destination environment:
+        destination_environment = migration.get(
+            "destination_environment", {})
+        destination_environment['network_map'] = network_map
+        self._endpoints_api.validate_target_environment(
+            context, destination_endpoint_id, destination_environment)
+
         storage_mappings = migration.get("storage_mappings", {})
         api_utils.validate_storage_mappings(storage_mappings)
+        # NOTE(aznashwan): we validate the destination environment for the
+        # import provider before appending the 'storage_mappings' parameter
+        # for plugins with strict property name checks which do not yet
+        # support storage mapping features:
         destination_environment['storage_mappings'] = storage_mappings
 
         return (origin_endpoint_id, destination_endpoint_id,

coriolis/api/v1/minion_pool_actions.py

@@ -4,7 +4,7 @@
 from webob import exc
 
 from coriolis import exception
-from coriolis.api.v1.views import minion_pool_tasks_execution_view
+from coriolis.api.v1.views import minion_pool_view
 from coriolis.api import wsgi as api_wsgi
 from coriolis.policies import minion_pools as minion_pool_policies
 from coriolis.minion_pools import api
@@ -15,63 +15,46 @@ class MinionPoolActionsController(api_wsgi.Controller):
         self.minion_pool_api = api.API()
         super(MinionPoolActionsController, self).__init__()
 
-    @api_wsgi.action('set-up-shared-resources')
-    def _set_up_shared_resources(self, req, id, body):
+    @api_wsgi.action('allocate')
+    def _allocate_pool(self, req, id, body):
         context = req.environ['coriolis.context']
         context.can(
             minion_pool_policies.get_minion_pools_policy_label(
-                "set_up_shared_resources"))
+                "allocate"))
         try:
-            return minion_pool_tasks_execution_view.single(
-                req, self.minion_pool_api.set_up_shared_pool_resources(
+            return minion_pool_view.single(
+                req, self.minion_pool_api.allocate_minion_pool(
                     context, id))
         except exception.NotFound as ex:
             raise exc.HTTPNotFound(explanation=ex.msg)
         except exception.InvalidParameterValue as ex:
             raise exc.HTTPNotFound(explanation=ex.msg)
 
-    @api_wsgi.action('tear-down-shared-resources')
-    def _tear_down_shared_resources(self, req, id, body):
+    @api_wsgi.action('refresh')
+    def _refresh_pool(self, req, id, body):
         context = req.environ['coriolis.context']
         context.can(
             minion_pool_policies.get_minion_pools_policy_label(
-                "tear_down_shared_resources"))
-        force = (body["tear-down-shared-resources"] or {}).get(
-            "force", False)
+                "refresh"))
         try:
-            return minion_pool_tasks_execution_view.single(
-                req, self.minion_pool_api.tear_down_shared_pool_resources(
-                    context, id, force=force))
-        except exception.NotFound as ex:
-            raise exc.HTTPNotFound(explanation=ex.msg)
-        except exception.InvalidParameterValue as ex:
-            raise exc.HTTPNotFound(explanation=ex.msg)
-
-    @api_wsgi.action('allocate-machines')
-    def _allocate_pool_machines(self, req, id, body):
-        context = req.environ['coriolis.context']
-        context.can(
-            minion_pool_policies.get_minion_pools_policy_label(
-                "allocate_machines"))
-        try:
-            return minion_pool_tasks_execution_view.single(
-                req, self.minion_pool_api.allocate_machines(
+            return minion_pool_view.single(
+                req, self.minion_pool_api.refresh_minion_pool(
                     context, id))
         except exception.NotFound as ex:
             raise exc.HTTPNotFound(explanation=ex.msg)
         except exception.InvalidParameterValue as ex:
             raise exc.HTTPNotFound(explanation=ex.msg)
 
-    @api_wsgi.action('deallocate-machines')
-    def _deallocate_pool_machines(self, req, id, body):
+    @api_wsgi.action('deallocate')
+    def _deallocate_pool(self, req, id, body):
         context = req.environ['coriolis.context']
         context.can(
             minion_pool_policies.get_minion_pools_policy_label(
-                "deallocate_machines"))
-        force = (body["deallocate-machines"] or {}).get("force", False)
+                "deallocate"))
+        force = (body["deallocate"] or {}).get("force", False)
         try:
-            return minion_pool_tasks_execution_view.single(
-                req, self.minion_pool_api.deallocate_machines(
+            return minion_pool_view.single(
+                req, self.minion_pool_api.deallocate_minion_pool(
                     context, id, force=force))
         except exception.NotFound as ex:
             raise exc.HTTPNotFound(explanation=ex.msg)

coriolis/api/v1/minion_pool_tasks_execution_actions.py

@@ -1,37 +0,0 @@
-# Copyright 2016 Cloudbase Solutions Srl
-# All Rights Reserved.
-
-from webob import exc
-
-from coriolis import exception
-from coriolis.api import wsgi as api_wsgi
-from coriolis.policies \
-    import minion_pool_tasks_executions as pool_execution_policies
-from coriolis.minion_pool_tasks_executions import api
-
-
-class MinionPoolTasksExecutionActionsController(api_wsgi.Controller):
-    def __init__(self):
-        self._minion_pool_tasks_executions_api = api.API()
-        super(MinionPoolTasksExecutionActionsController, self).__init__()
-
-    @api_wsgi.action('cancel')
-    def _cancel(self, req, minion_pool_id, id, body):
-        context = req.environ['coriolis.context']
-        context.can(
-            pool_execution_policies.get_minion_pool_executions_policy_label(
-                'cancel'))
-        try:
-            force = (body["cancel"] or {}).get("force", False)
-
-            self._minion_pool_tasks_executions_api.cancel(
-                context, minion_pool_id, id, force)
-            raise exc.HTTPNoContent()
-        except exception.NotFound as ex:
-            raise exc.HTTPNotFound(explanation=ex.msg)
-        except exception.InvalidParameterValue as ex:
-            raise exc.HTTPNotFound(explanation=ex.msg)
-
-
-def create_resource():
-    return api_wsgi.Resource(MinionPoolTasksExecutionActionsController())

coriolis/api/v1/minion_pool_tasks_executions.py

@@ -1,65 +0,0 @@
-# Copyright 2016 Cloudbase Solutions Srl
-# All Rights Reserved.
-
-from webob import exc
-
-from coriolis.api import wsgi as api_wsgi
-from coriolis.api.v1.views import minion_pool_tasks_execution_view
-from coriolis import exception
-from coriolis.minion_pool_tasks_executions import api
-from coriolis.policies \
-    import minion_pool_tasks_executions as pool_execution_policies
-
-
-class MinionPoolTasksExecutionController(api_wsgi.Controller):
-    def __init__(self):
-        self._pool_tasks_execution_api = api.API()
-        super(MinionPoolTasksExecutionController, self).__init__()
-
-    def show(self, req, minion_pool_id, id):
-        context = req.environ["coriolis.context"]
-        context.can(
-            pool_execution_policies.get_minion_pool_executions_policy_label(
-                "show"))
-        execution = self._pool_tasks_execution_api.get(
-            context, minion_pool_id, id)
-        if not execution:
-            raise exc.HTTPNotFound()
-
-        return minion_pool_tasks_execution_view.single(req, execution)
-
-    def index(self, req, minion_pool_id):
-        context = req.environ["coriolis.context"]
-        context.can(
-            pool_execution_policies.get_minion_pool_executions_policy_label(
-                "list"))
-
-        return minion_pool_tasks_execution_view.collection(
-            req, self._pool_tasks_execution_api.list(
-                context, minion_pool_id, include_tasks=False))
-
-    def detail(self, req, minion_pool_id):
-        context = req.environ["coriolis.context"]
-        context.can(
-            pool_execution_policies.get_minion_pool_executions_policy_label(
-                "show"))
-        return minion_pool_tasks_execution_view.collection(
-            req, self._pool_tasks_execution_api.list(
-                req.environ['coriolis.context'], minion_pool_id,
-                include_tasks=True))
-
-    def delete(self, req, minion_pool_id, id):
-        context = req.environ["coriolis.context"]
-        context.can(
-            pool_execution_policies.get_minion_pool_executions_policy_label(
-                "delete"))
-
-        try:
-            self._pool_tasks_execution_api.delete(context, minion_pool_id, id)
-            raise exc.HTTPNoContent()
-        except exception.NotFound as ex:
-            raise exc.HTTPNotFound(explanation=ex.msg)
-
-
-def create_resource():
-    return api_wsgi.Resource(MinionPoolTasksExecutionController())

coriolis/api/v1/minion_pools.py

@@ -76,15 +76,15 @@ class MinionPoolController(api_wsgi.Controller):
     @api_utils.format_keyerror_message(resource='minion_pool', method='create')
     def _validate_create_body(self, ctxt, body):
         minion_pool = body["minion_pool"]
-        name = minion_pool["pool_name"]
+        name = minion_pool["name"]
         endpoint_id = minion_pool["endpoint_id"]
-        pool_os_type = minion_pool["pool_os_type"]
+        pool_os_type = minion_pool["os_type"]
         if pool_os_type not in constants.VALID_OS_TYPES:
             raise Exception(
                 "The provided pool OS type '%s' is invalid. Must be one "
                 "of the following: %s" % (
                     pool_os_type, constants.VALID_OS_TYPES))
-        pool_platform = minion_pool["pool_platform"]
+        pool_platform = minion_pool["platform"]
         supported_pool_platforms = [
             constants.PROVIDER_PLATFORM_SOURCE,
             constants.PROVIDER_PLATFORM_DESTINATION]
@@ -120,22 +120,26 @@ class MinionPoolController(api_wsgi.Controller):
         self._check_pool_retention_strategy(
             minion_retention_strategy)
         notes = minion_pool.get("notes")
+
+        skip_allocation = minion_pool.get('skip_allocation', False)
         return (
             name, endpoint_id, pool_platform, pool_os_type,
             environment_options, minimum_minions, maximum_minions,
-            minion_max_idle_time, minion_retention_strategy, notes)
+            minion_max_idle_time, minion_retention_strategy, notes,
+            skip_allocation)
 
     def create(self, req, body):
         context = req.environ["coriolis.context"]
         context.can(pools_policies.get_minion_pools_policy_label("create"))
         (name, endpoint_id, pool_platform, pool_os_type, environment_options,
          minimum_minions, maximum_minions, minion_max_idle_time,
-         minion_retention_strategy, notes) = (
+         minion_retention_strategy, notes, skip_allocation) = (
             self._validate_create_body(context, body))
         return minion_pool_view.single(req, self._minion_pool_api.create(
             context, name, endpoint_id, pool_platform, pool_os_type,
             environment_options, minimum_minions, maximum_minions,
-            minion_max_idle_time, minion_retention_strategy, notes=notes))
+            minion_max_idle_time, minion_retention_strategy, notes=notes,
+            skip_allocation=skip_allocation))
 
     @api_utils.format_keyerror_message(resource='minion_pool', method='update')
     def _validate_update_body(self, id, context, body):
@@ -143,13 +147,13 @@ class MinionPoolController(api_wsgi.Controller):
         if 'endpoint_id' in minion_pool:
             raise Exception(
                 "The 'endpoint_id' of a minion pool cannot be updated.")
-        if 'pool_platform' in minion_pool:
+        if 'platform' in minion_pool:
             raise Exception(
-                "The 'pool_platform' of a minion pool cannot be updated.")
+                "The 'platform' of a minion pool cannot be updated.")
         vals = {k: minion_pool[k] for k in minion_pool.keys() &
-                {"pool_name", "environment_options", "minimum_minions",
+                {"name", "environment_options", "minimum_minions",
                  "maximum_minions", "minion_max_idle_time",
-                 "minion_retention_strategy", "notes", "pool_os_type"}}
+                 "minion_retention_strategy", "notes", "os_type"}}
         if 'minion_retention_strategy' in vals:
             self._check_pool_retention_strategy(
                 vals['minion_retention_strategy'])
@@ -167,24 +171,20 @@ class MinionPoolController(api_wsgi.Controller):
                 vals.get('minion_max_idle_time'))
 
             if 'environment_options' in vals:
-                if minion_pool['pool_platform'] == (
+                if minion_pool['platform'] == (
                         constants.PROVIDER_PLATFORM_SOURCE):
                     self._endpoints_api.validate_endpoint_source_minion_pool_options(
-                        # TODO(aznashwan): remove endpoint ID fields redundancy
-                        # once DB models are overhauled:
-                        context, minion_pool['origin_endpoint_id'],
+                        context, minion_pool['endpoint_id'],
                         vals['environment_options'])
-                elif minion_pool['pool_platform'] == (
+                elif minion_pool['platform'] == (
                         constants.PROVIDER_PLATFORM_DESTINATION):
                     self._endpoints_api.validate_endpoint_destination_minion_pool_options(
-                        # TODO(aznashwan): remove endpoint ID fields redundancy
-                        # once DB models are overhauled:
-                        context, minion_pool['origin_endpoint_id'],
+                        context, minion_pool['endpoint_id'],
                         vals['environment_options'])
                 else:
                     raise Exception(
                         "Unknown pool platform: %s" % minion_pool[
-                            'pool_platform'])
+                            'platform'])
         return vals
 
     def update(self, req, id, body):

coriolis/api/v1/replicas.py

@@ -60,23 +60,39 @@ class ReplicaController(api_wsgi.Controller):
         destination_endpoint_id = replica["destination_endpoint_id"]
         destination_environment = replica.get(
             "destination_environment", {})
-        instances = replica["instances"]
+        instances = api_utils.validate_instances_list_for_transfer(
+            replica.get('instances'))
+
         notes = replica.get("notes")
 
         source_environment = replica.get("source_environment", {})
         self._endpoints_api.validate_source_environment(
             context, origin_endpoint_id, source_environment)
 
-        network_map = replica.get("network_map", {})
-        api_utils.validate_network_map(network_map)
-        destination_environment['network_map'] = network_map
-
         origin_minion_pool_id = replica.get(
             'origin_minion_pool_id')
         destination_minion_pool_id = replica.get(
             'destination_minion_pool_id')
         instance_osmorphing_minion_pool_mappings = replica.get(
             'instance_osmorphing_minion_pool_mappings', {})
+        extras = [
+            instance
+            for instance in instance_osmorphing_minion_pool_mappings
+            if instance not in instances]
+        if extras:
+            raise ValueError(
+                "One or more instance OSMorphing pool mappings were "
+                "provided for instances (%s) which are not part of the "
+                "Replicas's declared instances (%s)" % (extras, instances))
+
+        # TODO(aznashwan): until the provider plugin interface is updated
+        # to have separate 'network_map' and 'storage_mappings' fields,
+        # we add them as part of the destination environment:
+        network_map = replica.get("network_map", {})
+        api_utils.validate_network_map(network_map)
+        destination_environment['network_map'] = network_map
+        self._endpoints_api.validate_target_environment(
+            context, destination_endpoint_id, destination_environment)
 
         user_scripts = replica.get('user_scripts', {})
         api_utils.validate_user_scripts(user_scripts)
@@ -87,15 +103,9 @@ class ReplicaController(api_wsgi.Controller):
         # import provider before appending the 'storage_mappings' parameter
         # for plugins with strict property name checks which do not yet
         # support storage mapping features:
-        self._endpoints_api.validate_target_environment(
-            context, destination_endpoint_id, destination_environment)
-
         storage_mappings = replica.get("storage_mappings", {})
         api_utils.validate_storage_mappings(storage_mappings)
 
-        # TODO(aznashwan): until the provider plugin interface is updated
-        # to have separate 'network_map' and 'storage_mappings' fields,
-        # we add them as part of the destination environment:
         destination_environment['storage_mappings'] = storage_mappings
 
         return (origin_endpoint_id, destination_endpoint_id,

coriolis/api/v1/router.py

@@ -18,8 +18,6 @@ from coriolis.api.v1 import migration_actions
 from coriolis.api.v1 import migrations
 from coriolis.api.v1 import minion_pools
 from coriolis.api.v1 import minion_pool_actions
-from coriolis.api.v1 import minion_pool_tasks_executions
-from coriolis.api.v1 import minion_pool_tasks_execution_actions
 from coriolis.api.v1 import provider_schemas
 from coriolis.api.v1 import providers
 from coriolis.api.v1 import regions
@@ -81,26 +79,6 @@ class APIRouter(api.APIRouter):
                        action='action',
                        conditions={'method': 'POST'})
 
-        self.resources['minion_pool_tasks_executions'] = \
-            minion_pool_tasks_executions.create_resource()
-        mapper.resource('minion_pools', 'minion_pools/{minion_pool_id}/executions',
-                        controller=self.resources['minion_pool_tasks_executions'],
-                        collection={'detail': 'GET'},
-                        member={'action': 'POST'})
-
-        minion_pool_tasks_execution_actions_resource = \
-            minion_pool_tasks_execution_actions.create_resource()
-        self.resources['minion_pool_tasks_execution_actions'] = \
-            minion_pool_tasks_execution_actions_resource
-        pool_execution_path = (
-            '/{project_id}/minion_pools/{minion_pool_id}/executions/{id}')
-        mapper.connect('minion_pool_tasks_execution_actions',
-                       pool_execution_path + '/actions',
-                       controller=self.resources[
-                           'minion_pool_tasks_execution_actions'],
-                       action='action',
-                       conditions={'method': 'POST'})
-
         self.resources['endpoint_source_minion_pool_options'] = \
             endpoint_source_minion_pool_options.create_resource()
         mapper.resource('minion_pool_options',

coriolis/api/v1/utils.py

@@ -125,3 +125,29 @@ def normalize_user_scripts(user_scripts, instances):
             user_scripts.pop(instance, None)
 
     return user_scripts
+
+
+def validate_instances_list_for_transfer(instances):
+    if not instances:
+        raise exception.InvalidInput(
+            "No instance identifiers provided for transfer action.")
+
+    if not isinstance(instances, list):
+        raise exception.InvalidInput(
+            "Instances must be a list. Got type %s: %s" % (
+                type(instances), instances))
+
+    appearances = {}
+    for instance in instances:
+        if instance in appearances:
+            appearances[instance] = appearances[instance] + 1
+        else:
+            appearances[instance] = 1
+    duplicates = {
+        inst: count for (inst, count) in appearances.items() if count > 1}
+    if duplicates:
+        raise exception.InvalidInput(
+            "Transfer action instances (%s) list contained duplicates: %s " % (
+                instances, duplicates))
+
+    return instances

coriolis/api/v1/views/minion_pool_tasks_execution_view.py

@@ -1,45 +0,0 @@
-# Copyright 2016 Cloudbase Solutions Srl
-# All Rights Reserved.
-
-import itertools
-
-from coriolis import constants
-from coriolis import utils
-
-
-def _sort_tasks(tasks, filter_error_only_tasks=True):
-    """ Sorts the given list of dicts representing tasks.
-    Tasks are sorted primarily based on their index.
-    """
-    if filter_error_only_tasks:
-        tasks = [
-            t for t in tasks
-            if t['status'] != (
-                constants.TASK_STATUS_ON_ERROR_ONLY)]
-    return sorted(
-        tasks, key=lambda t: t.get('index', 0))
-
-
-def format_minion_pool_tasks_execution(req, execution, keys=None):
-    def transform(key, value):
-        if keys and key not in keys:
-            return
-        yield (key, value)
-
-    if "tasks" in execution:
-        execution["tasks"] = _sort_tasks(execution["tasks"])
-
-    execution_dict = dict(itertools.chain.from_iterable(
-        transform(k, v) for k, v in execution.items()))
-
-    return execution_dict
-
-
-def single(req, execution):
-    return {"execution": format_minion_pool_tasks_execution(req, execution)}
-
-
-def collection(req, executions):
-    formatted_executions = [
-        format_minion_pool_tasks_execution(req, m) for m in executions]
-    return {'executions': formatted_executions}

coriolis/api/v1/views/minion_pool_view.py

@@ -13,21 +13,14 @@ def _format_minion_pool(req, minion_pool, keys=None):
     minion_pool_dict = dict(itertools.chain.from_iterable(
         transform(k, v) for k, v in minion_pool.items()))
 
-    # TODO(aznashwan): remove these redundancies once the base
-    # DB action model hirearchy will be overhauled:
-    for key in ["origin_endpoint_id", "destination_endpoint_id"]:
-        if key in minion_pool_dict:
-            minion_pool_dict["endpoint_id"] = minion_pool_dict.pop(key)
-    for key in ["source_environment", "destination_environment"]:
-        if key in minion_pool_dict:
-            minion_pool_dict["environment_options"] = minion_pool_dict.pop(key)
-
     def _hide_minion_creds(minion_conn):
-        if 'pkey' in minion_conn:
+        if not minion_conn:
+            return minion_conn
+        if minion_conn.get('pkey'):
             minion_conn['pkey'] = '***'
-        if 'password' in minion_conn:
+        if minion_conn.get('password'):
             minion_conn['password'] = '***'
-        if 'certificates' in minion_conn:
+        if minion_conn.get('certificates'):
             for key in minion_conn['certificates']:
                 minion_conn['certificates'][key] = '***'
     if 'minion_machines' in minion_pool_dict:
@@ -35,8 +28,9 @@ def _format_minion_pool(req, minion_pool, keys=None):
             if 'connection_info' in machine:
                 _hide_minion_creds(machine['connection_info'])
             if 'backup_writer_connection_info' in machine:
-                if 'connection_details' in machine[
-                        'backup_writer_connection_info']:
+                if machine.get('backup_writer_connection_info') and (
+                        'connection_details' in machine[
+                            'backup_writer_connection_info']):
                     _hide_minion_creds(
                         machine['backup_writer_connection_info'][
                             'connection_details'])

coriolis/cmd/minion_manager.py

@@ -0,0 +1,31 @@
+# Copyright 2020 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import sys
+
+from oslo_config import cfg
+
+from coriolis import constants
+from coriolis import service
+from coriolis import utils
+from coriolis.minion_manager.rpc import server as rpc_server
+
+CONF = cfg.CONF
+
+
+def main():
+    CONF(sys.argv[1:], project='coriolis',
+         version="1.0.0")
+    utils.setup_logging()
+
+    server = service.MessagingService(
+        constants.MINION_MANAGER_MAIN_MESSAGING_TOPIC,
+        [rpc_server.MinionManagerServerEndpoint()],
+        rpc_server.VERSION, worker_count=1)
+    launcher = service.service.launch(
+        CONF, server, workers=server.get_workers_count())
+    launcher.wait()
+
+
+if __name__ == "__main__":
+    main()

coriolis/conductor/rpc/client.py

@@ -2,12 +2,16 @@
 # All Rights Reserved.
 
 from oslo_config import cfg
+from oslo_log import log as logging
 import oslo_messaging as messaging
 
 from coriolis import constants
+from coriolis import events
 from coriolis import rpc
 
+
 VERSION = "1.0"
+LOG = logging.getLogger(__name__)
 
 conductor_opts = [
     cfg.IntOpt("conductor_rpc_timeout",
@@ -255,45 +259,50 @@ class ConductorClient(rpc.BaseRPCClient):
             ctxt, 'cancel_migration', migration_id=migration_id, force=force)
 
     def set_task_host(self, ctxt, task_id, host):
-        self._call(
+        self._cast(
             ctxt, 'set_task_host', task_id=task_id, host=host)
 
     def set_task_process(self, ctxt, task_id, process_id):
-        self._call(
+        self._cast(
             ctxt, 'set_task_process', task_id=task_id, process_id=process_id)
 
     def task_completed(self, ctxt, task_id, task_result):
-        self._call(
+        self._cast(
             ctxt, 'task_completed', task_id=task_id, task_result=task_result)
 
     def confirm_task_cancellation(self, ctxt, task_id, cancellation_details):
-        self._call(
+        self._cast(
             ctxt, 'confirm_task_cancellation', task_id=task_id,
             cancellation_details=cancellation_details)
 
     def set_task_error(self, ctxt, task_id, exception_details):
-        self._call(
+        self._cast(
             ctxt, 'set_task_error', task_id=task_id,
             exception_details=exception_details)
 
-    def task_event(self, ctxt, task_id, level, message):
-        self._cast(
-            ctxt, 'task_event', task_id=task_id, level=level, message=message)
-
-    def add_task_progress_update(self, ctxt, task_id, total_steps, message):
+    def add_task_event(self, ctxt, task_id, level, message):
         self._cast(
+            ctxt, 'add_task_event', task_id=task_id, level=level, message=message)
+
+    def add_task_progress_update(
+            self, ctxt, task_id, message, initial_step=0, total_steps=0,
+            return_event=False):
+        operation = self._cast
+        if return_event:
+            operation = self._call
+        return operation(
             ctxt, 'add_task_progress_update', task_id=task_id,
-            total_steps=total_steps, message=message)
+            message=message, initial_step=initial_step,
+            total_steps=total_steps)
 
-    def update_task_progress_update(self, ctxt, task_id, step,
-                                    total_steps, message):
+    def update_task_progress_update(
+            self, ctxt, task_id, progress_update_index, new_current_step,
+            new_total_steps=None, new_message=None):
         self._cast(
             ctxt, 'update_task_progress_update', task_id=task_id,
-            step=step, total_steps=total_steps, message=message)
-
-    def get_task_progress_step(self, ctxt, task_id):
-        return self._call(
-            ctxt, 'get_task_progress_step', task_id=task_id)
+            progress_update_index=progress_update_index,
+            new_current_step=new_current_step, new_total_steps=new_total_steps,
+            new_message=new_message)
 
     def create_replica_schedule(self, ctxt, replica_id,
                                 schedule, enabled, exp_date,
@@ -403,104 +412,71 @@ class ConductorClient(rpc.BaseRPCClient):
         return self._call(
             ctxt, 'delete_service', service_id=service_id)
 
-    def create_minion_pool(
-            self, ctxt, name, endpoint_id, pool_platform, pool_os_type,
-            environment_options, minimum_minions, maximum_minions,
-            minion_max_idle_time, minion_retention_strategy, notes=None):
-        return self._call(
-            ctxt, 'create_minion_pool', name=name, endpoint_id=endpoint_id,
-            pool_platform=pool_platform, pool_os_type=pool_os_type,
-            environment_options=environment_options,
-            minimum_minions=minimum_minions,
-            maximum_minions=maximum_minions,
-            minion_max_idle_time=minion_max_idle_time,
-            minion_retention_strategy=minion_retention_strategy,
-            notes=notes)
-
-    def set_up_shared_minion_pool_resources(self, ctxt, minion_pool_id):
-        return self._call(
-            ctxt, "set_up_shared_minion_pool_resources",
-            minion_pool_id=minion_pool_id)
-
-    def tear_down_shared_minion_pool_resources(
-            self, ctxt, minion_pool_id, force=False):
-        return self._call(
-            ctxt, "tear_down_shared_minion_pool_resources",
-            minion_pool_id=minion_pool_id, force=force)
-
-    def allocate_minion_pool_machines(self, ctxt, minion_pool_id):
-        return self._call(
-            ctxt, "allocate_minion_pool_machines",
-            minion_pool_id=minion_pool_id)
-
-    def deallocate_minion_pool_machines(
-            self, ctxt, minion_pool_id, force=False):
-        return self._call(
-            ctxt, "deallocate_minion_pool_machines",
-            minion_pool_id=minion_pool_id,
-            force=force)
-
-    def get_minion_pools(self, ctxt):
-        return self._call(ctxt, 'get_minion_pools')
-
-    def get_minion_pool(self, ctxt, minion_pool_id):
-        return self._call(
-            ctxt, 'get_minion_pool', minion_pool_id=minion_pool_id)
-
-    def update_minion_pool(self, ctxt, minion_pool_id, updated_values):
-        return self._call(
-            ctxt, 'update_minion_pool',
-            minion_pool_id=minion_pool_id, updated_values=updated_values)
-
-    def delete_minion_pool(self, ctxt, minion_pool_id):
-        return self._call(
-            ctxt, 'delete_minion_pool', minion_pool_id=minion_pool_id)
-
-    def get_minion_pool_lifecycle_executions(
-            self, ctxt, minion_pool_id, include_tasks=False):
-        return self._call(
-            ctxt, 'get_minion_pool_lifecycle_executions',
-            minion_pool_id=minion_pool_id, include_tasks=include_tasks)
-
-    def get_minion_pool_lifecycle_execution(
-            self, ctxt, minion_pool_id, execution_id):
-        return self._call(
-            ctxt, 'get_minion_pool_lifecycle_execution',
-            minion_pool_id=minion_pool_id, execution_id=execution_id)
-
-    def delete_minion_pool_lifecycle_execution(
-            self, ctxt, minion_pool_id, execution_id):
-        return self._call(
-            ctxt, 'delete_minion_pool_lifecycle_execution',
-            minion_pool_id=minion_pool_id, execution_id=execution_id)
-
-    def cancel_minion_pool_lifecycle_execution(
-            self, ctxt, minion_pool_id, execution_id, force):
-        return self._call(
-            ctxt, 'cancel_minion_pool_lifecycle_execution',
-            minion_pool_id=minion_pool_id, execution_id=execution_id,
-            force=force)
-
-    def get_endpoint_source_minion_pool_options(
-            self, ctxt, endpoint_id, env, option_names):
-        return self._call(
-            ctxt, 'get_endpoint_source_minion_pool_options',
-            endpoint_id=endpoint_id, env=env, option_names=option_names)
+    def confirm_replica_minions_allocation(
+            self, ctxt, replica_id, minion_machine_allocations):
+        self._call(
+            ctxt, 'confirm_replica_minions_allocation', replica_id=replica_id,
+            minion_machine_allocations=minion_machine_allocations)
 
-    def get_endpoint_destination_minion_pool_options(
-            self, ctxt, endpoint_id, env, option_names):
-        return self._call(
-            ctxt, 'get_endpoint_destination_minion_pool_options',
-            endpoint_id=endpoint_id, env=env, option_names=option_names)
+    def report_replica_minions_allocation_error(
+            self, ctxt, replica_id, minion_allocation_error_details):
+        self._call(
+            ctxt, 'report_replica_minions_allocation_error', replica_id=replica_id,
+            minion_allocation_error_details=minion_allocation_error_details)
 
-    def validate_endpoint_source_minion_pool_options(
-            self, ctxt, endpoint_id, pool_environment):
-        return self._call(
-            ctxt, 'validate_endpoint_source_minion_pool_options',
-            endpoint_id=endpoint_id, pool_environment=pool_environment)
+    def confirm_migration_minions_allocation(
+            self, ctxt, migration_id, minion_machine_allocations):
+        self._call(
+            ctxt, 'confirm_migration_minions_allocation',
+            migration_id=migration_id,
+            minion_machine_allocations=minion_machine_allocations)
 
-    def validate_endpoint_destination_minion_pool_options(
-            self, ctxt, endpoint_id, pool_environment):
-        return self._call(
-            ctxt, 'validate_endpoint_destination_minion_pool_options',
-            endpoint_id=endpoint_id, pool_environment=pool_environment)
+    def report_migration_minions_allocation_error(
+            self, ctxt, migration_id, minion_allocation_error_details):
+        self._call(
+            ctxt, 'report_migration_minions_allocation_error',
+            migration_id=migration_id,
+            minion_allocation_error_details=minion_allocation_error_details)
+
+
+class ConductorTaskRpcEventHandler(events.BaseEventHandler):
+    def __init__(self, ctxt, task_id):
+        self._ctxt = ctxt
+        self._task_id = task_id
+        self._rpc_conductor_client_instance = None
+
+    @property
+    def _rpc_conductor_client(self):
+        # NOTE(aznashwan): it is unsafe to fork processes with pre-instantiated
+        # oslo_messaging clients as the underlying eventlet thread queues will
+        # be invalidated.
+        if self._rpc_conductor_client_instance is None:
+            self._rpc_conductor_client_instance = ConductorClient()
+        return self._rpc_conductor_client_instance
+
+    @classmethod
+    def get_progress_update_identifier(self, progress_update):
+        return progress_update['index']
+
+    def add_progress_update(
+            self, message, initial_step=0, total_steps=0, return_event=False):
+        LOG.info(
+            "Sending progress update for task '%s' to conductor: %s",
+            self._task_id, message)
+        return self._rpc_conductor_client.add_task_progress_update(
+            self._ctxt, self._task_id, message, initial_step=initial_step,
+            total_steps=total_steps, return_event=return_event)
+
+    def update_progress_update(
+            self, update_identifier, new_current_step,
+            new_total_steps=None, new_message=None):
+        LOG.info(
+            "Updating progress update '%s' for task '%s' with new step %s",
+            update_identifier, self._task_id, new_current_step)
+        self._rpc_conductor_client.update_task_progress_update(
+            self._ctxt, self._task_id, update_identifier, new_current_step,
+            new_total_steps=new_total_steps, new_message=new_message)
+
+    def add_event(self, message, level=constants.TASK_EVENT_INFO):
+        self._rpc_conductor_client.add_task_event(
+            self._ctxt, self._task_id, level, message)

coriolis/constants.py

@@ -11,10 +11,13 @@ EXECUTION_STATUS_DEADLOCKED = "DEADLOCKED"
 EXECUTION_STATUS_CANCELED = "CANCELED"
 EXECUTION_STATUS_CANCELLING = "CANCELLING"
 EXECUTION_STATUS_CANCELED_FOR_DEBUGGING = "CANCELED_FOR_DEBUGGING"
+EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS = "AWAITING_MINION_ALLOCATIONS"
+EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS = "ERROR_ALLOCATING_MINIONS"
 
 ACTIVE_EXECUTION_STATUSES = [
     EXECUTION_STATUS_RUNNING,
-    EXECUTION_STATUS_CANCELLING
+    EXECUTION_STATUS_CANCELLING,
+    EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS
 ]
 
 FINALIZED_EXECUTION_STATUSES = [
@@ -22,7 +25,8 @@ FINALIZED_EXECUTION_STATUSES = [
     EXECUTION_STATUS_CANCELED,
     EXECUTION_STATUS_ERROR,
     EXECUTION_STATUS_CANCELED_FOR_DEBUGGING,
-    EXECUTION_STATUS_DEADLOCKED
+    EXECUTION_STATUS_DEADLOCKED,
+    EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS
 ]
 
 TASK_STATUS_SCHEDULED = "SCHEDULED"
@@ -98,7 +102,6 @@ TASK_TYPE_DEPLOY_OS_MORPHING_RESOURCES = "DEPLOY_OS_MORPHING_RESOURCES"
 TASK_TYPE_OS_MORPHING = "OS_MORPHING"
 TASK_TYPE_DELETE_OS_MORPHING_RESOURCES = "DELETE_OS_MORPHING_RESOURCES"
 
-
 TASK_TYPE_GET_INSTANCE_INFO = "GET_INSTANCE_INFO"
 TASK_TYPE_DEPLOY_REPLICA_DISKS = "DEPLOY_REPLICA_DISKS"
 TASK_TYPE_DELETE_REPLICA_SOURCE_DISK_SNAPSHOTS = (
@@ -172,6 +175,32 @@ TASK_TYPE_RELEASE_SOURCE_MINION = "RELEASE_SOURCE_MINION"
 TASK_TYPE_RELEASE_DESTINATION_MINION = "RELEASE_DESTINATION_MINION"
 TASK_TYPE_RELEASE_OSMORPHING_MINION = "RELEASE_OSMORPHING_MINION"
 TASK_TYPE_COLLECT_OSMORPHING_INFO = "COLLECT_OS_MORPHING_INFO"
+TASK_TYPE_HEALTHCHECK_SOURCE_MINION = "HEALTHCHECK_SOURCE_MINION"
+TASK_TYPE_HEALTHCHECK_DESTINATION_MINION = "HEALTHCHECK_DESTINATION_MINION"
+TASK_TYPE_POWER_ON_SOURCE_MINION = "POWER_ON_SOURCE_MINION"
+TASK_TYPE_POWER_OFF_SOURCE_MINION = "POWER_OFF_SOURCE_MINION"
+TASK_TYPE_POWER_ON_DESTINATION_MINION = "POWER_ON_DESTINATION_MINION"
+TASK_TYPE_POWER_OFF_DESTINATION_MINION = "POWER_OFF_DESTINATION_MINION"
+
+
+MINION_POOL_OPERATIONS_TASKS = [
+    TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_OPTIONS,
+    TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_OPTIONS,
+    TASK_TYPE_SET_UP_SOURCE_POOL_SHARED_RESOURCES,
+    TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES,
+    TASK_TYPE_SET_UP_DESTINATION_POOL_SHARED_RESOURCES,
+    TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES,
+    TASK_TYPE_CREATE_SOURCE_MINION_MACHINE,
+    TASK_TYPE_DELETE_SOURCE_MINION_MACHINE,
+    TASK_TYPE_CREATE_DESTINATION_MINION_MACHINE,
+    TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE,
+    TASK_TYPE_HEALTHCHECK_SOURCE_MINION,
+    TASK_TYPE_HEALTHCHECK_DESTINATION_MINION,
+    TASK_TYPE_POWER_ON_SOURCE_MINION,
+    TASK_TYPE_POWER_OFF_SOURCE_MINION,
+    TASK_TYPE_POWER_ON_DESTINATION_MINION,
+    TASK_TYPE_POWER_OFF_DESTINATION_MINION
+]
 
 TASK_PLATFORM_SOURCE = "source"
 TASK_PLATFORM_DESTINATION = "destination"
@@ -225,6 +254,10 @@ TASK_EVENT_INFO = "INFO"
 TASK_EVENT_WARNING = "WARNING"
 TASK_EVENT_ERROR = "ERROR"
 
+MINION_POOL_EVENT_INFO = "INFO"
+MINION_POOL_EVENT_WARNING = "WARNING"
+MINION_POOL_EVENT_ERROR = "ERROR"
+
 OS_TYPE_BSD = "bsd"
 OS_TYPE_LINUX = "linux"
 OS_TYPE_OS_X = "osx"
@@ -248,6 +281,9 @@ VALID_COMPRESSION_FORMATS = [
     COMPRESSION_FORMAT_ZLIB
 ]
 
+TRANSFER_ACTION_TYPE_MIGRATION = "migration"
+TRANSFER_ACTION_TYPE_REPLICA = "replica"
+
 EXECUTION_TYPE_REPLICA_EXECUTION = "replica_execution"
 EXECUTION_TYPE_REPLICA_DISKS_DELETE = "replica_disks_delete"
 EXECUTION_TYPE_REPLICA_DEPLOY = "replica_deploy"
@@ -263,15 +299,8 @@ EXECUTION_TYPE_MINION_POOL_ALLOCATE_MINIONS = "minion_pool_allocate_minions"
 EXECUTION_TYPE_MINION_POOL_DEALLOCATE_MINIONS = (
     "minion_pool_deallocate_minions")
 
-MINION_POOL_EXECUTION_TYPES = [
-    EXECUTION_TYPE_MINION_POOL_MAINTENANCE,
-    EXECUTION_TYPE_MINION_POOL_UPDATE,
-    EXECUTION_TYPE_MINION_POOL_SET_UP_SHARED_RESOURCES,
-    EXECUTION_TYPE_MINION_POOL_TEAR_DOWN_SHARED_RESOURCES,
-    EXECUTION_TYPE_MINION_POOL_ALLOCATE_MINIONS,
-    EXECUTION_TYPE_MINION_POOL_DEALLOCATE_MINIONS]
-
 TASK_LOCK_NAME_FORMAT = "task-%s"
+TASKFLOW_LOCK_NAME_FORMAT = "taskflow-%s"
 EXECUTION_LOCK_NAME_FORMAT = "execution-%s"
 ENDPOINT_LOCK_NAME_FORMAT = "endpoint-%s"
 MIGRATION_LOCK_NAME_FORMAT = "migration-%s"
@@ -280,6 +309,7 @@ SCHEDULE_LOCK_NAME_FORMAT = "schedule-%s"
 REGION_LOCK_NAME_FORMAT = "region-%s"
 SERVICE_LOCK_NAME_FORMAT = "service-%s"
 MINION_POOL_LOCK_NAME_FORMAT = "minion-pool-%s"
+MINION_MACHINE_LOCK_NAME_FORMAT = "minion-pool-%s-machine-%s"
 
 EXECUTION_TYPE_TO_ACTION_LOCK_NAME_FORMAT_MAP = {
     EXECUTION_TYPE_MIGRATION: MIGRATION_LOCK_NAME_FORMAT,
@@ -306,30 +336,46 @@ CONDUCTOR_MAIN_MESSAGING_TOPIC = "coriolis_conductor"
 WORKER_MAIN_MESSAGING_TOPIC = "coriolis_worker"
 SCHEDULER_MAIN_MESSAGING_TOPIC = "coriolis_scheduler"
 REPLICA_CRON_MAIN_MESSAGING_TOPIC = "coriolis_replica_cron_worker"
+MINION_MANAGER_MAIN_MESSAGING_TOPIC = "coriolis_minion_manager"
 
 MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE = "delete"
 MINION_POOL_MACHINE_RETENTION_STRATEGY_POWEROFF = "poweroff"
 
 MINION_POOL_STATUS_UNKNOWN = "UNKNOWN"
 MINION_POOL_STATUS_ERROR = "ERROR"
-MINION_POOL_STATUS_UNINITIALIZED = "UNINITIALIZED"
-MINION_POOL_STATUS_UNINITIALIZING = "UNINITIALIZING"
-MINION_POOL_STATUS_INITIALIZING = "INITIALIZING"
-MINION_POOL_STATUS_DEALLOCATING = "DEALLOCATING"
 MINION_POOL_STATUS_DEALLOCATED = "DEALLOCATED"
-MINION_POOL_STATUS_ALLOCATING = "ALLOCATING"
+MINION_POOL_STATUS_VALIDATING_INPUTS = "VALIDATING_INPUTS"
+MINION_POOL_STATUS_ALLOCATING_SHARED_RESOURCES = "ALLOCATING_SHARED_RESOURCES"
+MINION_POOL_STATUS_ALLOCATING_MACHINES = "ALLOCATING_MACHINES"
+MINION_POOL_STATUS_DEALLOCATING_MACHINES = "DEALLOCATING_MACHINES"
+MINION_POOL_STATUS_DEALLOCATING_SHARED_RESOURCES = (
+    "DEALLOCATING_SHARED_RESOURCES")
 MINION_POOL_STATUS_ALLOCATED = "ALLOCATED"
-MINION_POOL_STATUS_RECONFIGURING = "RECONFIGURING"
+MINION_POOL_STATUS_POOL_MAINTENANCE = "IN_MAINTENANCE"
 
 ACTIVE_MINION_POOL_STATUSES = [
-    MINION_POOL_STATUS_INITIALIZING,
-    MINION_POOL_STATUS_ALLOCATING,
-    MINION_POOL_STATUS_DEALLOCATING,
-    MINION_POOL_STATUS_UNINITIALIZING]
+    MINION_POOL_STATUS_VALIDATING_INPUTS,
+    MINION_POOL_STATUS_ALLOCATING_SHARED_RESOURCES,
+    MINION_POOL_STATUS_ALLOCATING_MACHINES,
+    MINION_POOL_STATUS_DEALLOCATING_MACHINES,
+    MINION_POOL_STATUS_DEALLOCATING_SHARED_RESOURCES]
 
 MINION_MACHINE_IDENTIFIER_FORMAT = "coriolis-pool-%(pool_id)s-minion-%(minion_id)s"
-MINION_MACHINE_STATUS_UNKNOWN = "UNKNOWN"
 MINION_MACHINE_STATUS_UNINITIALIZED = "UNINITIALIZED"
-MINION_MACHINE_STATUS_RECONFIGURING = "RECONFIGURING"
+MINION_MACHINE_STATUS_HEALTHCHECKING = "HEALTHCHECKING"
+MINION_MACHINE_STATUS_ALLOCATING = "ALLOCATING"
+MINION_MACHINE_STATUS_DEALLOCATING = "DEALLOCATING"
+MINION_MACHINE_STATUS_ERROR = "ERROR"
+MINION_MACHINE_STATUS_POWERING_OFF = "POWERING_OFF"
+MINION_MACHINE_STATUS_POWER_ERROR = "POWER_ERROR"
+MINION_MACHINE_STATUS_ERROR_DEPLOYING = "ERROR_DEPLOYING"
 MINION_MACHINE_STATUS_AVAILABLE = "AVAILABLE"
-MINION_MACHINE_STATUS_ALLOCATED = "ALLOCATED"
+MINION_MACHINE_STATUS_IN_USE = "IN_USE"
+MINION_MACHINE_STATUS_RESERVED = "RESERVED"
+
+MINION_MACHINE_POWER_STATUS_UNKNOWN = "UNKNOWN"
+MINION_MACHINE_POWER_STATUS_UNINITIALIZED = "UNINITIALIZED"
+MINION_MACHINE_POWER_STATUS_POWERED_ON = "POWERED_ON"
+MINION_MACHINE_POWER_STATUS_POWERED_OFF = "POWERED_OFF"
+MINION_MACHINE_POWER_STATUS_POWERING_ON = "POWERING_ON"
+MINION_MACHINE_POWER_STATUS_POWERING_OFF = "POWERING_OFF"

coriolis/replica_cron/cron.py → coriolis/cron/cron.py

@@ -155,17 +155,31 @@ class Cron(object):
         if not isinstance(job, CronJob):
             raise ValueError("Invalid job class")
         name = job.name
+        LOG.debug("Registering cron job with name '%s'", name)
         with self._semaphore:
             self._jobs[name] = job
 
     def unregister(self, name):
         job = self._jobs.get(name)
         if job:
+            LOG.debug("Unregistering cron job with name '%s'", name)
             with self._semaphore:
                 del self._jobs[name]
 
+    def unregister_jobs_with_prefix(self, prefix):
+        jobs = [
+            job for job in self._jobs
+            if job.startswith(prefix)]
+        if jobs:
+            LOG.debug(
+                "Unregistering the following cron jobs based on "
+                "the requested prefix ('%s'): %s", prefix, jobs)
+            with self._semaphore:
+                for job in jobs:
+                    del self._jobs[job]
+
     def _check_jobs(self):
-        LOG.debug("Checking jobs")
+        LOG.debug("Checking cron jobs")
         jobs = self._jobs.copy()
         job_nr = len(jobs)
         spawned = 0
@@ -206,8 +220,8 @@ class Cron(object):
                     "job_err": error})
             if result:
                 LOG.info("Job %(desc)s returned: %(ret)r" % {
-                    "job_desc": desc,
-                    "job_ret": result})
+                    "desc": desc,
+                    "ret": result})
 
     def _janitor(self):
         # remove expired jobs from memory. The check for expired

coriolis/db/api.py

@@ -1,6 +1,8 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+import uuid
+
 from oslo_config import cfg
 from oslo_db import api as db_api
 from oslo_db import options as db_options
@@ -709,56 +711,162 @@ def get_task(context, task_id):
 @enginefacade.writer
 def add_task_event(context, task_id, level, message):
     task_event = models.TaskEvent()
+    task_event.id = str(uuid.uuid4())
+    task_event.index = 0
+    last_event = _get_last_task_event(context, task_id)
+    if last_event:
+        task_event.index = last_event.index + 1
     task_event.task_id = task_id
     task_event.level = level
     task_event.message = message
     _session(context).add(task_event)
+    return task_event
 
 
-def _get_progress_update(context, task_id, current_step):
-    q = _soft_delete_aware_query(context, models.TaskProgressUpdate)
-    return q.filter(
-        models.TaskProgressUpdate.task_id == task_id,
-        models.TaskProgressUpdate.current_step == current_step).first()
+@enginefacade.reader
+def _get_last_task_event(context, task_id):
+    q = _soft_delete_aware_query(
+        context, models.TaskEvent)
+    last_event = q.filter(
+        models.TaskEvent.task_id == task_id).order_by(
+            models.TaskEvent.index.desc()).first()
+    return last_event
 
 
 @enginefacade.reader
-def get_task_progress_step(context, task_id):
-    curr_step = 0
-    q = _soft_delete_aware_query(context, models.TaskProgressUpdate)
-    last_step = q.filter(
+def _get_last_task_progress_update(context, task_id):
+    q = _soft_delete_aware_query(
+        context, models.TaskProgressUpdate)
+    last_update = q.filter(
         models.TaskProgressUpdate.task_id == task_id).order_by(
-            models.TaskProgressUpdate.current_step.desc()).first()
+            models.TaskProgressUpdate.index.desc()).first()
+    return last_update
 
-    if last_step:
-        curr_step = last_step.current_step
 
-    return curr_step
+@enginefacade.reader
+def _get_last_minion_pool_event(context, pool_id):
+    q = _soft_delete_aware_query(
+        context, models.MinionPoolEvent)
+    last_event = q.filter(
+        models.MinionPoolEvent.pool_id == pool_id).order_by(
+            models.MinionPoolEvent.index.desc()).first()
+    return last_event
+
+
+@enginefacade.reader
+def _get_last_minion_pool_progress_update(context, pool_id):
+    q = _soft_delete_aware_query(
+        context, models.MinionPoolProgressUpdate)
+    last_event = q.filter(
+        models.MinionPoolProgressUpdate.pool_id == pool_id).order_by(
+            models.MinionPoolProgressUpdate.index.desc()).first()
+    return last_event
 
 
 @enginefacade.writer
-def add_task_progress_update(context, task_id, total_steps, message):
-    current_step = get_task_progress_step(context, task_id) + 1
-    task_progress_update = models.TaskProgressUpdate(
-        task_id=task_id, current_step=current_step, total_steps=total_steps,
-        message=message)
-    _session(context).add(task_progress_update)
+def add_minion_pool_event(context, pool_id, level, message):
+    pool_event = models.MinionPoolEvent()
+    pool_event.id = str(uuid.uuid4())
+    pool_event.pool_id = pool_id
+    pool_event.level = level
+    pool_event.message = message
+
+    pool_event.index = 0
+    last_pool_event = _get_last_minion_pool_event(context, pool_id)
+    if last_pool_event:
+        pool_event.index = last_pool_event.index + 1
+
+    _session(context).add(pool_event)
+    return pool_event
+
+
+def _get_minion_pool_progress_update(context, pool_id, index):
+    q = _soft_delete_aware_query(context, models.MinionPoolProgressUpdate)
+    return q.filter(
+        models.MinionPoolProgressUpdate.pool_id == pool_id,
+        models.MinionPoolProgressUpdate.index == index).first()
 
 
 @enginefacade.writer
-def update_task_progress_update(context, task_id, step, total_steps, message):
-    task_progress_update = _get_progress_update(context, task_id, step)
-    if not task_progress_update:
-        task_progress_update = models.TaskProgressUpdate(
-            task_id=task_id, current_step=step, total_steps=total_steps,
-            message=message)
-        _session(context).add(task_progress_update)
+def add_minion_pool_progress_update(
+        context, pool_id, message, initial_step=0, total_steps=0):
+    pool_progress_update = models.MinionPoolProgressUpdate()
+    pool_progress_update.id = str(uuid.uuid4())
+    pool_progress_update.pool_id = pool_id
+    pool_progress_update.current_step = initial_step
+    pool_progress_update.total_steps = total_steps
+    pool_progress_update.message = message
+    pool_progress_update.index = 0
+    last_progress_update = _get_last_minion_pool_progress_update(
+        context, pool_id)
+    if last_progress_update:
+        pool_progress_update.index = last_progress_update.index + 1
+
+    _session(context).add(pool_progress_update)
+    return pool_progress_update
+
+
+@enginefacade.writer
+def update_minion_pool_progress_update(
+        context, pool_id, update_index, new_current_step,
+        new_total_steps=None, new_message=None):
+    pool_progress_update = _get_minion_pool_progress_update(
+        context, pool_id, update_index)
+    if not pool_progress_update:
+        raise exception.NotFound(
+            "Could not find progress update for minion pool with ID '%s' and "
+            "index %s in the DB for updating." % (pool_id, update_index))
+
+    pool_progress_update.current_step = new_current_step
+    if new_total_steps is not None:
+        pool_progress_update.total_steps = new_total_steps
+    if new_message is not None:
+        pool_progress_update.message = new_message
+    return pool_progress_update
 
+
+def _get_progress_update(context, task_id, index):
+    q = _soft_delete_aware_query(context, models.TaskProgressUpdate)
+    return q.filter(
+        models.TaskProgressUpdate.task_id == task_id,
+        models.TaskProgressUpdate.index == index).first()
+
+
+@enginefacade.writer
+def add_task_progress_update(
+        context, task_id, message, initial_step=0, total_steps=0):
+    task_progress_update = models.TaskProgressUpdate()
+    task_progress_update.id = str(uuid.uuid4())
     task_progress_update.task_id = task_id
-    task_progress_update.current_step = step
+    task_progress_update.current_step = initial_step
     task_progress_update.total_steps = total_steps
     task_progress_update.message = message
 
+    task_progress_update.index = 0
+    last_progress_update = _get_last_task_progress_update(context, task_id)
+    if last_progress_update:
+        task_progress_update.index = last_progress_update.index + 1
+
+    _session(context).add(task_progress_update)
+    return task_progress_update
+
+
+@enginefacade.writer
+def update_task_progress_update(
+        context, task_id, update_index, new_current_step,
+        new_total_steps=None, new_message=None):
+    task_progress_update = _get_progress_update(context, task_id, update_index)
+    if not task_progress_update:
+        raise exception.NotFound(
+            "Could not find progress update for task with ID '%s' and "
+            "index %s in the DB for updating." % (task_id, update_index))
+
+    task_progress_update.current_step = new_current_step
+    if new_total_steps is not None:
+        task_progress_update.total_steps = new_total_steps
+    if new_message is not None:
+        task_progress_update.message = new_message
+
 
 @enginefacade.writer
 def update_replica(context, replica_id, updated_values):
@@ -868,7 +976,7 @@ def add_endpoint_region_mapping(context, endpoint_region_mapping):
 def get_endpoint_region_mapping(context, endpoint_id, region_id):
     q = _soft_delete_aware_query(context, models.EndpointRegionMapping)
     q = q.filter(
-        models.EndpointRegionMapping.region == region_id)
+        models.EndpointRegionMapping.region_id == region_id)
     q = q.filter(
         models.EndpointRegionMapping.endpoint_id == endpoint_id)
     return q.all()
@@ -1074,7 +1182,7 @@ def add_service_region_mapping(context, service_region_mapping):
 def get_service_region_mapping(context, service_id, region_id):
     q = _soft_delete_aware_query(context, models.ServiceRegionMapping)
     q = q.filter(
-        models.ServiceRegionMapping.region == region_id)
+        models.ServiceRegionMapping.region_id == region_id)
     q = q.filter(
         models.ServiceRegionMapping.service_id == service_id)
     return q.all()
@@ -1122,6 +1230,13 @@ def get_mapped_services_for_region(context, region_id):
 def add_minion_machine(context, minion_machine):
     minion_machine.user_id = context.user
     minion_machine.project_id = context.tenant
+    # inherit pool user/tenant if none are given:
+    if None in [minion_machine.user_id, minion_machine.project_id]:
+        pool = get_minion_pool(context, minion_machine.pool_id)
+        if not minion_machine.user_id:
+            minion_machine.user_id = pool.user_id
+        if not minion_machine.project_id:
+            minion_machine.project_id = pool.project_id
     _session(context).add(minion_machine)
 
 
@@ -1152,15 +1267,31 @@ def update_minion_machine(context, minion_machine_id, updated_values):
             "MinionMachine with ID '%s' does not exist." % minion_machine_id)
 
     updateable_fields = [
-        "connection_info", "provider_properties", "status",
-        "backup_writer_connection_info", "allocated_action"]
+        "connection_info", "provider_properties", "allocation_status",
+        "backup_writer_connection_info", "allocated_action",
+        "last_used_at", "power_status"]
     _update_sqlalchemy_object_fields(
         minion_machine, updateable_fields, updated_values)
 
 
+@enginefacade.writer
+def set_minion_machine_allocation_status(context, minion_machine_id, status):
+    machine = get_minion_machine(context, minion_machine_id)
+    if not machine:
+        raise exception.NotFound(
+            "Minion machine with ID '%s' not found" % minion_machine_id)
+    LOG.debug(
+        "Transitioning minion machine '%s' (pool '%s') from status '%s' to "
+        "'%s' in the DB",
+        minion_machine_id, machine.pool_id, machine.allocation_status, status)
+    machine.allocation_status = status
+    setattr(machine, 'updated_at', timeutils.utcnow())
+
+
 @enginefacade.writer
 def set_minion_machines_allocation_statuses(
-        context, minion_machine_ids, action_id, allocation_status):
+        context, minion_machine_ids, action_id, allocation_status,
+        refresh_allocation_time=True):
     machines = get_minion_machines(context)
     existing_machine_id_mappings = {
         machine.id: machine for machine in machines}
@@ -1177,10 +1308,12 @@ def set_minion_machines_allocation_statuses(
         LOG.debug(
             "Changing allocation status in DB for minion machine '%s' "
             "from '%s' to '%s' and allocated action from '%s' to '%s'" % (
-                machine.id, machine.status, allocation_status,
+                machine.id, machine.allocation_status, allocation_status,
                 machine.allocated_action, action_id))
         machine.allocated_action = action_id
-        machine.status = allocation_status
+        if refresh_allocation_time:
+            machine.last_used_at = timeutils.utcnow()
+        machine.allocation_status = allocation_status
 
 
 @enginefacade.writer
@@ -1195,60 +1328,69 @@ def delete_minion_machine(context, minion_machine_id):
 
 
 @enginefacade.writer
-def add_minion_pool_lifecycle(context, minion_pool_lifecycle):
-    minion_pool_lifecycle.user_id = context.user
-    minion_pool_lifecycle.project_id = context.tenant
-    _session(context).add(minion_pool_lifecycle)
+def add_minion_pool(context, minion_pool):
+    minion_pool.user_id = context.user
+    minion_pool.project_id = context.tenant
+    _session(context).add(minion_pool)
 
 
 @enginefacade.writer
-def delete_minion_pool_lifecycle(context, minion_pool_id):
-    _delete_transfer_action(
-        context, models.MinionPoolLifecycle, minion_pool_id)
+def delete_minion_pool(context, minion_pool_id):
+    args = {"id": minion_pool_id}
+    if is_user_context(context):
+        args["project_id"] = context.tenant
+    count = _soft_delete_aware_query(context, models.MinionPool).filter_by(
+        **args).soft_delete()
+    if count == 0:
+        raise exception.NotFound("0 entries were soft deleted")
 
 
 @enginefacade.reader
-def get_minion_pool_lifecycle(
-        context, minion_pool_id, include_tasks_executions=True,
-        include_machines=True):
-    q = _soft_delete_aware_query(context, models.MinionPoolLifecycle)
-    if include_tasks_executions:
-        q = q.options(orm.joinedload(models.MinionPoolLifecycle.executions))
+def get_minion_pool(
+        context, minion_pool_id, include_machines=True, include_events=True,
+        include_progress_updates=True):
+    q = _soft_delete_aware_query(context, models.MinionPool)
     if include_machines:
         q = q.options(orm.joinedload('minion_machines'))
+    if include_events:
+        q = q.options(orm.joinedload('events'))
+    if include_progress_updates:
+        q = q.options(orm.joinedload('progress_updates'))
     if is_user_context(context):
         q = q.filter(
-            models.MinionPoolLifecycle.project_id == context.tenant)
+            models.MinionPool.project_id == context.tenant)
     return q.filter(
-        models.MinionPoolLifecycle.id == minion_pool_id).first()
+        models.MinionPool.id == minion_pool_id).first()
 
 
 @enginefacade.reader
-def get_minion_pool_lifecycles(
-        context, include_tasks_executions=False, include_info=False,
-        include_machines=False, to_dict=True):
-    q = _soft_delete_aware_query(context, models.MinionPoolLifecycle)
-    if include_tasks_executions:
-        q = q.options(orm.joinedload(models.MinionPoolLifecycle.executions))
-    if include_info is False:
-        q = q.options(orm.defer('info'))
+def get_minion_pools(
+        context, include_machines=False, include_events=False,
+        include_progress_updates=False, to_dict=True):
+    q = _soft_delete_aware_query(context, models.MinionPool)
     q = q.filter()
     if is_user_context(context):
         q = q.filter(
-            models.Replica.project_id == context.tenant)
+            models.MinionPool.project_id == context.tenant)
     if include_machines:
         q = q.options(orm.joinedload('minion_machines'))
+    if include_events:
+        q = q.options(orm.joinedload('events'))
+    if include_progress_updates:
+        q = q.options(orm.joinedload('progress_updates'))
     db_result = q.all()
     if to_dict:
-        return [i.to_dict(
-            include_info=include_info,
-            include_executions=include_tasks_executions,
-            include_machines=include_machines) for i in db_result]
+        return [
+            i.to_dict(
+                include_machines=include_machines,
+                include_events=include_events,
+                include_progress_updates=include_progress_updates)
+            for i in db_result]
     return db_result
 
 
 @enginefacade.writer
-def add_minion_pool_lifecycle_execution(context, execution):
+def add_minion_pool_execution(context, execution):
     if is_user_context(context):
         if execution.action.project_id != context.tenant:
             raise exception.NotAuthorized()
@@ -1263,22 +1405,23 @@ def add_minion_pool_lifecycle_execution(context, execution):
 
 
 @enginefacade.writer
-def set_minion_pool_lifecycle_status(context, minion_pool_id, status):
-    pool = get_minion_pool_lifecycle(
-        context, minion_pool_id, include_tasks_executions=False,
-        include_machines=False)
+def set_minion_pool_status(context, minion_pool_id, status):
+    pool = get_minion_pool(
+        context, minion_pool_id, include_machines=False)
+    if not pool:
+        raise exception.NotFound(
+            "Minion pool '%s' not found" % minion_pool_id)
     LOG.debug(
-        "Transitioning minion pool '%s' from status '%s' to '%s'in DB",
-        minion_pool_id, pool.pool_status, status)
-    pool.pool_status = status
+        "Transitioning minion pool '%s' from status '%s' to '%s' in DB",
+        minion_pool_id, pool.status, status)
+    pool.status = status
     setattr(pool, 'updated_at', timeutils.utcnow())
 
 
 @enginefacade.writer
-def update_minion_pool_lifecycle(context, minion_pool_id, updated_values):
-    lifecycle = get_minion_pool_lifecycle(
-        context, minion_pool_id, include_tasks_executions=False,
-        include_machines=False)
+def update_minion_pool(context, minion_pool_id, updated_values):
+    lifecycle = get_minion_pool(
+        context, minion_pool_id, include_machines=False)
     if not lifecycle:
         raise exception.NotFound(
             "Minion pool '%s' not found" % minion_pool_id)
@@ -1286,69 +1429,21 @@ def update_minion_pool_lifecycle(context, minion_pool_id, updated_values):
     updateable_fields = [
         "minimum_minions", "maximum_minions", "minion_max_idle_time",
         "minion_retention_strategy", "environment_options",
-        "pool_shared_resources", "notes", "pool_name", "pool_os_type"]
-    # TODO(aznashwan): this should no longer be required when the
-    # transfer action class hirearchy is to be overhauled:
-    redundancies = {
-        "environment_options": [
-            "source_environment", "destination_environment"]}
+        "shared_resources", "notes", "name", "os_type"]
     for field in updateable_fields:
         if field in updated_values:
-            if field in redundancies:
-                for old_key in redundancies[field]:
-                    LOG.debug(
-                        "Updating the '%s' field of Minion Pool '%s' to: '%s'",
-                        old_key, minion_pool_id, updated_values[field])
-                    setattr(lifecycle, old_key, updated_values[field])
-            else:
-                LOG.debug(
-                    "Updating the '%s' field of Minion Pool '%s' to: '%s'",
-                    field, minion_pool_id, updated_values[field])
-                setattr(lifecycle, field, updated_values[field])
+            LOG.debug(
+                "Updating the '%s' field of Minion Pool '%s' to: '%s'",
+                field, minion_pool_id, updated_values[field])
+            setattr(lifecycle, field, updated_values[field])
 
     non_updateable_fields = set(
         updated_values.keys()).difference(updateable_fields)
     if non_updateable_fields:
         LOG.warn(
-            "The following Replica fields can NOT be updated: %s",
+            "The following Minion Pool fields can NOT be updated: %s",
             non_updateable_fields)
 
     # the oslo_db library uses this method for both the `created_at` and
     # `updated_at` fields
     setattr(lifecycle, 'updated_at', timeutils.utcnow())
-
-@enginefacade.reader
-def get_minion_pool_lifecycle_executions(
-        context, lifecycle_id, include_tasks=True):
-    q = _soft_delete_aware_query(context, models.TasksExecution)
-    q = q.join(models.MinionPoolLifecycle)
-    if include_tasks:
-        q = _get_tasks_with_details_options(q)
-    if is_user_context(context):
-        q = q.filter(models.MinionPoolLifecycle.project_id == context.tenant)
-    return q.filter(
-        models.MinionPoolLifecycle.id == lifecycle_id).all()
-
-@enginefacade.reader
-def get_minion_pool_lifecycle_execution(context, lifecycle_id, execution_id):
-    q = _soft_delete_aware_query(context, models.TasksExecution).join(
-        models.MinionPoolLifecycle)
-    q = _get_tasks_with_details_options(q)
-    if is_user_context(context):
-        q = q.filter(models.MinionPoolLifecycle.project_id == context.tenant)
-    return q.filter(
-        models.MinionPoolLifecycle.id == lifecycle_id,
-        models.TasksExecution.id == execution_id).first()
-
-@enginefacade.writer
-def delete_minion_pool_lifecycle_execution(context, execution_id):
-    q = _soft_delete_aware_query(context, models.TasksExecution).filter(
-        models.TasksExecution.id == execution_id)
-    if is_user_context(context):
-        if not q.join(models.MinionPoolLifecycle).filter(
-                models.MinionPoolLifecycle.project_id == (
-                    context.tenant)).first():
-            raise exception.NotAuthorized()
-    count = q.soft_delete()
-    if count == 0:
-        raise exception.NotFound("0 entries were soft deleted")

coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py

@@ -15,19 +15,6 @@ def upgrade(migrate_engine):
     base_transfer_action = sqlalchemy.Table(
         'base_transfer_action', meta, autoload=True)
 
-    # add the pool option properties for the transfer:
-    origin_minion_pool_id = sqlalchemy.Column(
-        "origin_minion_pool_id", sqlalchemy.String(36), nullable=True)
-    destination_minion_pool_id = sqlalchemy.Column(
-        "destination_minion_pool_id", sqlalchemy.String(36), nullable=True)
-    instance_osmorphing_minion_pool_mappings = sqlalchemy.Column(
-        "instance_osmorphing_minion_pool_mappings", sqlalchemy.Text,
-        nullable=False, default='{}')
-    for col in [
-            origin_minion_pool_id, destination_minion_pool_id,
-            instance_osmorphing_minion_pool_mappings]:
-        base_transfer_action.create_column(col)
-
     # extend tasks execution 'type' column:
     tasks_execution = sqlalchemy.Table(
         'tasks_execution', meta, autoload=True)
@@ -38,23 +25,38 @@ def upgrade(migrate_engine):
     # add table for pool lifecycles:
     tables.append(
         sqlalchemy.Table(
-            'minion_pool_lifecycle',
+            'minion_pool',
             meta,
             sqlalchemy.Column(
                 "id", sqlalchemy.String(36),
-                sqlalchemy.ForeignKey('base_transfer_action.base_id'),
-                primary_key=True),
+                default=lambda: str(uuid.uuid4()), primary_key=True),
+            sqlalchemy.Column("notes", sqlalchemy.Text, nullable=True),
+            sqlalchemy.Column(
+                "user_id", sqlalchemy.String(255), nullable=False),
+            sqlalchemy.Column(
+                "project_id", sqlalchemy.String(255), nullable=False),
+            sqlalchemy.Column(
+                "maintenance_trust_id", sqlalchemy.String(255), nullable=True),
+            sqlalchemy.Column('created_at', sqlalchemy.DateTime),
+            sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
+            sqlalchemy.Column('deleted_at', sqlalchemy.DateTime),
+            sqlalchemy.Column('deleted', sqlalchemy.String(36)),
+            sqlalchemy.Column(
+                "name", sqlalchemy.String(255), nullable=False),
             sqlalchemy.Column(
-                "pool_name", sqlalchemy.String(255), nullable=False),
+                "endpoint_id", sqlalchemy.String(36),
+                sqlalchemy.ForeignKey('endpoint.id'), nullable=False),
             sqlalchemy.Column(
-                "pool_os_type", sqlalchemy.String(255), nullable=False),
+                "environment_options", sqlalchemy.Text, nullable=False),
             sqlalchemy.Column(
-                "pool_platform", sqlalchemy.String(255), nullable=True),
+                "os_type", sqlalchemy.String(255), nullable=False),
             sqlalchemy.Column(
-                "pool_status", sqlalchemy.String(255), nullable=False,
+                "platform", sqlalchemy.String(255), nullable=True),
+            sqlalchemy.Column(
+                "status", sqlalchemy.String(255), nullable=False,
                 default=lambda: "UNKNOWN"),
             sqlalchemy.Column(
-                "pool_shared_resources", sqlalchemy.Text, nullable=True),
+                "shared_resources", sqlalchemy.Text, nullable=True),
             sqlalchemy.Column(
                 'minimum_minions', sqlalchemy.Integer, nullable=False),
             sqlalchemy.Column(
@@ -82,13 +84,18 @@ def upgrade(migrate_engine):
             sqlalchemy.Column('deleted', sqlalchemy.String(36)),
             sqlalchemy.Column(
                 'pool_id', sqlalchemy.String(36),
-                sqlalchemy.ForeignKey('minion_pool_lifecycle.id'),
+                sqlalchemy.ForeignKey('minion_pool.id'),
                 nullable=False),
             sqlalchemy.Column(
                 'allocated_action', sqlalchemy.String(36), nullable=True),
             sqlalchemy.Column(
-                'status', sqlalchemy.String(255), nullable=False,
-                default=lambda: "UNKNOWN"),
+                'last_used_at', sqlalchemy.DateTime, nullable=True),
+            sqlalchemy.Column(
+                'allocation_status', sqlalchemy.String(255), nullable=False,
+                default=lambda: "UNINITIALIZED"),
+            sqlalchemy.Column(
+                'power_status', sqlalchemy.String(255), nullable=False,
+                default=lambda: "UNINITIALIZED"),
             sqlalchemy.Column('connection_info', sqlalchemy.Text),
             sqlalchemy.Column(
                 'backup_writer_connection_info', sqlalchemy.Text,
@@ -97,11 +104,66 @@ def upgrade(migrate_engine):
                 'provider_properties', sqlalchemy.Text,
                 nullable=True)))
 
-    for index, table in enumerate(tables):
-        try:
+    tables.append(sqlalchemy.Table(
+        'minion_pool_event', meta,
+        sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True,
+                          default=lambda: str(uuid.uuid4())),
+        sqlalchemy.Column('created_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('deleted_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('deleted', sqlalchemy.String(36)),
+        sqlalchemy.Column('index', sqlalchemy.Integer, default=0),
+        sqlalchemy.Column("pool_id", sqlalchemy.String(36),
+                          sqlalchemy.ForeignKey('minion_pool.id'),
+                          nullable=False),
+        sqlalchemy.Column("level", sqlalchemy.String(50), nullable=False),
+        sqlalchemy.Column("message", sqlalchemy.Text, nullable=False),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'))
+
+    tables.append(sqlalchemy.Table(
+        'minion_pool_progress_update', meta,
+        sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True,
+                          default=lambda: str(uuid.uuid4())),
+        sqlalchemy.Column('created_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('deleted_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('index', sqlalchemy.Integer, default=0),
+        sqlalchemy.Column('deleted', sqlalchemy.String(36)),
+        sqlalchemy.Column("pool_id", sqlalchemy.String(36),
+                          sqlalchemy.ForeignKey('minion_pool.id'),
+                          nullable=False),
+        sqlalchemy.Column(
+            "current_step", sqlalchemy.BigInteger, nullable=False),
+        sqlalchemy.Column("total_steps", sqlalchemy.BigInteger, nullable=True),
+        sqlalchemy.Column("message", sqlalchemy.Text, nullable=True),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'))
+
+    # add the pool option properties for the transfer:
+    origin_minion_pool_id = sqlalchemy.Column(
+        "origin_minion_pool_id", sqlalchemy.String(36),
+        sqlalchemy.ForeignKey('minion_pool.id'), nullable=True)
+    destination_minion_pool_id = sqlalchemy.Column(
+        "destination_minion_pool_id", sqlalchemy.String(36),
+        sqlalchemy.ForeignKey('minion_pool.id'), nullable=True)
+    instance_osmorphing_minion_pool_mappings = sqlalchemy.Column(
+        "instance_osmorphing_minion_pool_mappings", sqlalchemy.Text,
+        nullable=False, default='{}')
+
+    created_columns = []
+    try:
+        for index, table in enumerate(tables):
             table.create()
-        except Exception:
-            # If an error occurs, drop all tables created so far to return
-            # to the previously existing state.
-            meta.drop_all(tables=tables[:index])
-            raise
+        for col in [
+                origin_minion_pool_id, destination_minion_pool_id,
+                instance_osmorphing_minion_pool_mappings]:
+            base_transfer_action.create_column(col)
+            created_columns.append(col)
+    except Exception:
+        # If an error occurs, drop all tables created so far to return
+        # to the previously existing state.
+        for col in created_columns:
+            base_transfer_action.drop_column(col)
+        meta.drop_all(tables=tables[:index])
+        raise

coriolis/db/sqlalchemy/migrate_repo/versions/018_adds_task_progress_idices.py

@@ -0,0 +1,20 @@
+import sqlalchemy
+from sqlalchemy import types
+
+
+def upgrade(migrate_engine):
+    meta = sqlalchemy.MetaData()
+    meta.bind = migrate_engine
+
+    task_event = sqlalchemy.Table('task_event', meta, autoload=True)
+    event_index = sqlalchemy.Column(
+        "index", sqlalchemy.Integer, default=0, nullable=False)
+    task_event.create_column(event_index)
+
+    task_progress_update = sqlalchemy.Table(
+        'task_progress_update', meta, autoload=True)
+    progress_index = sqlalchemy.Column(
+        "index", sqlalchemy.Integer, default=0, nullable=False)
+    task_progress_update.create_column(progress_index)
+    task_progress_update.c.current_step.alter(type=sqlalchemy.BigInteger)
+    task_progress_update.c.total_steps.alter(type=sqlalchemy.BigInteger)

coriolis/db/sqlalchemy/models.py

@@ -26,6 +26,7 @@ class TaskEvent(BASE, models.TimestampMixin, models.SoftDeleteMixin,
                                 sqlalchemy.ForeignKey('task.id'),
                                 nullable=False)
     level = sqlalchemy.Column(sqlalchemy.String(20), nullable=False)
+    index = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
     message = sqlalchemy.Column(sqlalchemy.String(1024), nullable=False)
 
     def to_dict(self):
@@ -34,6 +35,36 @@ class TaskEvent(BASE, models.TimestampMixin, models.SoftDeleteMixin,
             "task_id": self.task_id,
             "level": self.level,
             "message": self.message,
+            "index": self.index,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "deleted_at": self.deleted_at,
+            "deleted": self.deleted,
+        }
+        return result
+
+
+class MinionPoolEvent(BASE, models.TimestampMixin, models.SoftDeleteMixin,
+                models.ModelBase):
+    __tablename__ = 'minion_pool_event'
+
+    id = sqlalchemy.Column(sqlalchemy.String(36),
+                           default=lambda: str(uuid.uuid4()),
+                           primary_key=True)
+    pool_id = sqlalchemy.Column(sqlalchemy.String(36),
+                                sqlalchemy.ForeignKey('minion_pool.id'),
+                                nullable=False)
+    level = sqlalchemy.Column(sqlalchemy.String(20), nullable=False)
+    index = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
+    message = sqlalchemy.Column(sqlalchemy.Text, nullable=False)
+
+    def to_dict(self):
+        result = {
+            "id": self.id,
+            "pool_id": self.pool_id,
+            "level": self.level,
+            "index": self.index,
+            "message": self.message,
             "created_at": self.created_at,
             "updated_at": self.updated_at,
             "deleted_at": self.deleted_at,
@@ -46,7 +77,7 @@ class TaskProgressUpdate(BASE, models.TimestampMixin, models.SoftDeleteMixin,
                          models.ModelBase):
     __tablename__ = 'task_progress_update'
     __table_args__ = (
-        schema.UniqueConstraint("task_id", "current_step", "deleted"),)
+        schema.UniqueConstraint("task_id", "index", "deleted"),)
 
     id = sqlalchemy.Column(sqlalchemy.String(36),
                            default=lambda: str(uuid.uuid4()),
@@ -54,14 +85,50 @@ class TaskProgressUpdate(BASE, models.TimestampMixin, models.SoftDeleteMixin,
     task_id = sqlalchemy.Column(sqlalchemy.String(36),
                                 sqlalchemy.ForeignKey('task.id'),
                                 nullable=False)
-    current_step = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
-    total_steps = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
-    message = sqlalchemy.Column(sqlalchemy.String(1024), nullable=True)
+
+    index = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
+    current_step = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False)
+    total_steps = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=True)
+    message = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
 
     def to_dict(self):
         result = {
             "id": self.id,
             "task_id": self.task_id,
+            "index": self.index,
+            "current_step": self.current_step,
+            "total_steps": self.total_steps,
+            "message": self.message,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "deleted_at": self.deleted_at,
+            "deleted": self.deleted,
+        }
+        return result
+
+
+class MinionPoolProgressUpdate(
+        BASE, models.TimestampMixin, models.SoftDeleteMixin, models.ModelBase):
+    __tablename__ = 'minion_pool_progress_update'
+    __table_args__ = (
+        schema.UniqueConstraint("pool_id", "index", "deleted"),)
+
+    id = sqlalchemy.Column(sqlalchemy.String(36),
+                           default=lambda: str(uuid.uuid4()),
+                           primary_key=True)
+    pool_id = sqlalchemy.Column(sqlalchemy.String(36),
+                                sqlalchemy.ForeignKey('minion_pool.id'),
+                                nullable=False)
+    index = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
+    current_step = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False)
+    total_steps = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=True)
+    message = sqlalchemy.Column(sqlalchemy.String(1024), nullable=True)
+
+    def to_dict(self):
+        result = {
+            "id": self.id,
+            "pool_id": self.pool_id,
+            "index": self.index,
             "current_step": self.current_step,
             "total_steps": self.total_steps,
             "message": self.message,
@@ -90,17 +157,18 @@ class Task(BASE, models.TimestampMixin, models.SoftDeleteMixin,
     task_type = sqlalchemy.Column(sqlalchemy.String(100), nullable=False)
     exception_details = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
     depends_on = sqlalchemy.Column(types.List, nullable=True)
-    index = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
+    index = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
     on_error = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False)
     # TODO(alexpilotti): Add soft delete filter
     events = orm.relationship(TaskEvent, cascade="all,delete",
-                              backref=orm.backref('task'))
+                              backref=orm.backref('task'),
+                              order_by=TaskEvent.index)
     # TODO(alexpilotti): Add soft delete filter
     progress_updates = orm.relationship(TaskProgressUpdate,
                                         cascade="all,delete",
                                         backref=orm.backref('task'),
                                         order_by=(
-                                            TaskProgressUpdate.current_step))
+                                            TaskProgressUpdate.index))
 
     def to_dict(self):
         result = {
@@ -201,9 +269,11 @@ class BaseTransferAction(BASE, models.TimestampMixin, models.ModelBase,
     storage_mappings = sqlalchemy.Column(types.Json, nullable=True)
     source_environment = sqlalchemy.Column(types.Json, nullable=True)
     origin_minion_pool_id = sqlalchemy.Column(
-        sqlalchemy.String(36), nullable=True)
+        sqlalchemy.String(36),
+        sqlalchemy.ForeignKey('minion_pool.id'), nullable=True)
     destination_minion_pool_id = sqlalchemy.Column(
-        sqlalchemy.String(36), nullable=True)
+        sqlalchemy.String(36),
+        sqlalchemy.ForeignKey('minion_pool.id'), nullable=True)
     instance_osmorphing_minion_pool_mappings = sqlalchemy.Column(
         types.Json, nullable=False, default=lambda: {})
     user_scripts = sqlalchemy.Column(types.Json, nullable=True)
@@ -407,54 +477,6 @@ class Region(
         secondary="service_region_mapping")
 
 
-class Endpoint(BASE, models.TimestampMixin, models.ModelBase,
-               models.SoftDeleteMixin):
-    __tablename__ = 'endpoint'
-
-    id = sqlalchemy.Column(sqlalchemy.String(36),
-                           default=lambda: str(uuid.uuid4()),
-                           primary_key=True)
-    user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
-    project_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
-    connection_info = sqlalchemy.Column(types.Json, nullable=False)
-    type = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
-    name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
-    description = sqlalchemy.Column(sqlalchemy.String(1024), nullable=True)
-    origin_actions = orm.relationship(
-        BaseTransferAction, backref=orm.backref('origin_endpoint'),
-        primaryjoin="and_(BaseTransferAction.origin_endpoint_id==Endpoint.id, "
-                    "BaseTransferAction.deleted=='0')")
-    destination_actions = orm.relationship(
-        BaseTransferAction, backref=orm.backref('destination_endpoint'),
-        primaryjoin="and_(BaseTransferAction.destination_endpoint_id=="
-                    "Endpoint.id, BaseTransferAction.deleted=='0')")
-    mapped_regions = orm.relationship(
-        'Region', back_populates='mapped_endpoints',
-        secondary="endpoint_region_mapping")
-
-
-class ReplicaSchedule(BASE, models.TimestampMixin, models.ModelBase,
-                      models.SoftDeleteMixin):
-    __tablename__ = "replica_schedules"
-
-    id = sqlalchemy.Column(sqlalchemy.String(36),
-                           default=lambda: str(uuid.uuid4()),
-                           primary_key=True)
-    replica_id = sqlalchemy.Column(
-        sqlalchemy.String(36),
-        sqlalchemy.ForeignKey('replica.id'), nullable=False)
-    replica = orm.relationship(
-        Replica, backref=orm.backref("schedules"), foreign_keys=[replica_id])
-    schedule = sqlalchemy.Column(types.Json, nullable=False)
-    expiration_date = sqlalchemy.Column(
-        sqlalchemy.types.DateTime, nullable=True)
-    enabled = sqlalchemy.Column(
-        sqlalchemy.Boolean, nullable=False, default=lambda: False)
-    shutdown_instance = sqlalchemy.Column(
-        sqlalchemy.Boolean, nullable=False, default=False)
-    trust_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
-
-
 class MinionMachine(BASE, models.TimestampMixin, models.ModelBase,
                     models.SoftDeleteMixin):
     __tablename__ = "minion_machine"
@@ -467,16 +489,22 @@ class MinionMachine(BASE, models.TimestampMixin, models.ModelBase,
 
     pool_id = sqlalchemy.Column(
         sqlalchemy.String(36),
-        sqlalchemy.ForeignKey('minion_pool_lifecycle.id'),
+        sqlalchemy.ForeignKey('minion_pool.id'),
         nullable=False)
 
-    status = sqlalchemy.Column(
+    allocation_status = sqlalchemy.Column(
         sqlalchemy.String(255), nullable=False,
-        default=lambda: constants.MINION_MACHINE_STATUS_UNKNOWN)
+        default=lambda: constants.MINION_MACHINE_STATUS_UNINITIALIZED)
 
     allocated_action = sqlalchemy.Column(
         sqlalchemy.String(36), nullable=True)
 
+    power_status = sqlalchemy.Column(
+        sqlalchemy.String(255), nullable=False)
+
+    last_used_at = sqlalchemy.Column(
+        sqlalchemy.types.DateTime, nullable=True)
+
     connection_info = sqlalchemy.Column(
         types.Json, nullable=True)
 
@@ -496,9 +524,11 @@ class MinionMachine(BASE, models.TimestampMixin, models.ModelBase,
             "deleted_at": self.deleted_at,
             "deleted": self.deleted,
             "pool_id": self.pool_id,
-            "status": self.status,
+            "allocation_status": self.allocation_status,
+            "power_status": self.power_status,
             "connection_info": self.connection_info,
             "allocated_action": self.allocated_action,
+            "last_used_at": self.last_used_at,
             "backup_writer_connection_info": (
                 self.backup_writer_connection_info),
             "provider_properties": self.provider_properties
@@ -506,29 +536,35 @@ class MinionMachine(BASE, models.TimestampMixin, models.ModelBase,
         return result
 
 
-class MinionPoolLifecycle(BaseTransferAction):
-    # TODO(aznashwan): this class inherits numerous redundant fields from
-    # BaseTransferAction. Ideally, the upper hirearchy should be split into a
-    # BaseAction, and a separate inheriting BaseTransferAction.
-    __tablename__ = 'minion_pool_lifecycle'
+class MinionPool(
+            BASE, models.TimestampMixin, models.ModelBase,
+            models.SoftDeleteMixin):
+    __tablename__ = 'minion_pool'
 
     id = sqlalchemy.Column(
         sqlalchemy.String(36),
-        sqlalchemy.ForeignKey(
-            'base_transfer_action.base_id'),
         primary_key=True)
+    user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    project_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    maintenance_trust_id = sqlalchemy.Column(
+        sqlalchemy.String(255), nullable=True)
 
-    pool_name = sqlalchemy.Column(
+    name = sqlalchemy.Column(
         sqlalchemy.String(255),
         nullable=False)
-    pool_os_type = sqlalchemy.Column(
+    notes = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
+    endpoint_id = sqlalchemy.Column(
+        sqlalchemy.String(36),
+        sqlalchemy.ForeignKey('endpoint.id'), nullable=False)
+    os_type = sqlalchemy.Column(
         sqlalchemy.String(255), nullable=False)
-    pool_platform = sqlalchemy.Column(
+    platform = sqlalchemy.Column(
         sqlalchemy.String(255), nullable=False)
-    pool_status = sqlalchemy.Column(
+    environment_options = sqlalchemy.Column(types.Json, nullable=True)
+    status = sqlalchemy.Column(
         sqlalchemy.String(255), nullable=False,
         default=lambda: constants.MINION_POOL_STATUS_UNKNOWN)
-    pool_shared_resources = sqlalchemy.Column(
+    shared_resources = sqlalchemy.Column(
         types.Json, nullable=True)
     minimum_minions = sqlalchemy.Column(
         sqlalchemy.Integer, nullable=False)
@@ -538,43 +574,102 @@ class MinionPoolLifecycle(BaseTransferAction):
         sqlalchemy.Integer, nullable=False)
     minion_retention_strategy = sqlalchemy.Column(
         sqlalchemy.String(255), nullable=False)
+
     minion_machines = orm.relationship(
         MinionMachine, backref=orm.backref('minion_pool'),
-        primaryjoin="and_(MinionMachine.pool_id==MinionPoolLifecycle.id, "
+        primaryjoin="and_(MinionMachine.pool_id==MinionPool.id, "
                     "MinionMachine.deleted=='0')")
-
-    __mapper_args__ = {
-        'polymorphic_identity': 'minion_pool_lifecycle'}
+    events = orm.relationship(MinionPoolEvent, cascade="all,delete",
+                              backref=orm.backref('minion_pool'),
+                              order_by=MinionPoolEvent.index)
+    progress_updates = orm.relationship(MinionPoolProgressUpdate,
+                                        cascade="all,delete",
+                                        backref=orm.backref('minion_pool'),
+                                        order_by=(
+                                            MinionPoolProgressUpdate.index))
 
     def to_dict(
-            self, include_info=True, include_machines=True,
-            include_executions=True):
-        base = super(MinionPoolLifecycle, self).to_dict(
-            include_info=include_info, include_executions=include_executions)
-        base.update({
+            self, include_machines=True, include_events=True,
+            include_progress_updates=True):
+        base = {
             "id": self.id,
-            "pool_name": self.pool_name,
-            "pool_os_type": self.pool_os_type,
-            "pool_platform": self.pool_platform,
-            "pool_shared_resources": self.pool_shared_resources,
-            "pool_status": self.pool_status,
+            "name": self.name,
+            "notes": self.notes,
+            "endpoint_id": self.endpoint_id,
+            "environment_options": self.environment_options,
+            "os_type": self.os_type,
+            "maintenance_trust_id": self.maintenance_trust_id,
+            "platform": self.platform,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "deleted_at": self.deleted_at,
+            "deleted": self.deleted,
+            "shared_resources": self.shared_resources,
+            "status": self.status,
             "minimum_minions": self.minimum_minions,
             "maximum_minions": self.maximum_minions,
             "minion_max_idle_time": self.minion_max_idle_time,
-            "minion_retention_strategy": self.minion_retention_strategy})
+            "minion_retention_strategy": self.minion_retention_strategy}
         base["minion_machines"] = []
         if include_machines:
             base["minion_machines"] = [
                 machine.to_dict() for machine in self.minion_machines]
-        # TODO(aznashwan): these nits should be avoided by splitting the
-        # BaseTransferAction class into a more specialized hireachy:
-        redundancies = {
-            "environment_options": [
-                "source_environment", "destination_environment"],
-            "endpoint_id": [
-                "origin_endpoint_id", "destination_endpoint_id"]}
-        for new_key, old_keys in redundancies.items():
-            for old_key in old_keys:
-                if old_key in base:
-                    base[new_key] = base.pop(old_key)
+        if include_events:
+            base["events"] = [
+                ev.to_dict() for ev in self.events]
+        if include_progress_updates:
+            base["progress_updates"] = [
+                pu.to_dict() for pu in self.progress_updates]
         return base
+
+
+class Endpoint(BASE, models.TimestampMixin, models.ModelBase,
+               models.SoftDeleteMixin):
+    __tablename__ = 'endpoint'
+
+    id = sqlalchemy.Column(sqlalchemy.String(36),
+                           default=lambda: str(uuid.uuid4()),
+                           primary_key=True)
+    user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    project_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    connection_info = sqlalchemy.Column(types.Json, nullable=False)
+    type = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    description = sqlalchemy.Column(sqlalchemy.String(1024), nullable=True)
+    origin_actions = orm.relationship(
+        BaseTransferAction, backref=orm.backref('origin_endpoint'),
+        primaryjoin="and_(BaseTransferAction.origin_endpoint_id==Endpoint.id, "
+                    "BaseTransferAction.deleted=='0')")
+    destination_actions = orm.relationship(
+        BaseTransferAction, backref=orm.backref('destination_endpoint'),
+        primaryjoin="and_(BaseTransferAction.destination_endpoint_id=="
+                    "Endpoint.id, BaseTransferAction.deleted=='0')")
+    minion_pools = orm.relationship(
+        MinionPool, backref=orm.backref('endpoint'),
+        primaryjoin="and_(MinionPool.endpoint_id=="
+                    "Endpoint.id, MinionPool.deleted=='0')")
+    mapped_regions = orm.relationship(
+        'Region', back_populates='mapped_endpoints',
+        secondary="endpoint_region_mapping")
+
+
+class ReplicaSchedule(BASE, models.TimestampMixin, models.ModelBase,
+                      models.SoftDeleteMixin):
+    __tablename__ = "replica_schedules"
+
+    id = sqlalchemy.Column(sqlalchemy.String(36),
+                           default=lambda: str(uuid.uuid4()),
+                           primary_key=True)
+    replica_id = sqlalchemy.Column(
+        sqlalchemy.String(36),
+        sqlalchemy.ForeignKey('replica.id'), nullable=False)
+    replica = orm.relationship(
+        Replica, backref=orm.backref("schedules"), foreign_keys=[replica_id])
+    schedule = sqlalchemy.Column(types.Json, nullable=False)
+    expiration_date = sqlalchemy.Column(
+        sqlalchemy.types.DateTime, nullable=True)
+    enabled = sqlalchemy.Column(
+        sqlalchemy.Boolean, nullable=False, default=lambda: False)
+    shutdown_instance = sqlalchemy.Column(
+        sqlalchemy.Boolean, nullable=False, default=False)
+    trust_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)

coriolis/endpoint_options/api.py

@@ -1,29 +1,32 @@
 # Copyright 2020 Cloudbase Solutions Srl
 # All Rights Reserved.
 
-from coriolis.conductor.rpc import client as rpc_client
+from coriolis.conductor.rpc import client as rpc_conductor_client
+from coriolis.minion_manager.rpc import client as rpc_minion_manager_client
 
 
 class API(object):
     def __init__(self):
-        self._rpc_client = rpc_client.ConductorClient()
+        self._rpc_minion_manager_client = (
+            rpc_minion_manager_client.MinionManagerClient())
+        self._rpc_conductor_client = rpc_conductor_client.ConductorClient()
 
-    def get_endpoint_destination_options(
+    def get_endpoint_source_options(
             self, ctxt, endpoint_id, env=None, option_names=None):
-        return self._rpc_client.get_endpoint_destination_options(
+        return self._rpc_conductor_client.get_endpoint_source_options(
             ctxt, endpoint_id, env, option_names)
 
-    def get_endpoint_source_minion_pool_options(
+    def get_endpoint_destination_options(
             self, ctxt, endpoint_id, env=None, option_names=None):
-        return self._rpc_client.get_endpoint_source_minion_pool_options(
+        return self._rpc_conductor_client.get_endpoint_destination_options(
             ctxt, endpoint_id, env, option_names)
 
-    def get_endpoint_destination_minion_pool_options(
+    def get_endpoint_source_minion_pool_options(
             self, ctxt, endpoint_id, env=None, option_names=None):
-        return self._rpc_client.get_endpoint_destination_minion_pool_options(
+        return self._rpc_minion_manager_client.get_endpoint_source_minion_pool_options(
             ctxt, endpoint_id, env, option_names)
 
-    def get_endpoint_source_options(
+    def get_endpoint_destination_minion_pool_options(
             self, ctxt, endpoint_id, env=None, option_names=None):
-        return self._rpc_client.get_endpoint_source_options(
+        return self._rpc_minion_manager_client.get_endpoint_destination_minion_pool_options(
             ctxt, endpoint_id, env, option_names)

coriolis/endpoints/api.py

@@ -2,55 +2,58 @@
 # All Rights Reserved.
 
 from coriolis import utils
-from coriolis.conductor.rpc import client as rpc_client
+from coriolis.conductor.rpc import client as rpc_conductor_client
+from coriolis.minion_manager.rpc import client as rpc_minion_manager_client
 
 
 class API(object):
     def __init__(self):
-        self._rpc_client = rpc_client.ConductorClient()
+        self._rpc_conductor_client = rpc_conductor_client.ConductorClient()
+        self._rpc_minion_manager_client = (
+            rpc_minion_manager_client.MinionManagerClient())
 
     def create(self, ctxt, name, endpoint_type, description,
                connection_info, mapped_regions):
-        return self._rpc_client.create_endpoint(
+        return self._rpc_conductor_client.create_endpoint(
             ctxt, name, endpoint_type, description, connection_info,
             mapped_regions)
 
     def update(self, ctxt, endpoint_id, properties):
-        return self._rpc_client.update_endpoint(
+        return self._rpc_conductor_client.update_endpoint(
             ctxt, endpoint_id, properties)
 
     def delete(self, ctxt, endpoint_id):
-        self._rpc_client.delete_endpoint(ctxt, endpoint_id)
+        self._rpc_conductor_client.delete_endpoint(ctxt, endpoint_id)
 
     def get_endpoints(self, ctxt):
-        return self._rpc_client.get_endpoints(ctxt)
+        return self._rpc_conductor_client.get_endpoints(ctxt)
 
     def get_endpoint(self, ctxt, endpoint_id):
-        return self._rpc_client.get_endpoint(ctxt, endpoint_id)
+        return self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id)
 
     def validate_connection(self, ctxt, endpoint_id):
-        return self._rpc_client.validate_endpoint_connection(
+        return self._rpc_conductor_client.validate_endpoint_connection(
             ctxt, endpoint_id)
 
     @utils.bad_request_on_error("Invalid destination environment: %s")
     def validate_target_environment(self, ctxt, endpoint_id, target_env):
-        return self._rpc_client.validate_endpoint_target_environment(
+        return self._rpc_conductor_client.validate_endpoint_target_environment(
             ctxt, endpoint_id, target_env)
 
     @utils.bad_request_on_error("Invalid source environment: %s")
     def validate_source_environment(self, ctxt, endpoint_id, source_env):
-        return self._rpc_client.validate_endpoint_source_environment(
+        return self._rpc_conductor_client.validate_endpoint_source_environment(
             ctxt, endpoint_id, source_env)
 
     @utils.bad_request_on_error("Invalid source minion pool environment: %s")
     def validate_endpoint_source_minion_pool_options(
             self, ctxt, endpoint_id, pool_environment):
-        return self._rpc_client.validate_endpoint_source_minion_pool_options(
+        return self._rpc_minion_manager_client.validate_endpoint_source_minion_pool_options(
             ctxt, endpoint_id, pool_environment)
 
     @utils.bad_request_on_error(
         "Invalid destination minion pool environment: %s")
     def validate_endpoint_destination_minion_pool_options(
             self, ctxt, endpoint_id, pool_environment):
-        return self._rpc_client.validate_endpoint_destination_minion_pool_options(
+        return self._rpc_minion_manager_client.validate_endpoint_destination_minion_pool_options(
             ctxt, endpoint_id, pool_environment)

coriolis/events.py

@@ -7,91 +7,93 @@ import collections
 from oslo_log import log as logging
 from six import with_metaclass
 
+from coriolis import constants
+
 
 LOG = logging.getLogger(__name__)
 
 _PercStepData = collections.namedtuple(
-    "_PercStepData", "last_value max_value perc_threshold message_format")
+    "_PercStepData", "progress_update_id last_value total_steps")
 
 
 class EventManager(object, with_metaclass(abc.ABCMeta)):
 
     def __init__(self, event_handler):
         self._event_handler = event_handler
-        self._total_steps = None
-        self._percentage_steps = {}
-
-    def set_total_progress_steps(self, total_steps):
-        self._total_steps = total_steps
 
-    def add_percentage_step(self, max_value, perc_threshold=1,
-                            message_format="{:.0f}%"):
-        if max_value < 0:
+    def _call_event_handler(self, method_name, *args, **kwargs):
+        if self._event_handler:
+            method_obj = getattr(self._event_handler, str(method_name), None)
+            if not method_obj:
+                raise AttributeError(
+                    "No method named '%s' for event handler of type '%s'." % (
+                        method_name, type(self._event_handler)))
+            return method_obj(*args, **kwargs)
+
+    def add_percentage_step(self, message, total_steps, initial_step=0):
+        if total_steps < 0:
             LOG.warn(
                 "Max percentage value was negative (%s). Reset to 0",
-                max_value)
-            max_value = 0
-        if max_value == 0:
+                total_steps)
+            total_steps = 0
+        if total_steps == 0:
             LOG.warn("Max percentage value set to 0 (zero)")
-        current_step = self._event_handler.get_task_progress_step() + 1
-        self._percentage_steps[current_step] = _PercStepData(
-            0, max_value, perc_threshold, message_format)
-        return current_step
-
-    def set_percentage_step(self, step, value):
-        step_data = self._percentage_steps[step]
-
-        old_perc = 100
-        perc = 100
-        if step_data.max_value != 0:
-            old_perc = (step_data.last_value * 100 / step_data.max_value //
-                        step_data.perc_threshold * step_data.perc_threshold)
-            perc = (value * 100 / step_data.max_value //
-                    step_data.perc_threshold * step_data.perc_threshold)
-
-        if perc > old_perc and self._event_handler:
-            self._event_handler.update_task_progress_update(
-                step, self._total_steps, step_data.message_format.format(perc))
-            self._percentage_steps[step] = _PercStepData(
-                value, step_data.max_value, step_data.perc_threshold,
-                step_data.message_format)
+
+        if initial_step > total_steps:
+            raise ValueError(
+                "Provided percent step initial value '%s' is larger than the "
+                "maximum value '%s'" % (initial_step, total_steps))
+        progress_update = self._call_event_handler(
+            'add_progress_update', message, initial_step=initial_step,
+            total_steps=total_steps, return_event=True)
+        progress_update_id = (
+            self._call_event_handler(
+                'get_progress_update_identifier', progress_update))
+
+        return _PercStepData(progress_update_id, initial_step, total_steps)
+
+    def set_percentage_step(self, step, new_current_step):
+        self._call_event_handler(
+            'update_progress_update', step.progress_update_id,
+            new_current_step)
 
     def progress_update(self, message):
-        if self._event_handler:
-            self._event_handler.add_task_progress_update(
-                self._total_steps, message)
+        self._call_event_handler(
+            'add_progress_update', message, return_event=False)
 
     def info(self, message):
-        if self._event_handler:
-            self._event_handler.info(message)
+        self._call_event_handler(
+            'add_event', message, level=constants.TASK_EVENT_INFO)
 
     def warn(self, message):
-        if self._event_handler:
-            self._event_handler.warn(message)
+        self._call_event_handler(
+            'add_event', message, level=constants.TASK_EVENT_WARNING)
 
     def error(self, message):
-        if self._event_handler:
-            self._event_handler.error(message)
+        self._call_event_handler(
+            'add_event', message, level=constants.TASK_EVENT_ERROR)
 
 
 class BaseEventHandler(object, with_metaclass(abc.ABCMeta)):
 
     @abc.abstractmethod
-    def add_task_progress_update(self, total_steps, message):
-        pass
-
-    @abc.abstractmethod
-    def update_task_progress_update(self, step, total_steps, message):
+    def add_progress_update(
+            self, message, initial_step=0, total_steps=0,
+            return_event=False):
         pass
 
     @abc.abstractmethod
-    def info(self, message):
+    def update_progress_update(
+            self, update_identifier, new_current_step,
+            new_total_steps=None, new_message=None):
         pass
 
+    @classmethod
     @abc.abstractmethod
-    def warn(self, message):
+    def get_progress_update_identifier(cls, progress_update):
+        """ Returns the identifier for a given progress update. """
         pass
 
     @abc.abstractmethod
-    def error(self, message):
+    def add_event(self, message, level=constants.TASK_EVENT_INFO):
         pass

coriolis/exception.py

@@ -153,6 +153,10 @@ class InvalidMinionPoolSelection(Invalid):
     message = _("The selected minion pool is incompatible.")
 
 
+class InvalidMinionMachineState(Invalid):
+    message = _("The selected minion machine is in an invalid state.")
+
+
 class MinionMachineAllocationFailure(Invalid):
     message = _("No minion machines were available for allocation")
 

coriolis/migrations/manager.py

@@ -27,9 +27,7 @@ def _copy_volume(volume, disk_image_reader, backup_writer, event_manager):
             disk_size = reader.disk_size
 
             perc_step = event_manager.add_percentage_step(
-                disk_size,
-                message_format="Disk copy progress for %s: "
-                               "{:.0f}%%" % disk_id)
+                "Copying data of disk %s" % disk_id, disk_size)
 
             offset = 0
             max_block_size = 10 * units.Mi  # 10 MB

coriolis/minion_manager/rpc/client.py

@@ -0,0 +1,216 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+import oslo_messaging as messaging
+
+from coriolis import constants
+from coriolis import events
+from coriolis import rpc
+
+
+VERSION = "1.0"
+LOG = logging.getLogger(__name__)
+
+MINION_MANAGER_OPTS = [
+    cfg.IntOpt("minion_mananger_rpc_timeout",
+               help="Number of seconds until RPC calls to the "
+                    "minion manager timeout.")
+]
+
+CONF = cfg.CONF
+CONF.register_opts(MINION_MANAGER_OPTS, 'minion_manager')
+
+
+class MinionManagerClient(rpc.BaseRPCClient):
+
+    def __init__(self, timeout=None):
+        target = messaging.Target(topic='coriolis_minion_manager', version=VERSION)
+        if timeout is None:
+            timeout = CONF.minion_manager.minion_mananger_rpc_timeout
+        super(MinionManagerClient, self).__init__(
+            target, timeout=timeout)
+
+    def add_minion_pool_progress_update(
+            self, ctxt, minion_pool_id, message, initial_step=0, total_steps=0,
+            return_event=False):
+        operation = self._cast
+        if return_event:
+            operation = self._call
+        return operation(
+            ctxt, 'add_minion_pool_progress_update',
+            minion_pool_id=minion_pool_id, message=message,
+            initial_step=initial_step, total_steps=total_steps)
+
+    def update_minion_pool_progress_update(
+            self, ctxt, minion_pool_id, progress_update_index, new_current_step,
+            new_total_steps=None, new_message=None):
+        self._cast(
+            ctxt, 'update_minion_pool_progress_update',
+            minion_pool_id=minion_pool_id,
+            progress_update_index=progress_update_index,
+            new_current_step=new_current_step, new_total_steps=new_total_steps,
+            new_message=new_message)
+
+    def add_minion_pool_event(self, ctxt, minion_pool_id, level, message):
+        return self._cast(
+            ctxt, 'add_minion_pool_event', minion_pool_id=minion_pool_id,
+            level=level, message=message)
+
+    def get_diagnostics(self, ctxt):
+        return self._call(ctxt, 'get_diagnostics')
+
+    def validate_minion_pool_selections_for_action(self, ctxt, action):
+        return self._call(
+            ctxt, 'validate_minion_pool_selections_for_action',
+            action=action)
+
+    def allocate_minion_machines_for_replica(
+            self, ctxt, replica):
+        return self._cast(
+            ctxt, 'allocate_minion_machines_for_replica', replica=replica)
+
+    def allocate_minion_machines_for_migration(
+            self, ctxt, migration, include_transfer_minions=True,
+            include_osmorphing_minions=True):
+        return self._cast(
+            ctxt, 'allocate_minion_machines_for_migration',
+            migration=migration,
+            include_transfer_minions=include_transfer_minions,
+            include_osmorphing_minions=include_osmorphing_minions)
+
+    def deallocate_minion_machine(self, ctxt, minion_machine_id):
+         return self._cast(
+            ctxt, 'deallocate_minion_machine',
+            minion_machine_id=minion_machine_id)
+
+    def deallocate_minion_machines_for_action(self, ctxt, action_id):
+        return self._cast(
+            ctxt, 'deallocate_minion_machines_for_action', action_id=action_id)
+
+    def create_minion_pool(
+            self, ctxt, name, endpoint_id, pool_platform, pool_os_type,
+            environment_options, minimum_minions, maximum_minions,
+            minion_max_idle_time, minion_retention_strategy, notes=None,
+            skip_allocation=False):
+        return self._call(
+            ctxt, 'create_minion_pool', name=name, endpoint_id=endpoint_id,
+            pool_platform=pool_platform, pool_os_type=pool_os_type,
+            environment_options=environment_options,
+            minimum_minions=minimum_minions,
+            maximum_minions=maximum_minions,
+            minion_max_idle_time=minion_max_idle_time,
+            minion_retention_strategy=minion_retention_strategy,
+            notes=notes, skip_allocation=skip_allocation)
+
+    def set_up_shared_minion_pool_resources(self, ctxt, minion_pool_id):
+        return self._call(
+            ctxt, "set_up_shared_minion_pool_resources",
+            minion_pool_id=minion_pool_id)
+
+    def tear_down_shared_minion_pool_resources(
+            self, ctxt, minion_pool_id, force=False):
+        return self._call(
+            ctxt, "tear_down_shared_minion_pool_resources",
+            minion_pool_id=minion_pool_id, force=force)
+
+    def allocate_minion_pool(self, ctxt, minion_pool_id):
+        return self._call(
+            ctxt, "allocate_minion_pool",
+            minion_pool_id=minion_pool_id)
+
+    def refresh_minion_pool(self, ctxt, minion_pool_id):
+        return self._call(
+            ctxt, "refresh_minion_pool",
+            minion_pool_id=minion_pool_id)
+
+    def deallocate_minion_pool(
+            self, ctxt, minion_pool_id, force=False):
+        return self._call(
+            ctxt, "deallocate_minion_pool",
+            minion_pool_id=minion_pool_id,
+            force=force)
+
+    def get_minion_pools(self, ctxt):
+        return self._call(ctxt, 'get_minion_pools')
+
+    def get_minion_pool(self, ctxt, minion_pool_id):
+        return self._call(
+            ctxt, 'get_minion_pool', minion_pool_id=minion_pool_id)
+
+    def update_minion_pool(self, ctxt, minion_pool_id, updated_values):
+        return self._call(
+            ctxt, 'update_minion_pool',
+            minion_pool_id=minion_pool_id, updated_values=updated_values)
+
+    def delete_minion_pool(self, ctxt, minion_pool_id):
+        return self._call(
+            ctxt, 'delete_minion_pool', minion_pool_id=minion_pool_id)
+
+    def get_endpoint_source_minion_pool_options(
+            self, ctxt, endpoint_id, env, option_names):
+        return self._call(
+            ctxt, 'get_endpoint_source_minion_pool_options',
+            endpoint_id=endpoint_id, env=env, option_names=option_names)
+
+    def get_endpoint_destination_minion_pool_options(
+            self, ctxt, endpoint_id, env, option_names):
+        return self._call(
+            ctxt, 'get_endpoint_destination_minion_pool_options',
+            endpoint_id=endpoint_id, env=env, option_names=option_names)
+
+    def validate_endpoint_source_minion_pool_options(
+            self, ctxt, endpoint_id, pool_environment):
+        return self._call(
+            ctxt, 'validate_endpoint_source_minion_pool_options',
+            endpoint_id=endpoint_id, pool_environment=pool_environment)
+
+    def validate_endpoint_destination_minion_pool_options(
+            self, ctxt, endpoint_id, pool_environment):
+        return self._call(
+            ctxt, 'validate_endpoint_destination_minion_pool_options',
+            endpoint_id=endpoint_id, pool_environment=pool_environment)
+
+
+class MinionManagerPoolRpcEventHandler(events.BaseEventHandler):
+    def __init__(self, ctxt, pool_id):
+        self._ctxt = ctxt
+        self._pool_id = pool_id
+        self._rpc_minion_manager_client_instance = None
+
+    @property
+    def _rpc_minion_manager_client(self):
+        # NOTE(aznashwan): it is unsafe to fork processes with pre-instantiated
+        # oslo_messaging clients as the underlying eventlet thread queues will
+        # be invalidated.
+        if self._rpc_minion_manager_client_instance is None:
+            self._rpc_minion_manager_client_instance = MinionManagerClient()
+        return self._rpc_minion_manager_client_instance
+
+    @classmethod
+    def get_progress_update_identifier(self, progress_update):
+        return progress_update['index']
+
+    def add_progress_update(
+            self, message, initial_step=0, total_steps=0, return_event=False):
+        LOG.info(
+            "Sending progress update for pool '%s' to minion manager : %s",
+            self._pool_id, message)
+        return self._rpc_minion_manager_client.add_minion_pool_progress_update(
+            self._ctxt, self._pool_id, message, initial_step=initial_step,
+            total_steps=total_steps, return_event=return_event)
+
+    def update_progress_update(
+            self, update_identifier, new_current_step,
+            new_total_steps=None, new_message=None):
+        LOG.info(
+            "Updating progress update '%s' for pool '%s' with new step %s",
+            update_identifier, self._pool_id, new_current_step)
+        self._rpc_minion_manager_client.update_minion_pool_progress_update(
+            self._ctxt, self._pool_id, update_identifier, new_current_step,
+            new_total_steps=new_total_steps, new_message=new_message)
+
+    def add_event(self, message, level=constants.TASK_EVENT_INFO):
+        self._rpc_minion_manager_client.add_minion_pool_event(
+            self._ctxt, self._pool_id, level, message)

coriolis/minion_manager/rpc/server.py

@@ -0,0 +1,1836 @@
+# Copyright 2020 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import datetime
+import math
+import uuid
+
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_utils import timeutils
+from taskflow import deciders as taskflow_deciders
+from taskflow.patterns import graph_flow
+from taskflow.patterns import linear_flow
+from taskflow.patterns import unordered_flow
+
+from coriolis import constants
+from coriolis import context
+from coriolis import exception
+from coriolis import keystone
+from coriolis import utils
+from coriolis.conductor.rpc import client as rpc_conductor_client
+from coriolis.cron import cron
+from coriolis.db import api as db_api
+from coriolis.db.sqlalchemy import models
+from coriolis.minion_manager.rpc import client as rpc_minion_manager_client
+from coriolis.minion_manager.rpc import tasks as minion_manager_tasks
+from coriolis.minion_manager.rpc import utils as minion_manager_utils
+from coriolis.scheduler.rpc import client as rpc_scheduler_client
+from coriolis.taskflow import runner as taskflow_runner
+from coriolis.taskflow import utils as taskflow_utils
+from coriolis.worker.rpc import client as rpc_worker_client
+
+
+VERSION = "1.0"
+
+LOG = logging.getLogger(__name__)
+
+MINION_MANAGER_OPTS = [
+    cfg.IntOpt(
+        "minion_pool_default_refresh_period_minutes",
+        default=10,
+        help="Number of minutes in which to refresh minion pools."
+             "Set to 0 to completely disable automatic refreshing.")]
+
+CONF = cfg.CONF
+CONF.register_opts(MINION_MANAGER_OPTS, 'minion_manager')
+
+MINION_POOL_REFRESH_JOB_PREFIX_FORMAT = "pool-%s-refresh"
+MINION_POOL_REFRESH_CRON_JOB_NAME_FORMAT = "pool-%s-refresh-minute-%d"
+MINION_POOL_REFRESH_CRON_JOB_DESCRIPTION_FORMAT = (
+    "Regularly scheduled refresh job for minion pool '%s' on minute %d.")
+
+
+def _trigger_pool_refresh(ctxt, minion_manager_client, minion_pool_id):
+    try:
+        minion_manager_client.refresh_minion_pool(
+            ctxt, minion_pool_id)
+    except exception.InvalidMinionPoolState as ex:
+        LOG.warn(
+            "Minion Pool '%s' is in an invalid state for having a refresh run."
+            " Skipping for now. Error was: %s", minion_pool_id, str(ex))
+
+
+class MinionManagerServerEndpoint(object):
+
+    def __init__(self):
+        self._admin_ctxt = context.get_admin_context()
+        self._scheduler_client_instance = None
+        self._worker_client_instance = None
+        self._conductor_client_instance = None
+        self._replica_cron_client_instance = None
+        self._minion_manager_client_instance = None
+        try:
+            self._cron = cron.Cron()
+            self._init_pools_refresh_cron_jobs()
+            self._cron.start()
+        except Exception as ex:
+            LOG.warn(
+                "A fatal exception occurred while attempting to set up cron "
+                "jobs for automatic pool refreshing. Automatic refreshing will"
+                " not be perfomed until the issue is fixed and the service is "
+                "restarted. Exception details were: %s",
+                utils.get_exception_details())
+
+    def _init_pools_refresh_cron_jobs(self):
+        minion_pools = db_api.get_minion_pools(
+            self._admin_ctxt, include_machines=False,
+            include_progress_updates=False, include_events=False,
+            to_dict=False)
+
+        for minion_pool in minion_pools:
+            active_pool_statuses = [constants.MINION_POOL_STATUS_ALLOCATED]
+            if minion_pool.status not in active_pool_statuses:
+                LOG.debug(
+                    "Not setting any refresh schedules for minion pool '%s' "
+                    "as it is in an inactive status '%s'.",
+                    minion_pool.id, minion_pool.status)
+                continue
+
+            if not minion_pool.maintenance_trust_id:
+                LOG.warn(
+                    "Minion Pool with ID '%s' had no maintenance trust "
+                    "ID associated with it. Cannot set up automatic "
+                    "refreshing during startup. Skipping.",
+                    minion_pool.id)
+                continue
+
+            LOG.debug(
+                "Adding refresh schedule for minion pool '%s' as part of "
+                "server startup.", minion_pool.id)
+            try:
+                self._register_refresh_jobs_for_minion_pool(minion_pool)
+            except Exception as ex:
+                LOG.warn(
+                    "An Exception occurred while setting up automatic "
+                    "refreshing for minion pool with ID '%s'. Error was: %s",
+                    minion_pool.id, utils.get_exception_details())
+
+    def _register_refresh_jobs_for_minion_pool(
+            self, minion_pool, period_minutes=None):
+        if period_minutes is None:
+            period_minutes = CONF.minion_manager.minion_pool_default_refresh_period_minutes
+
+        if period_minutes < 0:
+            LOG.warn(
+                "Got negative pool refresh period %s. Defaulting to 0.",
+                period_minutes)
+            period_minutes = 0
+
+        if period_minutes == 0:
+            LOG.info(
+                "Minon pool refresh period is set to zero. Not setting up "
+                "any automatic refresh jobs for Minion Pool '%s'.",
+                minion_pool.id)
+            return
+
+        if period_minutes > 60:
+            LOG.warn(
+                "Selected pool refresh period_minutes is greater than 60, defaulting "
+                "to 10. Original value was: %s", period_minutes)
+            period_minutes = 10
+        admin_ctxt = context.get_admin_context(
+            minion_pool.maintenance_trust_id)
+        description = (
+            "Scheduled refresh job for minion pool '%s'" % minion_pool.id)
+
+        # NOTE: we need to generate hourly schedules for each minute in
+        # the hour we would like the refresh to be triggered:
+        for minute in [
+                period_minutes * i for i in range(
+                    math.ceil(60 / period_minutes))]:
+            name = MINION_POOL_REFRESH_CRON_JOB_NAME_FORMAT % (
+                minion_pool.id, minute)
+            description = MINION_POOL_REFRESH_CRON_JOB_DESCRIPTION_FORMAT % (
+                minion_pool.id, minute)
+            self._cron.register(
+                cron.CronJob(
+                    name, description, {"minute": minute}, True, None, None,
+                    None, _trigger_pool_refresh, admin_ctxt,
+                    self._rpc_minion_manager_client, minion_pool.id))
+
+    def _unregister_refresh_jobs_for_minion_pool(
+            self, minion_pool, raise_on_error=True):
+        job_prefix = MINION_POOL_REFRESH_JOB_PREFIX_FORMAT % (
+            minion_pool.id)
+        try:
+            self._cron.unregister_jobs_with_prefix(job_prefix)
+        except Exception as ex:
+            if not raise_on_error:
+                LOG.warn(
+                    "Exception occurred while unregistering minion pool "
+                    "refresh  cron jobs for pool with ID '%s'. "
+                    "Exception was: %s",
+                    minion_pool.id, utils.get_exception_details())
+            else:
+                raise
+
+    @property
+    def _taskflow_runner(self):
+        return taskflow_runner.TaskFlowRunner(
+            constants.MINION_MANAGER_MAIN_MESSAGING_TOPIC,
+            max_workers=25)
+
+    # NOTE(aznashwan): it is unsafe to fork processes with pre-instantiated
+    # oslo_messaging clients as the underlying eventlet thread queues will
+    # be invalidated. Considering this class both serves from a "main
+    # process" as well as forking child processes, it is safest to
+    # instantiate the clients only when needed:
+    @property
+    def _rpc_worker_client(self):
+        if not getattr(self, '_worker_client_instance', None):
+            self._worker_client_instance = (
+                rpc_worker_client.WorkerClient())
+        return self._worker_client_instance
+
+    @property
+    def _rpc_scheduler_client(self):
+        if not getattr(self, '_scheduler_client_instance', None):
+            self._scheduler_client_instance = (
+                rpc_scheduler_client.SchedulerClient())
+        return self._scheduler_client_instance
+
+    @property
+    def _rpc_conductor_client(self):
+        if not getattr(self, '_conductor_client_instance', None):
+            self._conductor_client_instance = (
+                rpc_conductor_client.ConductorClient())
+        return self._conductor_client_instance
+
+    @property
+    def _rpc_minion_manager_client(self):
+        if not getattr(self, '_minion_manager_client_instance', None):
+            self._minion_manager_client_instance = (
+                rpc_minion_manager_client.MinionManagerClient())
+        return self._minion_manager_client_instance
+
+    def get_diagnostics(self, ctxt):
+        return utils.get_diagnostics_info()
+
+    def get_endpoint_source_minion_pool_options(
+            self, ctxt, endpoint_id, env, option_names):
+        endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id)
+
+        worker_service = self._rpc_scheduler_client.get_worker_service_for_specs(
+            ctxt, enabled=True,
+            region_sets=[[reg['id'] for reg in endpoint['mapped_regions']]],
+            provider_requirements={
+                endpoint['type']: [
+                    constants.PROVIDER_TYPE_SOURCE_MINION_POOL]})
+        worker_rpc = rpc_worker_client.WorkerClient.from_service_definition(
+            worker_service)
+
+        return worker_rpc.get_endpoint_source_minion_pool_options(
+            ctxt, endpoint['type'], endpoint['connection_info'], env,
+            option_names)
+
+    def get_endpoint_destination_minion_pool_options(
+            self, ctxt, endpoint_id, env, option_names):
+        endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id)
+
+        worker_service = self._rpc_scheduler_client.get_worker_service_for_specs(
+            ctxt, enabled=True,
+            region_sets=[[reg['id'] for reg in endpoint['mapped_regions']]],
+            provider_requirements={
+                endpoint['type']: [
+                    constants.PROVIDER_TYPE_DESTINATION_MINION_POOL]})
+        worker_rpc = rpc_worker_client.WorkerClient.from_service_definition(
+            worker_service)
+
+        return worker_rpc.get_endpoint_destination_minion_pool_options(
+            ctxt, endpoint['type'], endpoint['connection_info'], env,
+            option_names)
+
+    def validate_endpoint_source_minion_pool_options(
+            self, ctxt, endpoint_id, pool_environment):
+        endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id)
+
+        worker_service = self._rpc_scheduler_client.get_worker_service_for_specs(
+            ctxt, enabled=True,
+            region_sets=[[reg['id'] for reg in endpoint['mapped_regions']]],
+            provider_requirements={
+                endpoint['type']: [
+                    constants.PROVIDER_TYPE_SOURCE_MINION_POOL]})
+        worker_rpc = rpc_worker_client.WorkerClient.from_service_definition(
+            worker_service)
+
+        return worker_rpc.validate_endpoint_source_minion_pool_options(
+            ctxt, endpoint['type'], pool_environment)
+
+    def validate_endpoint_destination_minion_pool_options(
+            self, ctxt, endpoint_id, pool_environment):
+        endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id)
+
+        worker_service = self._rpc_scheduler_client.get_worker_service_for_specs(
+            ctxt, enabled=True,
+            region_sets=[[reg['id'] for reg in endpoint['mapped_regions']]],
+            provider_requirements={
+                endpoint['type']: [
+                    constants.PROVIDER_TYPE_DESTINATION_MINION_POOL]})
+        worker_rpc = rpc_worker_client.WorkerClient.from_service_definition(
+            worker_service)
+
+        return worker_rpc.validate_endpoint_destination_minion_pool_options(
+            ctxt, endpoint['type'], pool_environment)
+
+    def _add_minion_pool_event(self, ctxt, minion_pool_id, level, message):
+        LOG.info(
+            "Minion pool event for pool %s: %s", minion_pool_id, message)
+        pool = db_api.get_minion_pool(ctxt, minion_pool_id)
+        db_api.add_minion_pool_event(ctxt, pool.id, level, message)
+
+    @minion_manager_utils.minion_pool_synchronized_op
+    def add_minion_pool_event(self, ctxt, minion_pool_id, level, message):
+        self._add_minion_pool_event(ctxt, minion_pool_id, level, message)
+
+    def _add_minion_pool_progress_update(
+            self, ctxt, minion_pool_id, message, initial_step=0, total_steps=0):
+        LOG.info(
+            "Adding pool progress update for %s: %s", minion_pool_id, message)
+        db_api.add_minion_pool_progress_update(
+            ctxt, minion_pool_id, message, initial_step=initial_step,
+            total_steps=total_steps)
+
+    @minion_manager_utils.minion_pool_synchronized_op
+    def add_minion_pool_progress_update(
+            self, ctxt, minion_pool_id, message, initial_step=0, total_steps=0):
+        self._add_minion_pool_progress_update(
+            ctxt, minion_pool_id, message, initial_step=initial_step,
+            total_steps=total_steps)
+
+    @minion_manager_utils.minion_pool_synchronized_op
+    def update_minion_pool_progress_update(
+            self, ctxt, minion_pool_id, progress_update_index,
+            new_current_step, new_total_steps=None, new_message=None):
+        LOG.info(
+            "Updating minion pool '%s' progress update '%s': %s",
+            minion_pool_id, progress_update_index, new_current_step)
+        db_api.update_minion_pool_progress_update(
+            ctxt, minion_pool_id, progress_update_index, new_current_step,
+            new_total_steps=new_total_steps, new_message=new_message)
+
+    def _check_keys_for_action_dict(
+            self, action, required_action_properties, operation=None):
+        if not isinstance(action, dict):
+            raise exception.InvalidInput(
+                "Action must be a dict, got '%s': %s" % (
+                    type(action), action))
+        missing = [
+            prop for prop in required_action_properties
+            if prop not in action]
+        if missing:
+            raise exception.InvalidInput(
+                "Missing the following required action properties for "
+                "%s: %s. Got %s" % (
+                    operation, missing, action))
+
+    def validate_minion_pool_selections_for_action(self, ctxt, action):
+        """ Validates the minion pool selections for a given action. """
+        required_action_properties = [
+            'id', 'origin_endpoint_id', 'destination_endpoint_id',
+            'origin_minion_pool_id', 'destination_minion_pool_id',
+            'instance_osmorphing_minion_pool_mappings', 'instances']
+        self._check_keys_for_action_dict(
+            action, required_action_properties,
+            operation="minion pool selection validation")
+
+        minion_pools = {
+            pool.id: pool
+            # NOTE: we can just load all the pools in one go to
+            # avoid extraneous DB queries:
+            for pool in db_api.get_minion_pools(
+                ctxt, include_machines=False, include_events=False,
+                include_progress_updates=False, to_dict=False)}
+        def _get_pool(pool_id):
+            pool = minion_pools.get(pool_id)
+            if not pool:
+                raise exception.NotFound(
+                    "Could not find minion pool with ID '%s'." % pool_id)
+            return pool
+        def _check_pool_minion_count(
+                minion_pool, instances, minion_pool_type=""):
+            desired_minion_count = len(instances)
+            if minion_pool.status != constants.MINION_POOL_STATUS_ALLOCATED:
+                raise exception.InvalidMinionPoolState(
+                    "Minion Pool '%s' is an invalid state ('%s') to be "
+                    "used as a %s pool for action '%s'. The pool must be "
+                    "in '%s' status."  % (
+                        minion_pool.id, minion_pool.status,
+                        minion_pool_type.lower(), action['id'],
+                        constants.MINION_POOL_STATUS_ALLOCATED))
+            if desired_minion_count > minion_pool.maximum_minions:
+                msg = (
+                    "Minion Pool with ID '%s' has a lower maximum minion "
+                    "count (%d) than the requested number of minions "
+                    "(%d) to handle all of the instances of action '%s': "
+                    "%s" % (
+                        minion_pool.id, minion_pool.maximum_minions,
+                        desired_minion_count, action['id'], instances))
+                if minion_pool_type:
+                    msg = "%s %s" % (minion_pool_type, msg)
+                raise exception.InvalidMinionPoolSelection(msg)
+
+        # check source pool:
+        instances = action['instances']
+        if action['origin_minion_pool_id']:
+            origin_pool = _get_pool(action['origin_minion_pool_id'])
+            if origin_pool.endpoint_id != action['origin_endpoint_id']:
+                raise exception.InvalidMinionPoolSelection(
+                    "The selected origin minion pool ('%s') belongs to a "
+                    "different Coriolis endpoint ('%s') than the requested "
+                    "origin endpoint ('%s')" % (
+                        action['origin_minion_pool_id'],
+                        origin_pool.endpoint_id,
+                        action['origin_endpoint_id']))
+            if origin_pool.platform != constants.PROVIDER_PLATFORM_SOURCE:
+                raise exception.InvalidMinionPoolSelection(
+                    "The selected origin minion pool ('%s') is configured as a"
+                    " '%s' pool. The pool must be of type %s to be used for "
+                    "data exports." % (
+                        action['origin_minion_pool_id'],
+                        origin_pool.platform,
+                        constants.PROVIDER_PLATFORM_SOURCE))
+            if origin_pool.os_type != constants.OS_TYPE_LINUX:
+                raise exception.InvalidMinionPoolSelection(
+                    "The selected origin minion pool ('%s') is of OS type '%s'"
+                    " instead of the Linux OS type required for a source "
+                    "transfer minion pool." % (
+                        action['origin_minion_pool_id'],
+                        origin_pool.os_type))
+            _check_pool_minion_count(
+                origin_pool, instances, minion_pool_type="Source")
+            LOG.debug(
+                "Successfully validated compatibility of origin minion pool "
+                "'%s' for use with action '%s'." % (
+                    action['origin_minion_pool_id'], action['id']))
+
+        # check destination pool:
+        if action['destination_minion_pool_id']:
+            destination_pool = _get_pool(action['destination_minion_pool_id'])
+            if destination_pool.endpoint_id != (
+                    action['destination_endpoint_id']):
+                raise exception.InvalidMinionPoolSelection(
+                    "The selected destination minion pool ('%s') belongs to a "
+                    "different Coriolis endpoint ('%s') than the requested "
+                    "destination endpoint ('%s')" % (
+                        action['destination_minion_pool_id'],
+                        destination_pool.endpoint_id,
+                        action['destination_endpoint_id']))
+            if destination_pool.platform != (
+                    constants.PROVIDER_PLATFORM_DESTINATION):
+                raise exception.InvalidMinionPoolSelection(
+                    "The selected destination minion pool ('%s') is configured"
+                    " as a '%s'. The pool must be of type %s to be used for "
+                    "data imports." % (
+                        action['destination_minion_pool_id'],
+                        destination_pool.platform,
+                        constants.PROVIDER_PLATFORM_DESTINATION))
+            if destination_pool.os_type != constants.OS_TYPE_LINUX:
+                raise exception.InvalidMinionPoolSelection(
+                    "The selected destination minion pool ('%s') is of OS type"
+                    " '%s' instead of the Linux OS type required for a source "
+                    "transfer minion pool." % (
+                        action['destination_minion_pool_id'],
+                        destination_pool.os_type))
+            _check_pool_minion_count(
+                destination_pool, instances,
+                minion_pool_type="Destination")
+            LOG.debug(
+                "Successfully validated compatibility of destination minion "
+                "pool '%s' for use with action '%s'." % (
+                    action['origin_minion_pool_id'], action['id']))
+
+        # check OSMorphing pool(s):
+        instance_osmorphing_minion_pool_mappings = action.get(
+            'instance_osmorphing_minion_pool_mappings')
+        if instance_osmorphing_minion_pool_mappings:
+            osmorphing_pool_mappings = {}
+            for (instance_id, pool_id) in (
+                    instance_osmorphing_minion_pool_mappings).items():
+                if instance_id not in instances:
+                    LOG.warn(
+                        "Ignoring OSMorphing pool validation for instance with"
+                        " ID '%s' (mapped pool '%s') as it is not part of  "
+                        "action '%s's declared instances: %s",
+                        instance_id, pool_id, action['id'], instances)
+                    continue
+                if pool_id not in osmorphing_pool_mappings:
+                    osmorphing_pool_mappings[pool_id] = [instance_id]
+                else:
+                    osmorphing_pool_mappings[pool_id].append(instance_id)
+
+            for (pool_id, instances_to_osmorph) in osmorphing_pool_mappings.items():
+                osmorphing_pool = _get_pool(pool_id)
+                if osmorphing_pool.endpoint_id != (
+                        action['destination_endpoint_id']):
+                    raise exception.InvalidMinionPoolSelection(
+                        "The selected OSMorphing minion pool for instances %s"
+                        " ('%s') belongs to a different Coriolis endpoint "
+                        "('%s') than the destination endpoint ('%s')" % (
+                            instances_to_osmorph, pool_id,
+                            osmorphing_pool.endpoint_id,
+                            action['destination_endpoint_id']))
+                if osmorphing_pool.platform != (
+                        constants.PROVIDER_PLATFORM_DESTINATION):
+                    raise exception.InvalidMinionPoolSelection(
+                        "The selected OSMorphing minion pool for instances %s "
+                        "('%s') is configured as a '%s' pool. The pool must "
+                        "be of type %s to be used for OSMorphing." % (
+                            instances_to_osmorph, pool_id,
+                            osmorphing_pool.platform,
+                            constants.PROVIDER_PLATFORM_DESTINATION))
+                _check_pool_minion_count(
+                    osmorphing_pool, instances_to_osmorph,
+                    minion_pool_type="OSMorphing")
+                LOG.debug(
+                    "Successfully validated compatibility of destination "
+                    "minion pool '%s' for use as OSMorphing minion for "
+                    "instances %s during action '%s'." % (
+                        pool_id, instances_to_osmorph, action['id']))
+        LOG.debug(
+            "Successfully validated minion pool selections for action '%s' "
+            "with properties: %s", action['id'], action)
+
+    def allocate_minion_machines_for_replica(
+            self, ctxt, replica):
+        try:
+            minion_allocations = self._run_machine_allocation_subflow_for_action(
+                ctxt, replica, constants.TRANSFER_ACTION_TYPE_REPLICA,
+                include_transfer_minions=True,
+                include_osmorphing_minions=False)
+        except Exception as ex:
+            LOG.warn(
+                "Error occurred while allocating minion machines for "
+                "Replica with ID '%s'. Removing all allocations. "
+                "Error was: %s" % (
+                    replica['id'], utils.get_exception_details()))
+            self._cleanup_machines_with_statuses_for_action(
+                ctxt, replica['id'],
+                [constants.MINION_MACHINE_STATUS_UNINITIALIZED])
+            self.deallocate_minion_machines_for_action(
+                ctxt, replica['id'])
+            self._rpc_conductor_client.report_replica_minions_allocation_error(
+                ctxt, replica['id'], str(ex))
+            raise
+
+    def allocate_minion_machines_for_migration(
+            self, ctxt, migration, include_transfer_minions=True,
+            include_osmorphing_minions=True):
+        try:
+            self._run_machine_allocation_subflow_for_action(
+                ctxt, migration,
+                constants.TRANSFER_ACTION_TYPE_MIGRATION,
+                include_transfer_minions=include_transfer_minions,
+                include_osmorphing_minions=include_osmorphing_minions)
+        except Exception as ex:
+            LOG.warn(
+                "Error occurred while allocating minion machines for "
+                "Migration with ID '%s'. Removing all allocations. "
+                "Error was: %s" % (
+                    migration['id'], utils.get_exception_details()))
+            self._cleanup_machines_with_statuses_for_action(
+                ctxt, migration['id'],
+                [constants.MINION_MACHINE_STATUS_UNINITIALIZED])
+            self.deallocate_minion_machines_for_action(
+                ctxt, migration['id'])
+            self._rpc_conductor_client.report_migration_minions_allocation_error(
+                ctxt, migration['id'], str(ex))
+            raise
+
+    def _make_minion_machine_allocation_subflow_for_action(
+            self, ctxt, minion_pool, action_id, action_instances,
+            subflow_name, inject_for_tasks=None):
+        """ Creates a subflow for allocating minion machines from the
+        provided minion pool to the given action (one for each instance)
+
+        Returns a mapping between the action's instaces' IDs and the minion
+        machine ID, as well as the subflow to execute for said machines.
+
+        Returns dict of the form: {
+            "flow": TheFlowClass(),
+            "action_instance_minion_allocation_mappings": {
+                "<action_instance_id>": "<allocated_minion_id>"}}
+        """
+        currently_available_machines = [
+            machine for machine in minion_pool.minion_machines
+            if machine.allocation_status == constants.MINION_MACHINE_STATUS_AVAILABLE]
+        extra_available_machine_slots = (
+            minion_pool.maximum_minions - len(minion_pool.minion_machines))
+        num_instances = len(action_instances)
+        num_currently_available_machines = len(currently_available_machines)
+        if num_instances > (len(currently_available_machines) + (
+                                extra_available_machine_slots)):
+            raise exception.InvalidMinionPoolState(
+                "Minion pool '%s' is unable to accommodate the requested "
+                "number of machines (%s) for transfer action '%s', as it only "
+                "has %d currently available machines, with room to upscale a "
+                "further %d until the maximum is reached. Please either "
+                "increase the number of maximum machines for the pool "
+                "or wait for other minions to become available before "
+                "retrying." % (
+                    minion_pool.id, num_instances, action_id,
+                    num_currently_available_machines,
+                    extra_available_machine_slots))
+
+        def _select_machine(minion_pool, exclude=None):
+            selected_machine = None
+            # NOTE(aznashwan): this will iterate through machines in a set
+            # order every time, thus ensuring that some are preferred over
+            # others and facilitating some to be left unused and thus torn
+            # down during the periodic refreshes:
+            for machine in minion_pool.minion_machines:
+                if exclude and machine.id in exclude:
+                    LOG.debug(
+                        "Excluding minion machine '%s' from search for use "
+                        "action '%s'", machine.id, action_id)
+                    continue
+                if machine.allocation_status != constants.MINION_MACHINE_STATUS_AVAILABLE:
+                    LOG.debug(
+                        "Minion machine with ID '%s' is in status '%s' "
+                        "instead of the expected '%s'. Skipping for use "
+                        "with action '%s'.",
+                        machine.id, machine.allocation_status,
+                        constants.MINION_MACHINE_STATUS_AVAILABLE, action_id)
+                    continue
+                selected_machine = machine
+                break
+            return selected_machine
+
+        allocation_subflow = unordered_flow.Flow(subflow_name)
+        instance_minion_allocations = {}
+        machine_db_entries_to_add = []
+        existing_machines_to_allocate = {}
+        for instance in action_instances:
+
+            if instance in instance_minion_allocations:
+                raise exception.InvalidInput(
+                    "Instance with identifier '%s' passed twice for "
+                    "minion machine allocation from pool '%s' for action "
+                    "'%s'. Full instances list was: %s" % (
+                        instance, minion_pool.id, action_id, action_instances))
+            minion_machine = _select_machine(
+                minion_pool, exclude=instance_minion_allocations.values())
+            if minion_machine:
+                # take note of the machine and setup a healthcheck:
+                instance_minion_allocations[instance] = minion_machine.id
+                existing_machines_to_allocate[minion_machine.id] = instance
+                LOG.debug(
+                    "Allocating pre-existing machine '%s' from pool '%s' for "
+                    "use with action with ID '%s'.",
+                    minion_machine.id, minion_pool.id, action_id)
+                allocation_subflow.add(
+                    self._get_healtchcheck_flow_for_minion_machine(
+                        minion_pool, minion_machine,
+                        allocate_to_action=action_id,
+                        power_on_machine=True,
+                        inject_for_tasks=inject_for_tasks,
+                        machine_status_on_success=(
+                            constants.MINION_MACHINE_STATUS_IN_USE)))
+            else:
+                # add task which creates the new machine:
+                new_machine_id = str(uuid.uuid4())
+                LOG.debug(
+                    "New minion machine with ID '%s' will be created for "
+                    "minion pool '%s' for use with action '%s'.",
+                    new_machine_id, minion_pool.id, action_id)
+
+                new_minion_machine = models.MinionMachine()
+                new_minion_machine.id = new_machine_id
+                new_minion_machine.pool_id = minion_pool.id
+                new_minion_machine.allocation_status = (
+                    constants.MINION_MACHINE_STATUS_UNINITIALIZED)
+                new_minion_machine.power_status = (
+                    constants.MINION_MACHINE_POWER_STATUS_UNINITIALIZED)
+                new_minion_machine.allocated_action = action_id
+                machine_db_entries_to_add.append(new_minion_machine)
+
+                instance_minion_allocations[instance] = new_machine_id
+                allocation_subflow.add(
+                    minion_manager_tasks.AllocateMinionMachineTask(
+                        minion_pool.id, new_machine_id, minion_pool.platform,
+                        allocate_to_action=action_id,
+                        raise_on_cleanup_failure=False,
+                        inject=inject_for_tasks))
+
+        new_machine_db_entries_added = []
+        try:
+            if existing_machines_to_allocate:
+                # mark any existing machines as allocated:
+                LOG.debug(
+                    "Marking the following pre-existing minion machines "
+                    "from pool '%s' of action '%s' for each instance as "
+                    "allocated with the DB: %s",
+                    minion_pool.id, action_id, existing_machines_to_allocate)
+                db_api.set_minion_machines_allocation_statuses(
+                    ctxt, list(existing_machines_to_allocate.keys()),
+                    action_id, constants.MINION_MACHINE_STATUS_RESERVED,
+                    refresh_allocation_time=True)
+                self._add_minion_pool_event(
+                    ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                    "The following pre-existing minion machines will be "
+                    "allocated to transfer action '%s': %s" % (
+                        action_id, list(existing_machines_to_allocate.keys())))
+
+            # add any new machine entries to the DB:
+            if machine_db_entries_to_add:
+                for new_machine in machine_db_entries_to_add:
+                    LOG.info(
+                        "Adding new minion machine with ID '%s' to the DB for pool "
+                        "'%s' for use with action '%s'.",
+                        new_machine_id, minion_pool.id, action_id)
+                    db_api.add_minion_machine(ctxt, new_machine)
+                    new_machine_db_entries_added.append(new_machine.id)
+                self._add_minion_pool_event(
+                    ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                    "The following new minion machines will be created for use"
+                    " in transfer action '%s': %s" % (
+                        action_id, [m.id for m in machine_db_entries_to_add]))
+        except Exception as ex:
+            LOG.warn(
+                "Exception occurred while adding new minion machine entries to"
+                " the DB for pool '%s' for use with action '%s'. Clearing "
+                "any DB entries added so far (%s). Error was: %s",
+                minion_pool.id, action_id,
+                [m.id for m in new_machine_db_entries_added],
+                utils.get_exception_details())
+            try:
+                LOG.debug(
+                    "Reverting the following pre-existing minion machines from"
+                    " pool '%s' to '%s' due to allocation error for action "
+                    "'%s': %s",
+                    minion_pool.id,
+                    constants.MINION_MACHINE_STATUS_AVAILABLE,
+                    action_id,
+                    list(existing_machines_to_allocate.keys()))
+                db_api.set_minion_machines_allocation_statuses(
+                    ctxt, list(existing_machines_to_allocate.keys()),
+                    None, constants.MINION_MACHINE_STATUS_AVAILABLE,
+                    refresh_allocation_time=False)
+            except Exception:
+                LOG.warn(
+                    "Failed to deallocate the following machines from pool "
+                    "'%s' following allocation error for action '%s': %s. "
+                    "Error trace was: %s",
+                    minion_pool.id, action_id, existing_machines_to_allocate,
+                    utils.get_exception_details())
+            for new_machine in new_machine_db_entries_added:
+                try:
+                    db_api.delete_minion_machine(ctxt, new_machine.id)
+                except Exception as ex:
+                    LOG.warn(
+                        "Error occurred while removing minion machine entry "
+                        "'%s' from the DB. This may leave the pool in an "
+                        "inconsistent state. Error trace was: %s" % (
+                            new_machine.id, utils.get_exception_details()))
+                    continue
+            raise
+
+        LOG.debug(
+            "The following minion machine allocation from pool '%s' were or "
+            "will be made for action '%s': %s",
+            minion_pool.id, action_id, instance_minion_allocations)
+        return {
+            "flow": allocation_subflow,
+            "action_instance_minion_allocation_mappings": (
+                instance_minion_allocations)}
+
+    def _run_machine_allocation_subflow_for_action(
+            self, ctxt, action, action_type, include_transfer_minions=True,
+            include_osmorphing_minions=True):
+        """ Defines and starts a taskflow subflow for allocating minion
+        machines for the given action.
+        If there are no more minion machines available, upscaling will occur.
+        Also adds to the DB/marks as allocated any minion machines on the
+        spot.
+        """
+        required_action_properties = [
+            'id', 'instances', 'origin_minion_pool_id',
+            'destination_minion_pool_id',
+            'instance_osmorphing_minion_pool_mappings']
+        self._check_keys_for_action_dict(
+            action, required_action_properties,
+            operation="minion machine selection")
+
+        allocation_flow_name_format = None
+        machines_allocation_subflow_name_format = None
+        machine_action_allocation_subflow_name_format = None
+        allocation_failure_reporting_task_class = None
+        allocation_confirmation_reporting_task_class = None
+        if action_type == constants.TRANSFER_ACTION_TYPE_MIGRATION:
+            allocation_flow_name_format = (
+                minion_manager_tasks.MINION_POOL_MIGRATION_ALLOCATION_FLOW_NAME_FORMAT)
+            allocation_failure_reporting_task_class = (
+                minion_manager_tasks.ReportMinionAllocationFailureForMigrationTask)
+            allocation_confirmation_reporting_task_class = (
+                minion_manager_tasks.ConfirmMinionAllocationForMigrationTask)
+            machines_allocation_subflow_name_format = (
+                minion_manager_tasks.MINION_POOL_MIGRATION_ALLOCATION_SUBFLOW_NAME_FORMAT)
+            machine_action_allocation_subflow_name_format = (
+                minion_manager_tasks.MINION_POOL_ALLOCATE_MACHINES_FOR_MIGRATION_SUBFLOW_NAME_FORMAT)
+        elif action_type == constants.TRANSFER_ACTION_TYPE_REPLICA:
+            allocation_flow_name_format = (
+                minion_manager_tasks.MINION_POOL_REPLICA_ALLOCATION_FLOW_NAME_FORMAT)
+            allocation_failure_reporting_task_class = (
+                minion_manager_tasks.ReportMinionAllocationFailureForReplicaTask)
+            allocation_confirmation_reporting_task_class = (
+                minion_manager_tasks.ConfirmMinionAllocationForReplicaTask)
+            machines_allocation_subflow_name_format = (
+                minion_manager_tasks.MINION_POOL_REPLICA_ALLOCATION_SUBFLOW_NAME_FORMAT)
+            machine_action_allocation_subflow_name_format = (
+                minion_manager_tasks.MINION_POOL_ALLOCATE_MACHINES_FOR_REPLICA_SUBFLOW_NAME_FORMAT)
+        else:
+            raise exception.InvalidInput(
+                "Unknown transfer action type '%s'" % action_type)
+
+        # define main flow:
+        main_allocation_flow_name = (
+            allocation_flow_name_format % action['id'])
+        main_allocation_flow = linear_flow.Flow(main_allocation_flow_name)
+        instance_machine_allocations = {
+            instance: {} for instance in action['instances']}
+
+        # add allocation failure reporting task:
+        main_allocation_flow.add(
+            allocation_failure_reporting_task_class(
+                action['id']))
+
+        # define subflow for all the pool minions allocations:
+        machines_subflow = unordered_flow.Flow(
+            machines_allocation_subflow_name_format % action['id'])
+        new_pools_machines_db_entries = {}
+        pools_used = []
+
+        # add subflow for origin pool:
+        if include_transfer_minions and action['origin_minion_pool_id']:
+            pools_used.append(action['origin_minion_pool_id'])
+            with minion_manager_utils.get_minion_pool_lock(
+                    action['origin_minion_pool_id'], external=True):
+                # fetch pool, origin endpoint, and initial store:
+                minion_pool = self._get_minion_pool(
+                    ctxt, action['origin_minion_pool_id'],
+                    include_machines=True, include_events=False,
+                    include_progress_updates=False)
+                endpoint_dict = self._rpc_conductor_client.get_endpoint(
+                    ctxt, minion_pool.endpoint_id)
+                origin_pool_store = self._get_pool_initial_taskflow_store_base(
+                    ctxt, minion_pool, endpoint_dict)
+
+                # add subflow for machine allocations from origin pool:
+                subflow_name = machine_action_allocation_subflow_name_format % (
+                    minion_pool.id, action['id'])
+                # NOTE: required to avoid internal taskflow conflicts
+                subflow_name = "origin-%s" % subflow_name
+                allocations_subflow_result = (
+                    self._make_minion_machine_allocation_subflow_for_action(
+                        ctxt, minion_pool, action['id'], action['instances'],
+                        subflow_name, inject_for_tasks=origin_pool_store))
+                machines_subflow.add(allocations_subflow_result['flow'])
+
+                # register each instances' origin minion:
+                source_machine_allocations = allocations_subflow_result[
+                    'action_instance_minion_allocation_mappings']
+                for (action_instance_id, allocated_minion_id) in (
+                        source_machine_allocations.items()):
+                    instance_machine_allocations[
+                        action_instance_id]['origin_minion_id'] = (
+                            allocated_minion_id)
+
+        # add subflow for destination pool:
+        if include_transfer_minions and action['destination_minion_pool_id']:
+            pools_used.append(action['destination_minion_pool_id'])
+            with minion_manager_utils.get_minion_pool_lock(
+                    action['destination_minion_pool_id'], external=True):
+                # fetch pool, destination endpoint, and initial store:
+                minion_pool = self._get_minion_pool(
+                    ctxt, action['destination_minion_pool_id'],
+                    include_machines=True, include_events=False,
+                    include_progress_updates=False)
+                endpoint_dict = self._rpc_conductor_client.get_endpoint(
+                    ctxt, minion_pool.endpoint_id)
+                destination_pool_store = (
+                    self._get_pool_initial_taskflow_store_base(
+                        ctxt, minion_pool, endpoint_dict))
+
+                # add subflow for machine allocations from destination pool:
+                subflow_name = machine_action_allocation_subflow_name_format % (
+                    minion_pool.id, action['id'])
+                # NOTE: required to avoid internal taskflow conflicts
+                subflow_name = "destination-%s" % subflow_name
+                allocations_subflow_result = (
+                    self._make_minion_machine_allocation_subflow_for_action(
+                        ctxt, minion_pool, action['id'], action['instances'],
+                        subflow_name,
+                        inject_for_tasks=destination_pool_store))
+                machines_subflow.add(allocations_subflow_result['flow'])
+                destination_machine_allocations = allocations_subflow_result[
+                    'action_instance_minion_allocation_mappings']
+
+                # register each instances' destination minion:
+                for (action_instance_id, allocated_minion_id) in (
+                        destination_machine_allocations.items()):
+                    instance_machine_allocations[
+                        action_instance_id]['destination_minion_id'] = (
+                            allocated_minion_id)
+
+        # add subflow for OSMorphing minions:
+        osmorphing_pool_instance_mappings = {}
+        for (action_instance_id, mapped_pool_id) in action[
+                'instance_osmorphing_minion_pool_mappings'].items():
+            if mapped_pool_id not in osmorphing_pool_instance_mappings:
+                osmorphing_pool_instance_mappings[
+                    mapped_pool_id] = [action_instance_id]
+            else:
+                osmorphing_pool_instance_mappings[mapped_pool_id].append(
+                    action_instance_id)
+        if include_osmorphing_minions and osmorphing_pool_instance_mappings:
+            for (osmorphing_pool_id, action_instance_ids) in (
+                    osmorphing_pool_instance_mappings.items()):
+                # if the destination pool was selected as an OSMorphing pool
+                # for any instances, we simply re-use all of the destination
+                # minions for said instances:
+                if action['destination_minion_pool_id'] and (
+                        include_osmorphing_minions and (
+                            osmorphing_pool_id == (
+                                action['destination_minion_pool_id']))):
+                    LOG.debug(
+                        "Reusing destination minion pool with ID '%s' for the "
+                        "following instances which had it selected as an "
+                        "OSMorphing pool for action '%s': %s",
+                        osmorphing_pool_id, action['id'], action_instance_ids)
+                    for instance in action_instance_ids:
+                        instance_machine_allocations[
+                            instance]['osmorphing_minion_id'] = (
+                                instance_machine_allocations[
+                                    instance]['destination_minion_id'])
+                    continue
+
+                with minion_manager_utils.get_minion_pool_lock(
+                        osmorphing_pool_id, external=True):
+                    pools_used.append(osmorphing_pool_id)
+                    # fetch pool, destination endpoint, and initial store:
+                    minion_pool = self._get_minion_pool(
+                        ctxt, osmorphing_pool_id,
+                        include_machines=True, include_events=False,
+                        include_progress_updates=False)
+                    endpoint_dict = self._rpc_conductor_client.get_endpoint(
+                        ctxt, minion_pool.endpoint_id)
+                    osmorphing_pool_store = self._get_pool_initial_taskflow_store_base(
+                        ctxt, minion_pool, endpoint_dict)
+
+                    # add subflow for machine allocations from osmorphing pool:
+                    subflow_name = machine_action_allocation_subflow_name_format % (
+                        minion_pool.id, action['id'])
+                    # NOTE: required to avoid internal taskflow conflicts
+                    subflow_name = "osmorphing-%s" % subflow_name
+                    allocations_subflow_result = (
+                        self._make_minion_machine_allocation_subflow_for_action(
+                            ctxt, minion_pool, action['id'],
+                            action_instance_ids,
+                            subflow_name, inject_for_tasks=osmorphing_pool_store))
+                    machines_subflow.add(allocations_subflow_result['flow'])
+
+                    # register each instances' osmorphing minion:
+                    osmorphing_machine_allocations = allocations_subflow_result[
+                        'action_instance_minion_allocation_mappings']
+                    for (action_instance_id, allocated_minion_id) in (
+                            osmorphing_machine_allocations.items()):
+                        instance_machine_allocations[
+                            action_instance_id]['osmorphing_minion_id'] = (
+                                allocated_minion_id)
+
+        # add the machines subflow to the main flow:
+        main_allocation_flow.add(machines_subflow)
+
+        # add final task to report minion machine availablity
+        # to the conductor at the end of the flow:
+        main_allocation_flow.add(
+            allocation_confirmation_reporting_task_class(
+                action['id'], instance_machine_allocations))
+
+        LOG.info(
+            "Starting main minion allocation flow '%s' for with ID '%s'. "
+            "The minion allocations will be: %s" % (
+                main_allocation_flow_name, action['id'],
+                instance_machine_allocations))
+
+        try:
+            self._taskflow_runner.run_flow_in_background(
+                main_allocation_flow, store={"context": ctxt})
+        except Exception as ex:
+            minion_pool_id = None
+            try:
+                for minion_pool_id in pools_used:
+                    self._add_minion_pool_event(
+                        ctxt, minion_pool.id, constants.TASK_EVENT_ERROR,
+                        "A fatal exception occurred while attempting to start "
+                        "the task flow for allocating machines for %s '%s'. "
+                        "Forced deallocation and reallocation may be required."
+                        " Please review the minion manager logs for additional"
+                        " details. Error was: %s" % (
+                            action_type, action['id'], str(ex)))
+            except Exception:
+                LOG.warn(
+                    "Failed to add minion pool error event for pool '%s' "
+                    "during allocation of machines for %s '%s'. Ignoring. "
+                    "Exception was: %s",
+                    minion_pool_id, action_type, action['id'],
+                    utils.get_exception_details())
+            raise
+
+        return main_allocation_flow
+
+    def _cleanup_machines_with_statuses_for_action(
+            self, ctxt, action_id, targeted_statuses, exclude_pools=None):
+        """ Deletes all minion machines which are marked with the given
+        from the DB.
+        """
+        if exclude_pools is None:
+            exclude_pools = []
+        machines = db_api.get_minion_machines(ctxt, action_id)
+        if not machines:
+            LOG.debug(
+                "No minion machines allocated to action '%s'. Returning.",
+                action_id)
+            return
+
+        pool_machine_mappings = {}
+        for machine in machines:
+            if machine.allocation_status not in targeted_statuses:
+                LOG.debug(
+                    "Skipping deletion of machine '%s' from pool '%s' as "
+                    "its status (%s) is not one of the targeted statuses (%s)",
+                    machine.id, machine.pool_id, machine.allocation_status,
+                    targeted_statuses)
+                continue
+            if machine.pool_id in exclude_pools:
+                LOG.debug(
+                    "Skipping deletion of machine '%s' (status '%s') from "
+                    "whitelisted pool '%s'", machine.id, machine.allocation_status,
+                    machine.pool_id)
+                continue
+
+            if machine.pool_id not in pool_machine_mappings:
+                pool_machine_mappings[machine.pool_id] = [machine]
+            else:
+                pool_machine_mappings[machine.pool_id].append(machine)
+
+        for (pool_id, machines) in pool_machine_mappings.items():
+            with minion_manager_utils.get_minion_pool_lock(
+                   pool_id, external=True):
+                for machine in machines:
+                    LOG.debug(
+                        "Deleting machine with ID '%s' (pool '%s', status '%s') "
+                        "from the DB.", machine.id, pool_id, machine.allocation_status)
+                    db_api.delete_minion_machine(ctxt, machine.id)
+
+    def deallocate_minion_machine(self, ctxt, minion_machine_id):
+
+        minion_machine = db_api.get_minion_machine(
+            ctxt, minion_machine_id)
+        if not minion_machine:
+            LOG.warn(
+                "Could not find minion machine with ID '%s' for deallocation. "
+                "Presuming it was deleted and returning early",
+                minion_machine_id)
+            return
+
+        machine_allocated_status = constants.MINION_MACHINE_STATUS_IN_USE
+        with minion_manager_utils.get_minion_pool_lock(
+                minion_machine.pool_id, external=True):
+            if minion_machine.allocation_status != machine_allocated_status or (
+                    not minion_machine.allocated_action):
+                LOG.warn(
+                    "Minion machine '%s' was either in an improper status (%s)"
+                    ", or did not have an associated action ('%s') for "
+                    "deallocation request. Marking as available anyway.",
+                    minion_machine.id, minion_machine.allocation_status,
+                    minion_machine.allocated_action)
+            LOG.debug(
+                "Attempting to deallocate all minion pool machine '%s' "
+                "(currently allocated to action '%s' with status '%s')",
+                minion_machine.id, minion_machine.allocated_action,
+                minion_machine.allocation_status)
+            db_api.update_minion_machine(
+                ctxt, minion_machine.id, {
+                    "allocation_status": constants.MINION_MACHINE_STATUS_AVAILABLE,
+                    "allocated_action": None})
+            LOG.debug(
+                "Successfully deallocated minion machine with '%s'.",
+                minion_machine.id)
+
+    def deallocate_minion_machines_for_action(self, ctxt, action_id):
+
+        allocated_minion_machines = db_api.get_minion_machines(
+            ctxt, allocated_action_id=action_id)
+
+        if not allocated_minion_machines:
+            LOG.debug(
+                "No minion machines seem to have been used for action with "
+                "base_id '%s'. Skipping minion machine deallocation.",
+                action_id)
+            return
+
+        # categorise machine objects by pool:
+        pool_machine_mappings = {}
+        for machine in allocated_minion_machines:
+            if machine.pool_id not in pool_machine_mappings:
+                pool_machine_mappings[machine.pool_id] = []
+            pool_machine_mappings[machine.pool_id].append(machine)
+
+        # iterate over each pool and its machines allocated to this action:
+        for (pool_id, pool_machines) in pool_machine_mappings.items():
+            with minion_manager_utils.get_minion_pool_lock(
+                    pool_id, external=True):
+                machine_ids_to_deallocate = []
+                # NOTE: this is a workaround in case some crash/restart happens
+                # in the minion-manager service while new machine DB entries
+                # are added to the DB without their point of deployment being
+                # reached for them to ever get out of 'UNINITIALIZED' status:
+                for machine in pool_machines:
+                    if machine.allocation_status == (
+                            constants.MINION_MACHINE_STATUS_UNINITIALIZED):
+                        LOG.warn(
+                            "Found minion machine '%s' in pool '%s' which "
+                            "is in '%s' status. Removing from the DB "
+                            "entirely." % (
+                                machine.id, pool_id, machine.allocation_status))
+                        db_api.delete_minion_machine(
+                            ctxt, machine.id)
+                        LOG.info(
+                            "Successfully deleted minion machine entry '%s' "
+                            "from pool '%s' from the DB.", machine.id, pool_id)
+                        continue
+                    LOG.debug(
+                        "Going to mark minion machine '%s' (current status "
+                        "'%s') of pool '%s' as available following machine "
+                        "deallocation request for action '%s'.",
+                        machine.id, machine.allocation_status, pool_id, action_id)
+                    machine_ids_to_deallocate.append(machine.id)
+
+                LOG.info(
+                    "Marking minion machines '%s' from pool '%s' for "
+                    "as available after having been allocated to action '%s'.",
+                    machine_ids_to_deallocate, pool_id, action_id)
+                db_api.set_minion_machines_allocation_statuses(
+                    ctxt, machine_ids_to_deallocate, None,
+                    constants.MINION_MACHINE_STATUS_AVAILABLE,
+                    refresh_allocation_time=False)
+
+        LOG.debug(
+            "Successfully released all minion machines associated "
+            "with action with base_id '%s'.", action_id)
+
+    def _get_healtchcheck_flow_for_minion_machine(
+            self, minion_pool, minion_machine, allocate_to_action=None,
+            machine_status_on_success=constants.MINION_MACHINE_STATUS_AVAILABLE,
+            power_on_machine=True, inject_for_tasks=None):
+        """ Returns a taskflow graph flow with a healtcheck task
+        and redeployment subflow on error. """
+        # define healthcheck subflow for each machine:
+        machine_healthcheck_subflow = graph_flow.Flow(
+            minion_manager_tasks.MINION_POOL_HEALTHCHECK_MACHINE_SUBFLOW_NAME_FORMAT % (
+                minion_pool.id, minion_machine.id))
+
+        # add healtcheck task to healthcheck subflow:
+        machine_healthcheck_task = (
+            minion_manager_tasks.HealthcheckMinionMachineTask(
+                minion_pool.id, minion_machine.id, minion_pool.platform,
+                machine_status_on_success=machine_status_on_success,
+                inject=inject_for_tasks,
+                # we prevent a raise here as the healthcheck subflow
+                # will take care of redeploying the instance later:
+                fail_on_error=False))
+        machine_healthcheck_subflow.add(machine_healthcheck_task)
+
+        # optionally add minion machine power on task:
+        if power_on_machine:
+            if minion_machine.power_status == (
+                    constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF):
+                power_on_task = minion_manager_tasks.PowerOnMinionMachineTask(
+                    minion_pool.id, minion_machine.id, minion_pool.platform,
+                    inject=inject_for_tasks,
+                    # we prevent a raise here as the healthcheck subflow
+                    # will take care of redeploying the instance later:
+                    fail_on_error=False)
+                machine_healthcheck_subflow.add(
+                    power_on_task,
+                    # NOTE: this is required to not have taskflow attempt
+                    # (and fail) to automatically link the above Healthcheck
+                    # task to the power on task based on inputs/outputs alone:
+                    resolve_existing=False)
+                machine_healthcheck_subflow.link(
+                    power_on_task, machine_healthcheck_task,
+                    # NOTE: taskflow gets confused when a task in the graph
+                    # flow is linked to two others with no decider for each
+                    # so we have to add a dummy decider which will always
+                    # greenlight the rest of the execution here:
+                    decider=taskflow_utils.DummyDecider(allow=True),
+                    decider_depth=taskflow_deciders.Depth.FLOW)
+            else:
+                LOG.debug(
+                    "Minion Machine with ID '%s' of pool '%s' is in power "
+                    "state '%s' during healtchcheck subflow definition. "
+                    "Not adding any power on task for it.",
+                    minion_machine.id, minion_machine.pool_id,
+                    minion_machine.power_status)
+
+        # define reallocation subflow:
+        machine_reallocation_subflow = linear_flow.Flow(
+            minion_manager_tasks.MINION_POOL_REALLOCATE_MACHINE_SUBFLOW_NAME_FORMAT % (
+                minion_pool.id, minion_machine.id))
+        machine_reallocation_subflow.add(
+            minion_manager_tasks.DeallocateMinionMachineTask(
+                minion_pool.id, minion_machine.id, minion_pool.platform,
+                inject=inject_for_tasks))
+        machine_reallocation_subflow.add(
+            minion_manager_tasks.AllocateMinionMachineTask(
+                minion_pool.id, minion_machine.id, minion_pool.platform,
+                allocate_to_action=allocate_to_action,
+                inject=inject_for_tasks))
+        machine_healthcheck_subflow.add(
+            machine_reallocation_subflow,
+            # NOTE: this is required to not have taskflow attempt (and fail)
+            # to automatically link the above Healthcheck task to the
+            # new subflow based on inputs/outputs alone:
+            resolve_existing=False)
+
+        # link reallocation subflow to healthcheck task:
+        machine_healthcheck_subflow.link(
+            machine_healthcheck_task, machine_reallocation_subflow,
+            # NOTE: this is required to prevent any parent flows from skipping:
+            decider_depth=taskflow_deciders.Depth.FLOW,
+            decider=minion_manager_tasks.MinionMachineHealtchcheckDecider(
+                minion_pool.id, minion_machine.id,
+                on_successful_healthcheck=False))
+
+        return machine_healthcheck_subflow
+
+    def _get_minion_pool_refresh_flow(
+            self, ctxt, minion_pool, requery=True):
+
+        if requery:
+            minion_pool = self._get_minion_pool(
+                ctxt, minion_pool.id, include_machines=True,
+                include_progress_updates=False, include_events=False)
+
+        # determine how many machines could be feasibily downscaled:
+        machine_statuses = {
+            machine.id: machine.allocation_status
+            for machine in minion_pool.minion_machines}
+        ignorable_machine_statuses = [
+            constants.MINION_MACHINE_STATUS_DEALLOCATING,
+            constants.MINION_MACHINE_STATUS_POWERING_OFF,
+            constants.MINION_MACHINE_STATUS_ERROR,
+            constants.MINION_MACHINE_STATUS_POWER_ERROR,
+            constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING]
+        max_minions_to_deallocate = (
+            len([
+                mid for mid in machine_statuses
+                if machine_statuses[mid] not in ignorable_machine_statuses]) - (
+                    minion_pool.minimum_minions))
+        LOG.debug(
+            "Determined minion pool '%s' machine deallocation number to be %d "
+            "(pool minimum is '%d') based on current machines stauses: %s",
+            minion_pool.id, max_minions_to_deallocate,
+            minion_pool.minimum_minions, machine_statuses)
+
+        # define refresh flow and process all relevant machines:
+        pool_refresh_flow = unordered_flow.Flow(
+            minion_manager_tasks.MINION_POOL_REFRESH_FLOW_NAME_FORMAT % (
+                minion_pool.id))
+        now = timeutils.utcnow()
+        machines_to_deallocate = []
+        machines_to_healthcheck = []
+        skipped_machines = {}
+        healthcheckable_machine_statuses = [
+            constants.MINION_MACHINE_STATUS_AVAILABLE,
+            # NOTE(aznashwan): this should help account for 'transient' issues
+            # where a minion which may have been marked as error'd at some
+            # point may be back online. Event if it isn't, the
+            # sublow redeploy it after the healthcheck fails:
+            constants.MINION_MACHINE_STATUS_ERROR,
+            constants.MINION_MACHINE_STATUS_POWER_ERROR,
+            constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING]
+
+        for machine in minion_pool.minion_machines:
+            if machine.allocation_status not in (
+                    healthcheckable_machine_statuses):
+                skipped_machines[machine.id] = (
+                    machine.allocation_status, machine.power_status)
+                continue
+
+            minion_expired = True
+            if machine.last_used_at:
+                expiry_time = (
+                    machine.last_used_at + datetime.timedelta(
+                        seconds=minion_pool.minion_max_idle_time))
+                minion_expired = expiry_time <= now
+
+            # deallocate the machine if it is expired:
+            if max_minions_to_deallocate > 0 and minion_expired:
+                if minion_pool.minion_retention_strategy == (
+                        constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_POWEROFF):
+                    if machine.power_status in (
+                            constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF,
+                            constants.MINION_MACHINE_POWER_STATUS_POWERING_OFF):
+                        LOG.debug(
+                            "Skipping powering off minion machine '%s' of pool"
+                            " '%s' as it is already in powered off state.",
+                            machine.id, minion_pool.id)
+                        # NOTE: we count this machine out of the downscaling:
+                        max_minions_to_deallocate = (
+                            max_minions_to_deallocate - 1)
+                        continue
+                    LOG.debug(
+                        "Minion machine '%s' of pool '%s' will be powered off "
+                        "as part of the pool refresh process (current "
+                        "deallocation count %d excluding the current machine)",
+                        machine.id, minion_pool.id, max_minions_to_deallocate)
+                    pool_refresh_flow.add(
+                        minion_manager_tasks.PowerOffMinionMachineTask(
+                            minion_pool.id, machine.id, minion_pool.platform,
+                            fail_on_error=False,
+                            status_once_powered_off=(
+                                constants.MINION_MACHINE_STATUS_AVAILABLE)))
+                elif minion_pool.minion_retention_strategy == (
+                        constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE):
+                    pool_refresh_flow.add(
+                        minion_manager_tasks.DeallocateMinionMachineTask(
+                                minion_pool.id, machine.id,
+                                minion_pool.platform))
+                else:
+                    raise exception.InvalidMinionPoolState(
+                        "Unknown minion pool retention strategy '%s' for pool "
+                        "'%s'" % (
+                            minion_pool.minion_retention_strategy,
+                            minion_pool.id))
+                max_minions_to_deallocate = max_minions_to_deallocate - 1
+                machines_to_deallocate.append(machine.id)
+            # else, perform a healthcheck on the machine if it is powered on:
+            elif machine.power_status == (
+                    constants.MINION_MACHINE_POWER_STATUS_POWERED_ON):
+                pool_refresh_flow.add(
+                    self._get_healtchcheck_flow_for_minion_machine(
+                        minion_pool, machine, allocate_to_action=None,
+                        machine_status_on_success=(
+                            constants.MINION_MACHINE_STATUS_AVAILABLE)))
+                machines_to_healthcheck.append(machine.id)
+            else:
+                skipped_machines[machine.id] = (
+                    machine.allocation_status, machine.power_status)
+
+        # update DB entried for all machines and emit relevant events:
+        if skipped_machines:
+            base_msg =  (
+                "The following minion machines were skipped during the "
+                "refreshing of the minion pool as they were in other "
+                "statuses than the serviceable ones: %s")
+            LOG.debug(
+                "[Pool '%s'] %s: %s",
+                minion_pool.id, base_msg, skipped_machines)
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                base_msg % list(skipped_machines.keys()))
+
+        if machines_to_deallocate:
+            deallocation_action = "deallocated"
+            status_for_deallocated_machines = (
+                constants.MINION_MACHINE_STATUS_DEALLOCATING)
+            if minion_pool.minion_retention_strategy == (
+                    constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_POWEROFF):
+                deallocation_action = "powered off"
+                status_for_deallocated_machines = (
+                    constants.MINION_MACHINE_STATUS_POWERING_OFF)
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                "The following minion machines will be %s as part "
+                "of the refreshing of the minion pool: %s" % (
+                    deallocation_action, machines_to_deallocate))
+            for machine in machines_to_deallocate:
+                db_api.set_minion_machine_allocation_status(
+                    ctxt, machine, status_for_deallocated_machines)
+        else:
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                "No minion machines require deallocation during pool refresh")
+
+        if machines_to_healthcheck:
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                "The following minion machines will be healthchecked as part "
+                "of the refreshing of the minion pool: %s" % (
+                    machines_to_healthcheck))
+            for machine in machines_to_healthcheck:
+                db_api.set_minion_machine_allocation_status(
+                    ctxt, machine,
+                    constants.MINION_MACHINE_STATUS_HEALTHCHECKING)
+        else:
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                "No minion machines require healthchecking during "
+                "pool refresh")
+
+        return pool_refresh_flow
+
+    @minion_manager_utils.minion_pool_synchronized_op
+    def refresh_minion_pool(self, ctxt, minion_pool_id):
+        LOG.info("Attempting to healthcheck Minion Pool '%s'.", minion_pool_id)
+        minion_pool = self._get_minion_pool(
+            ctxt, minion_pool_id, include_events=False, include_machines=True,
+            include_progress_updates=False)
+        endpoint_dict = self._rpc_conductor_client.get_endpoint(
+            ctxt, minion_pool.endpoint_id)
+        acceptable_allocation_statuses = [
+            constants.MINION_POOL_STATUS_ALLOCATED]
+        current_status = minion_pool.status
+        if current_status not in acceptable_allocation_statuses:
+            raise exception.InvalidMinionPoolState(
+                "Minion machines for pool '%s' cannot be healthchecked as the "
+                "pool is in '%s' state instead of the expected %s." % (
+                    minion_pool_id, current_status,
+                    acceptable_allocation_statuses))
+
+        refresh_flow = self._get_minion_pool_refresh_flow(
+            ctxt, minion_pool, requery=False)
+        if not refresh_flow:
+            msg = (
+                "There are no minion machine refresh operations to be performed "
+                "at this time")
+            db_api.add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_INFO, msg)
+            return self._get_minion_pool(ctxt, minion_pool.id)
+
+        initial_store = self._get_pool_initial_taskflow_store_base(
+            ctxt, minion_pool, endpoint_dict)
+        self._taskflow_runner.run_flow_in_background(
+            refresh_flow, store=initial_store)
+        self._add_minion_pool_event(
+            ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+            "Begun minion pool refreshing process")
+
+        return self._get_minion_pool(ctxt, minion_pool.id)
+
+    def _get_minion_pool_allocation_flow(self, minion_pool):
+        """ Returns a taskflow.Flow object pertaining to all the tasks
+        required for allocating a minion pool (validation, shared resource
+        setup, and actual minion creation)
+        """
+        # create task flow:
+        allocation_flow = linear_flow.Flow(
+            minion_manager_tasks.MINION_POOL_ALLOCATION_FLOW_NAME_FORMAT % (
+                minion_pool.id))
+
+        # tansition pool to VALIDATING:
+        allocation_flow.add(minion_manager_tasks.UpdateMinionPoolStatusTask(
+            minion_pool.id, constants.MINION_POOL_STATUS_VALIDATING_INPUTS,
+            status_to_revert_to=constants.MINION_POOL_STATUS_ERROR))
+
+        # add pool options validation task:
+        allocation_flow.add(minion_manager_tasks.ValidateMinionPoolOptionsTask(
+            # NOTE: we pass in the ID of the minion pool itself as both
+            # the task ID and the instance ID for tasks which are strictly
+            # pool-related.
+            minion_pool.id,
+            minion_pool.id,
+            minion_pool.platform))
+
+        # transition pool to 'DEPLOYING_SHARED_RESOURCES':
+        allocation_flow.add(minion_manager_tasks.UpdateMinionPoolStatusTask(
+            minion_pool.id,
+            constants.MINION_POOL_STATUS_ALLOCATING_SHARED_RESOURCES))
+
+        # add pool shared resources deployment task:
+        allocation_flow.add(
+            minion_manager_tasks.AllocateSharedPoolResourcesTask(
+                minion_pool.id, minion_pool.id, minion_pool.platform,
+                # NOTE: the shared resource deployment task will always get
+                # run by itself so it is safe to have it override task_info:
+                provides='task_info'))
+
+        # add subflow for deploying all of the minion machines:
+        fmt = (
+            minion_manager_tasks.MINION_POOL_ALLOCATE_MINIONS_SUBFLOW_NAME_FORMAT)
+        machines_flow = unordered_flow.Flow(fmt % minion_pool.id)
+        pool_machine_ids = []
+        for _ in range(minion_pool.minimum_minions):
+            machine_id = str(uuid.uuid4())
+            pool_machine_ids.append(machine_id)
+            machines_flow.add(
+                minion_manager_tasks.AllocateMinionMachineTask(
+                    minion_pool.id, machine_id, minion_pool.platform))
+        # NOTE: bool(flow) == False if the flow has no child flows/tasks:
+        if machines_flow:
+            allocation_flow.add(minion_manager_tasks.UpdateMinionPoolStatusTask(
+                minion_pool.id,
+                constants.MINION_POOL_STATUS_ALLOCATING_MACHINES))
+            LOG.debug(
+                "The following minion machine IDs will be created for "
+                "pool with ID '%s': %s" % (minion_pool.id, pool_machine_ids))
+            allocation_flow.add(machines_flow)
+        else:
+            LOG.debug(
+                "No upfront minion machine deployments required for minion "
+                "pool with ID '%s'", minion_pool.id)
+
+        # transition pool to ALLOCATED:
+        allocation_flow.add(minion_manager_tasks.UpdateMinionPoolStatusTask(
+            minion_pool.id, constants.MINION_POOL_STATUS_ALLOCATED))
+
+        return allocation_flow
+
+    def create_minion_pool(
+            self, ctxt, name, endpoint_id, pool_platform, pool_os_type,
+            environment_options, minimum_minions, maximum_minions,
+            minion_max_idle_time, minion_retention_strategy, notes=None,
+            skip_allocation=False):
+
+        endpoint_dict = self._rpc_conductor_client.get_endpoint(
+            ctxt, endpoint_id)
+        minion_pool = models.MinionPool()
+        minion_pool.id = str(uuid.uuid4())
+        minion_pool.name = name
+        minion_pool.notes = notes
+        minion_pool.platform = pool_platform
+        minion_pool.os_type = pool_os_type
+        minion_pool.endpoint_id = endpoint_id
+        minion_pool.environment_options = environment_options
+        minion_pool.status = constants.MINION_POOL_STATUS_DEALLOCATED
+        minion_pool.minimum_minions = minimum_minions
+        minion_pool.maximum_minions = maximum_minions
+        minion_pool.minion_max_idle_time = minion_max_idle_time
+        minion_pool.minion_retention_strategy = minion_retention_strategy
+
+        cleanup_trust = not bool(ctxt.trust_id)
+        try:
+            keystone.create_trust(ctxt)
+            minion_pool.maintenance_trust_id = ctxt.trust_id
+            db_api.add_minion_pool(ctxt, minion_pool)
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                "Successfully added minion pool to the DB")
+            self._register_refresh_jobs_for_minion_pool(
+                minion_pool)
+        except Exception:
+            if cleanup_trust:
+                keystone.delete_trust(ctxt)
+            raise
+
+        if not skip_allocation:
+            allocation_flow = self._get_minion_pool_allocation_flow(
+                minion_pool)
+            # start the deployment flow:
+            initial_store = self._get_pool_initial_taskflow_store_base(
+                ctxt, minion_pool, endpoint_dict)
+            try:
+                self._taskflow_runner.run_flow_in_background(
+                    allocation_flow, store=initial_store)
+            except Exception as ex:
+                self._add_minion_pool_event(
+                    ctxt, minion_pool.id, constants.TASK_EVENT_ERROR,
+                    "A fatal exception occurred while attempting to start the "
+                    "task flow for allocating the minion pool. Forced "
+                    "deallocation and reallocation may be required. Please "
+                    "review the manager logs for additional details. "
+                    "Error was: %s" % str(ex))
+                raise
+
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                "Begun minion pool allocation process")
+
+        return self.get_minion_pool(ctxt, minion_pool.id)
+
+    def _get_pool_initial_taskflow_store_base(
+            self, ctxt, minion_pool, endpoint_dict):
+        # NOTE: considering pools are associated to strictly one endpoint,
+        # we can duplicate the 'origin/destination':
+        origin_info = {
+            "id": endpoint_dict['id'],
+            "connection_info": endpoint_dict['connection_info'],
+            "mapped_regions": endpoint_dict['mapped_regions'],
+            "type": endpoint_dict['type']}
+        initial_store = {
+            "context": ctxt,
+            "origin": origin_info,
+            "destination": origin_info,
+            "task_info": {
+                "pool_identifier": minion_pool.id,
+                "pool_os_type": minion_pool.os_type,
+                "pool_environment_options": minion_pool.environment_options}}
+        shared_resources = minion_pool.shared_resources
+        if shared_resources is None:
+            shared_resources = {}
+        initial_store['task_info']['pool_shared_resources'] = shared_resources
+        return initial_store
+
+    def _check_pool_machines_in_use(
+            self, ctxt, minion_pool, raise_if_in_use=False, requery=False):
+        """ Checks whether the given pool has any machines currently in-use.
+        Returns a list of the used machines if so, or an empty list of not.
+        """
+        if requery:
+            minion_pool = self._get_minion_pool(
+                ctxt, minion_pool.id, include_machines=True,
+                include_events=False, include_progress_updates=False)
+        unused_machine_states = [
+            constants.MINION_MACHINE_STATUS_AVAILABLE,
+            constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING,
+            constants.MINION_MACHINE_STATUS_POWER_ERROR,
+            constants.MINION_MACHINE_STATUS_ERROR]
+        used_machines = {
+            mch for mch in minion_pool.minion_machines
+            if mch.allocation_status not in unused_machine_states}
+        if used_machines and raise_if_in_use:
+            raise exception.InvalidMinionPoolState(
+                "Minion pool '%s' has one or more machines which are in an"
+                " active state: %s" % (
+                    minion_pool.id, {
+                        mch.id: mch.allocation_status for mch in used_machines}))
+        return used_machines
+
+    @minion_manager_utils.minion_pool_synchronized_op
+    def allocate_minion_pool(self, ctxt, minion_pool_id):
+        LOG.info("Attempting to allocate Minion Pool '%s'.", minion_pool_id)
+        minion_pool = self._get_minion_pool(
+            ctxt, minion_pool_id, include_events=False, include_machines=False,
+            include_progress_updates=False)
+        endpoint_dict = self._rpc_conductor_client.get_endpoint(
+            ctxt, minion_pool.endpoint_id)
+        acceptable_allocation_statuses = [
+            constants.MINION_POOL_STATUS_DEALLOCATED]
+        current_status = minion_pool.status
+        if current_status not in acceptable_allocation_statuses:
+            raise exception.InvalidMinionPoolState(
+                "Minion machines for pool '%s' cannot be allocated as the pool"
+                " is in '%s' state instead of the expected %s. Please "
+                "force-deallocate the pool and try again." % (
+                    minion_pool_id, minion_pool.status,
+                    acceptable_allocation_statuses))
+
+        allocation_flow = self._get_minion_pool_allocation_flow(minion_pool)
+        initial_store = self._get_pool_initial_taskflow_store_base(
+            ctxt, minion_pool, endpoint_dict)
+
+        try:
+            db_api.set_minion_pool_status(
+                ctxt, minion_pool_id,
+                constants.MINION_POOL_STATUS_POOL_MAINTENANCE)
+            self._taskflow_runner.run_flow_in_background(
+                allocation_flow, store=initial_store)
+            self._register_refresh_jobs_for_minion_pool(
+                minion_pool)
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                "Begun minion pool allocation process")
+        except Exception as ex:
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_ERROR,
+                "A fatal exception occurred while attempting to start the "
+                "task flow for allocating the minion pool. Forced "
+                "deallocation and reallocation may be required. Please "
+                "review the manager logs for additional details. "
+                "Error was: %s" % str(ex))
+            db_api.set_minion_pool_status(
+                ctxt, minion_pool_id, current_status)
+            raise
+
+        return self._get_minion_pool(ctxt, minion_pool.id)
+
+    def _get_minion_pool_deallocation_flow(
+            self, minion_pool, raise_on_error=True):
+        """ Returns a taskflow.Flow object pertaining to all the tasks
+        required for deallocating a minion pool (machines and shared resources)
+        """
+        # create task flow:
+        deallocation_flow = linear_flow.Flow(
+            minion_manager_tasks.MINION_POOL_DEALLOCATION_FLOW_NAME_FORMAT % (
+                minion_pool.id))
+
+        # add subflow for deallocating all of the minion machines:
+        fmt = (
+            minion_manager_tasks.MINION_POOL_DEALLOCATE_MACHINES_SUBFLOW_NAME_FORMAT)
+        machines_flow = unordered_flow.Flow(fmt % minion_pool.id)
+        for machine in minion_pool.minion_machines:
+            machines_flow.add(
+                minion_manager_tasks.DeallocateMinionMachineTask(
+                    minion_pool.id, machine.id, minion_pool.platform,
+                    raise_on_cleanup_failure=raise_on_error))
+        # NOTE: bool(flow) == False if the flow has no child flows/tasks:
+        if machines_flow:
+            # tansition pool to DEALLOCATING_MACHINES:
+            deallocation_flow.add(minion_manager_tasks.UpdateMinionPoolStatusTask(
+                minion_pool.id,
+                constants.MINION_POOL_STATUS_DEALLOCATING_MACHINES,
+                status_to_revert_to=constants.MINION_POOL_STATUS_ERROR))
+            deallocation_flow.add(machines_flow)
+        else:
+            LOG.debug(
+                "No machines for pool '%s' require deallocating.", minion_pool.id)
+
+        # transition pool to DEALLOCATING_SHARED_RESOURCES:
+        deallocation_flow.add(minion_manager_tasks.UpdateMinionPoolStatusTask(
+            minion_pool.id,
+            constants.MINION_POOL_STATUS_DEALLOCATING_SHARED_RESOURCES,
+            status_to_revert_to=constants.MINION_POOL_STATUS_ERROR))
+
+        # add pool shared resources deletion task:
+        deallocation_flow.add(
+            minion_manager_tasks.DeallocateSharedPoolResourcesTask(
+                minion_pool.id, minion_pool.id, minion_pool.platform))
+
+        # transition pool to DEALLOCATED:
+        deallocation_flow.add(minion_manager_tasks.UpdateMinionPoolStatusTask(
+            minion_pool.id, constants.MINION_POOL_STATUS_DEALLOCATED))
+
+        return deallocation_flow
+
+    def _get_pool_deallocation_initial_store(
+            self, ctxt, minion_pool, endpoint_dict):
+        base = self._get_pool_initial_taskflow_store_base(
+            ctxt, minion_pool, endpoint_dict)
+        if 'task_info' not in base:
+            base['task_info'] = {}
+        base['task_info']['pool_shared_resources'] = (
+            minion_pool.shared_resources)
+        return base
+
+    @minion_manager_utils.minion_pool_synchronized_op
+    def deallocate_minion_pool(self, ctxt, minion_pool_id, force=False):
+        LOG.info("Attempting to deallocate Minion Pool '%s'.", minion_pool_id)
+        minion_pool = self._get_minion_pool(
+            ctxt, minion_pool_id, include_events=False, include_machines=True,
+            include_progress_updates=False)
+        current_status = minion_pool.status
+        if current_status == constants.MINION_POOL_STATUS_DEALLOCATED:
+            LOG.debug(
+                "Deallocation requested on already deallocated pool '%s'. "
+                "Nothing to do so returning early.", minion_pool_id)
+            return self._get_minion_pool(ctxt, minion_pool.id)
+        acceptable_deallocation_statuses = [
+            constants.MINION_POOL_STATUS_ALLOCATED,
+            constants.MINION_POOL_STATUS_ERROR]
+        if current_status not in acceptable_deallocation_statuses:
+            if not force:
+                raise exception.InvalidMinionPoolState(
+                    "Minion pool '%s' cannot be deallocated as the pool"
+                    " is in '%s' state instead of one of the expected %s"% (
+                        minion_pool_id, minion_pool.status,
+                        acceptable_deallocation_statuses))
+            else:
+                LOG.warn(
+                    "Forcibly deallocating minion pool '%s' at user request.",
+                    minion_pool_id)
+        self._check_pool_machines_in_use(
+            ctxt, minion_pool, raise_if_in_use=not force)
+        endpoint_dict = self._rpc_conductor_client.get_endpoint(
+            ctxt, minion_pool.endpoint_id)
+
+        deallocation_flow = self._get_minion_pool_deallocation_flow(
+            minion_pool, raise_on_error=not force)
+        initial_store = self._get_pool_deallocation_initial_store(
+            ctxt, minion_pool, endpoint_dict)
+
+        try:
+            db_api.set_minion_pool_status(
+                ctxt, minion_pool_id,
+                constants.MINION_POOL_STATUS_POOL_MAINTENANCE)
+            self._taskflow_runner.run_flow_in_background(
+                deallocation_flow, store=initial_store)
+            self._unregister_refresh_jobs_for_minion_pool(
+                minion_pool, raise_on_error=False)
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+                "Begun minion pool deallocation process")
+        except Exception as ex:
+            self._add_minion_pool_event(
+                ctxt, minion_pool.id, constants.TASK_EVENT_ERROR,
+                "A fatal exception occurred while attempting to start the "
+                "task flow for deallocating the minion pool. Forced "
+                "deallocation and reallocation may be required. Please "
+                "review the manager logs for additional details. "
+                "Error was: %s" % str(ex))
+            db_api.set_minion_pool_status(
+                ctxt, minion_pool_id, current_status)
+            raise
+
+        return self._get_minion_pool(ctxt, minion_pool.id)
+
+    def get_minion_pools(self, ctxt, include_machines=True):
+        return db_api.get_minion_pools(
+            ctxt, include_machines=include_machines, include_events=False,
+            include_progress_updates=False)
+
+    def _get_minion_pool(
+            self, ctxt, minion_pool_id, include_machines=True,
+            include_events=True, include_progress_updates=True):
+        minion_pool = db_api.get_minion_pool(
+            ctxt, minion_pool_id, include_machines=include_machines,
+            include_events=include_events,
+            include_progress_updates=include_progress_updates)
+        if not minion_pool:
+            raise exception.NotFound(
+                "Minion pool with ID '%s' not found." % minion_pool_id)
+        return minion_pool
+
+    @minion_manager_utils.minion_pool_synchronized_op
+    def get_minion_pool(self, ctxt, minion_pool_id):
+        return self._get_minion_pool(
+            ctxt, minion_pool_id, include_machines=True, include_events=True,
+            include_progress_updates=True)
+
+    @minion_manager_utils.minion_pool_synchronized_op
+    def update_minion_pool(self, ctxt, minion_pool_id, updated_values):
+        minion_pool = self._get_minion_pool(
+            ctxt, minion_pool_id, include_machines=False)
+        if minion_pool.status != constants.MINION_POOL_STATUS_DEALLOCATED:
+            raise exception.InvalidMinionPoolState(
+                "Minion Pool '%s' cannot be updated as it is in '%s' status "
+                "instead of the expected '%s'. Please ensure the pool machines"
+                "have been deallocated and the pool's supporting resources "
+                "have been torn down before updating the pool." % (
+                    minion_pool_id, minion_pool.status,
+                    constants.MINION_POOL_STATUS_DEALLOCATED))
+        LOG.info(
+            "Attempting to update minion_pool '%s' with payload: %s",
+            minion_pool_id, updated_values)
+        db_api.update_minion_pool(ctxt, minion_pool_id, updated_values)
+        LOG.info("Minion Pool '%s' successfully updated", minion_pool_id)
+        self._add_minion_pool_event(
+            ctxt, minion_pool.id, constants.TASK_EVENT_INFO,
+            "Successfully updated minion pool properties")
+        return db_api.get_minion_pool(ctxt, minion_pool_id)
+
+    @minion_manager_utils.minion_pool_synchronized_op
+    def delete_minion_pool(self, ctxt, minion_pool_id):
+        minion_pool = self._get_minion_pool(
+            ctxt, minion_pool_id, include_machines=True)
+        acceptable_deletion_statuses = [
+            constants.MINION_POOL_STATUS_DEALLOCATED,
+            constants.MINION_POOL_STATUS_ERROR]
+        if minion_pool.status not in acceptable_deletion_statuses:
+            raise exception.InvalidMinionPoolState(
+                "Minion Pool '%s' cannot be deleted as it is in '%s' status "
+                "instead of one of the expected '%s'. Please ensure the pool "
+                "machines have been deallocated and the pool's supporting "
+                "resources have been torn down before deleting the pool." % (
+                    minion_pool_id, minion_pool.status,
+                    acceptable_deletion_statuses))
+        self._unregister_refresh_jobs_for_minion_pool(
+            minion_pool, raise_on_error=False)
+
+        LOG.info("Deleting minion pool with ID '%s'" % minion_pool_id)
+        db_api.delete_minion_pool(ctxt, minion_pool_id)
+        if minion_pool.maintenance_trust_id:
+            maintenance_ctxt = context.get_admin_context(
+                minion_pool.maintenance_trust_id)
+            keystone.delete_trust(maintenance_ctxt)

coriolis/minion_manager/rpc/tasks.py

@@ -0,0 +1,1328 @@
+# Copyright 2020 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import abc
+import copy
+
+from oslo_log import log as logging
+from oslo_utils import timeutils
+
+from coriolis import constants
+from coriolis import exception
+from coriolis import utils
+from coriolis.conductor.rpc import client as rpc_conductor_client
+from coriolis.db import api as db_api
+from coriolis.db.sqlalchemy import models
+from coriolis.minion_manager.rpc import client as rpc_minion_manager_client
+from coriolis.minion_manager.rpc import utils as minion_manager_utils
+from coriolis.taskflow import base as coriolis_taskflow_base
+from taskflow.types import failure
+
+
+LOG = logging.getLogger(__name__)
+
+MINION_POOL_MIGRATION_ALLOCATION_FLOW_NAME_FORMAT = (
+    "migration-%s-minions-allocation")
+MINION_POOL_REPLICA_ALLOCATION_FLOW_NAME_FORMAT = (
+    "replica-%s-minions-allocation")
+MINION_POOL_MIGRATION_ALLOCATION_SUBFLOW_NAME_FORMAT = (
+    "migration-%s-minions-machines-allocation")
+MINION_POOL_REPLICA_ALLOCATION_SUBFLOW_NAME_FORMAT = (
+    "replica-%s-minions-machines-allocation")
+MINION_POOL_ALLOCATION_FLOW_NAME_FORMAT = "pool-%s-allocation"
+MINION_POOL_DEALLOCATION_FLOW_NAME_FORMAT = "pool-%s-deallocation"
+MINION_POOL_REFRESH_FLOW_NAME_FORMAT = "pool-%s-refresh"
+MINION_POOL_VALIDATION_TASK_NAME_FORMAT = "pool-%s-validation"
+MINION_POOL_UPDATE_STATUS_TASK_NAME_FORMAT = "pool-%s-update-status-%s"
+MINION_POOL_HEALTHCHECK_MACHINE_TASK_NAME_FORMAT = (
+    "pool-%s-machine-%s-healthcheck")
+MINION_POOL_ALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT = (
+    "pool-%s-allocate-shared-resources")
+MINION_POOL_DEALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT = (
+    "pool-%s-deallocate-shared-resources")
+MINION_POOL_ALLOCATE_MINIONS_SUBFLOW_NAME_FORMAT = (
+    "pool-%s-machines-allocation")
+MINION_POOL_DEALLOCATE_MACHINES_SUBFLOW_NAME_FORMAT = (
+    "pool-%s-machines-deallocation")
+MINION_POOL_HEALTHCHECK_MACHINE_SUBFLOW_NAME_FORMAT = (
+    "pool-%s-machine-%s-healthcheck")
+MINION_POOL_REALLOCATE_MACHINE_SUBFLOW_NAME_FORMAT = (
+    "pool-%s-machine-%s-reallocation")
+MINION_POOL_ALLOCATE_MACHINES_FOR_REPLICA_SUBFLOW_NAME_FORMAT = (
+    "pool-%s-allocate-replica-%s-machines")
+MINION_POOL_ALLOCATE_MACHINES_FOR_MIGRATION_SUBFLOW_NAME_FORMAT = (
+    "pool-%s-allocate-migration-%s-machines")
+MINION_POOL_ALLOCATE_MACHINE_TASK_NAME_FORMAT = (
+    "pool-%s-machine-%s-allocation")
+MINION_POOL_DEALLOCATE_MACHINE_TASK_NAME_FORMAT = (
+    "pool-%s-machine-%s-deallocation")
+MINION_POOL_CONFIRM_MIGRATION_MINION_ALLOCATION_TASK_NAME_FORMAT = (
+    "migration-%s-minion-allocation-confirmation")
+MINION_POOL_CONFIRM_REPLICA_MINION_ALLOCATION_TASK_NAME_FORMAT = (
+    "replica-%s-minion-allocation-confirmation")
+MINION_POOL_REPORT_MIGRATION_ALLOCATION_FAILURE_TASK_NAME_FORMAT = (
+    "migration-%s-minion-allocation-failure")
+MINION_POOL_REPORT_REPLICA_ALLOCATION_FAILURE_TASK_NAME_FORMAT = (
+    "replica-%s-minion-allocation-failure")
+MINION_POOL_POWER_ON_MACHINE_TASK_NAME_FORMAT = (
+    "pool-%s-machine-%s-power-on")
+MINION_POOL_POWER_OFF_MACHINE_TASK_NAME_FORMAT = (
+    "pool-%s-machine-%s-power-off")
+
+
+class MinionManagerTaskEventMixin(object):
+
+    # NOTE(aznashwan): it is unsafe to fork processes with pre-instantiated
+    # oslo_messaging clients as the underlying eventlet thread queues will
+    # be invalidated. Considering this class both serves from a "main
+    # process" as well as forking child processes, it is safest to
+    # re-instantiate the clients every time:
+    @property
+    def _conductor_client(self):
+        if not getattr(self, '_conductor_client_instance', None):
+            self._conductor_client_instance = (
+                rpc_conductor_client.ConductorClient())
+        return self._conductor_client_instance
+
+    @property
+    def _minion_manager_client(self):
+        if not getattr(self, '_minion_manager_client_instance', None):
+            self._minion_manager_client_instance = (
+                rpc_minion_manager_client.MinionManagerClient())
+        return self._minion_manager_client_instance
+
+    def _add_minion_pool_event(
+            self, context, message, level=constants.TASK_EVENT_INFO):
+        LOG.debug("Minion pool '%s' event: %s", self._minion_pool_id, message)
+        db_api.add_minion_pool_event(
+            context, self._minion_pool_id, level, message)
+
+    def _get_minion_machine(
+            self, context, minion_machine_id,
+            raise_if_not_found=False):
+        machine = db_api.get_minion_machine(context, minion_machine_id)
+        if not machine and raise_if_not_found:
+            raise exception.NotFound(
+                "Could not find minion machine with ID '%s'" % (
+                    minion_machine_id))
+        return machine
+
+    def _set_minion_pool_status(self, ctxt, minion_pool_id, new_status):
+        with minion_manager_utils.get_minion_pool_lock(
+                minion_pool_id, external=True):
+            db_api.set_minion_pool_status(ctxt, minion_pool_id, new_status)
+
+    def _update_minion_machine(
+            self, ctxt, minion_pool_id, minion_machine_id, updated_values):
+        with minion_manager_utils.get_minion_pool_lock(
+                minion_pool_id, external=True):
+            db_api.update_minion_machine(
+                ctxt, minion_machine_id, updated_values)
+
+    def _set_minion_machine_allocation_status(
+            self, ctxt, minion_pool_id, minion_machine_id, new_status):
+        with minion_manager_utils.get_minion_pool_lock(
+                minion_pool_id, external=True):
+            db_api.set_minion_machine_allocation_status(
+                ctxt, minion_machine_id, new_status)
+
+    def _set_minion_machine_power_status(
+            self, ctxt, minion_pool_id, minion_machine_id, new_status):
+        self._update_minion_machine(
+            ctxt, minion_pool_id, minion_machine_id,
+            {"power_status": new_status})
+
+
+class _BaseReportMinionAllocationFailureForActionTask(
+        coriolis_taskflow_base.BaseCoriolisTaskflowTask,
+        MinionManagerTaskEventMixin):
+    """ Task with no operation on `execute()`, but whose `revert()` method
+    reports a minion allocation failure to the conductor for the afferent
+    transfer action. """
+
+    def __init__(self, action_id, **kwargs):
+        self._action_id = action_id
+        self._task_name = self._get_task_name(action_id)
+        super(_BaseReportMinionAllocationFailureForActionTask, self).__init__(
+            name=self._task_name, **kwargs)
+
+    @abc.abstractmethod
+    def _get_task_name(self, action_id):
+        raise NotImplementedError(
+            "No allocation failure task name provided")
+
+    @abc.abstractmethod
+    def _report_machine_allocation_failure(
+            self, context, action_id, failure_str):
+        raise NotImplementedError(
+            "No allocation failure operation defined")
+
+    def execute(self, context):
+        super(
+            _BaseReportMinionAllocationFailureForActionTask, self).execute()
+        LOG.debug(
+            "Nothing to execute for task '%s'", self._task_name)
+
+    def revert(self, context, *args, **kwargs):
+        super(
+            _BaseReportMinionAllocationFailureForActionTask, self).revert(
+                *args, **kwargs)
+        flow_failures = kwargs.get('flow_failures', {})
+        flow_failures_str = self._get_error_str_for_flow_failures(
+            flow_failures, full_tracebacks=False)
+        LOG.info(
+            "Reporting minion allocation failure for action '%s': %s",
+            self._action_id, flow_failures_str)
+        self._minion_manager_client.deallocate_minion_machines_for_action(
+            context, self._action_id)
+        self._report_machine_allocation_failure(
+            context, self._action_id, flow_failures_str)
+
+
+class ReportMinionAllocationFailureForMigrationTask(
+        _BaseReportMinionAllocationFailureForActionTask):
+
+    def _get_task_name(self, action_id):
+        return MINION_POOL_REPORT_MIGRATION_ALLOCATION_FAILURE_TASK_NAME_FORMAT % (
+            action_id)
+
+    def _report_machine_allocation_failure(
+            self, context, action_id, failure_str):
+        self._conductor_client.report_migration_minions_allocation_error(
+            context, action_id, failure_str)
+
+
+class ReportMinionAllocationFailureForReplicaTask(
+        _BaseReportMinionAllocationFailureForActionTask):
+
+    def _get_task_name(self, action_id):
+        return MINION_POOL_REPORT_REPLICA_ALLOCATION_FAILURE_TASK_NAME_FORMAT % (
+            action_id)
+
+    def _report_machine_allocation_failure(
+            self, context, action_id, failure_str):
+        self._conductor_client.report_replica_minions_allocation_error(
+            context, action_id, failure_str)
+
+
+class _BaseConfirmMinionAllocationForActionTask(
+        coriolis_taskflow_base.BaseCoriolisTaskflowTask,
+        MinionManagerTaskEventMixin):
+    """ Task which confirms the minion machine allocations for the given action
+    to the conductor.
+    """
+
+    def __init__(self, action_id, allocated_machine_id_mappings, **kwargs):
+        """
+        param allocated_machine_id_mappings: dict of the form:
+        {
+            "<action_instance_identifier>": {
+                "origin_minion_id": "<origin_minion_id>",
+                "destination_minion_id": "<destination_minion_id>",
+                "osmorphing_minion_id": "<osmorphing_minion_id>"}}
+        """
+        self._action_id = action_id
+        self._task_name = self._get_task_name(action_id)
+        self._allocated_machine_id_mappings = allocated_machine_id_mappings
+        super(_BaseConfirmMinionAllocationForActionTask, self).__init__(
+            name=self._task_name, **kwargs)
+
+    @abc.abstractmethod
+    def _get_task_name(self, action_id):
+        raise NotImplementedError(
+            "No minion allocation confirmation task name defined")
+
+    @abc.abstractmethod
+    def _confirm_machine_allocation_for_action(
+            self, context, action_id, machine_allocations):
+        raise NotImplementedError(
+            "No minion allocation confrimation operation defined")
+
+    def execute(self, context):
+        machines_cache = {}
+        machine_allocations = {}
+
+        def _check_minion_properties(
+                minion_machine, instance, minion_purpose="unknown"):
+            if minion_machine.allocation_status != (
+                    constants.MINION_MACHINE_STATUS_IN_USE):
+                raise exception.InvalidMinionMachineState(
+                    "Minion machine with ID '%s' of pool '%s' is in '%s' "
+                    "status instead of the expected '%s' for it to be used "
+                    "as a '%s' minion for instance '%s' of transfer "
+                    "action '%s'." % (
+                        minion_machine.id, minion_machine.pool_id,
+                        minion_machine.allocation_status,
+                        constants.MINION_MACHINE_STATUS_IN_USE,
+                        minion_purpose, instance, self._action_id))
+
+            if minion_machine.allocated_action != self._action_id:
+                raise exception.InvalidMinionMachineState(
+                    "Minion machine with ID '%s' of pool '%s' appears to be "
+                    "allocated to action with ID '%s' instead of the expected"
+                    " '%s' for it to be used as a '%s' minion for instance '%s'." % (
+                        minion_machine.id, minion_machine.pool_id,
+                        minion_machine.allocated_action, self._action_id,
+                        minion_purpose, instance))
+
+            if minion_machine.power_status != (
+                    constants.MINION_MACHINE_POWER_STATUS_POWERED_ON):
+                raise exception.InvalidMinionMachineState(
+                    "Minion machine with ID '%s' of pool '%s' is in '%s' "
+                    "power status instead of the expected '%s' for it to be "
+                    "used as a '%s' minion for instance '%s' of transfer "
+                    "action '%s'." % (
+                        minion_machine.id, minion_machine.pool_id,
+                        minion_machine.power_status,
+                        constants.MINION_MACHINE_POWER_STATUS_POWERED_ON,
+                        minion_purpose, instance, self._action_id))
+
+            # TODO(aznashwan): add extra checks for conn info schemas here?
+            required_props = {
+                "provider_properties": minion_machine.provider_properties,
+                "connection_info": minion_machine.connection_info}
+            if not all(required_props.values()):
+                raise exception.InvalidMinionMachineState(
+                    "One or more required paroperties for minion machine '%s' "
+                    "(to be used as a '%s' minion for instance '%s' of action "
+                    "'%s') were missing: %s" % (
+                        minion_machine.id, minion_purpose, instance,
+                        self._action_id, required_props))
+
+        for (instance, allocated_machines_for_instance) in (
+                self._allocated_machine_id_mappings.items()):
+
+            if not allocated_machines_for_instance:
+                LOG.warn(
+                    "No machine allocations were provided for instance '%s' "
+                    "for action '%s'. Skipping. mappings were: %s",
+                    instance, self._action_id, allocated_machines_for_instance)
+                continue
+
+            if instance not in machine_allocations:
+                machine_allocations[instance] = {}
+
+            # check for and fetch the source minion:
+            origin_minion_id = allocated_machines_for_instance.get(
+                'origin_minion_id')
+            if origin_minion_id:
+                origin_minion_machine = machines_cache.get(origin_minion_id)
+                if not origin_minion_machine:
+                    origin_minion_machine = self._get_minion_machine(
+                        context, origin_minion_id, raise_if_not_found=True)
+                    machines_cache[origin_minion_id] = origin_minion_machine
+                    _check_minion_properties(
+                        origin_minion_machine, instance, minion_purpose="source")
+                machine_allocations[instance]['origin_minion'] = (
+                    origin_minion_machine.to_dict())
+
+            # check for and fetch the destination minion:
+            destination_minion_id = allocated_machines_for_instance.get(
+                'destination_minion_id')
+            if destination_minion_id:
+                destination_minion_machine = machines_cache.get(
+                    destination_minion_id)
+                if not destination_minion_machine:
+                    destination_minion_machine = self._get_minion_machine(
+                        context, destination_minion_id,
+                        raise_if_not_found=True)
+                    _check_minion_properties(
+                        destination_minion_machine, instance,
+                        minion_purpose="destination")
+                    machines_cache[destination_minion_id] = (
+                        destination_minion_machine)
+                machine_allocations[instance]['destination_minion'] = (
+                    destination_minion_machine.to_dict())
+
+            # check for and fetch the OSMorphing minion:
+            osmorphing_minion_id = allocated_machines_for_instance.get(
+                'osmorphing_minion_id')
+            if osmorphing_minion_id:
+                osmorphing_minion_machine = machines_cache.get(
+                    osmorphing_minion_id)
+                if not osmorphing_minion_machine:
+                    osmorphing_minion_machine = self._get_minion_machine(
+                        context, osmorphing_minion_id, raise_if_not_found=True)
+                    _check_minion_properties(
+                        osmorphing_minion_machine, instance,
+                        minion_purpose="OSMorphing")
+                    machines_cache[osmorphing_minion_id] = (
+                        osmorphing_minion_machine)
+                machine_allocations[instance]['osmorphing_minion'] = (
+                    osmorphing_minion_machine.to_dict())
+
+        self._confirm_machine_allocation_for_action(
+            context, self._action_id, machine_allocations)
+
+
+class ConfirmMinionAllocationForMigrationTask(
+        _BaseConfirmMinionAllocationForActionTask):
+
+    def _get_task_name(self, action_id):
+        return MINION_POOL_CONFIRM_MIGRATION_MINION_ALLOCATION_TASK_NAME_FORMAT % (
+            action_id)
+
+    def _confirm_machine_allocation_for_action(
+            self, context, action_id, machine_allocations):
+        self._conductor_client.confirm_migration_minions_allocation(
+            context, action_id, machine_allocations)
+
+
+class ConfirmMinionAllocationForReplicaTask(
+        _BaseConfirmMinionAllocationForActionTask):
+
+    def _get_task_name(self, action_id):
+        return MINION_POOL_CONFIRM_REPLICA_MINION_ALLOCATION_TASK_NAME_FORMAT % (
+            action_id)
+
+    def _confirm_machine_allocation_for_action(
+            self, context, action_id, machine_allocations):
+        self._conductor_client.confirm_replica_minions_allocation(
+            context, action_id, machine_allocations)
+
+
+class UpdateMinionPoolStatusTask(
+        coriolis_taskflow_base.BaseCoriolisTaskflowTask,
+        MinionManagerTaskEventMixin):
+    """ Task which updates the status of the given pool.
+    Is capable of recording and reverting the state.
+    """
+
+    default_provides = ["latest_status"]
+
+    def __init__(
+            self, minion_pool_id, target_status,
+            status_to_revert_to=None, **kwargs):
+
+        self._target_status = target_status
+        self._minion_pool_id = minion_pool_id
+        self._task_name = (MINION_POOL_UPDATE_STATUS_TASK_NAME_FORMAT % (
+            self._minion_pool_id, self._target_status)).lower()
+        self._previous_status = None
+        self._status_to_revert_to = status_to_revert_to
+
+        super(UpdateMinionPoolStatusTask, self).__init__(
+            name=self._task_name, **kwargs)
+
+    def execute(self, context, *args):
+        super(UpdateMinionPoolStatusTask, self).execute(*args)
+
+        if not self._previous_status:
+            minion_pool = db_api.get_minion_pool(
+                context, self._minion_pool_id, include_machines=False,
+                include_events=False, include_progress_updates=False)
+            self._previous_status = minion_pool.status
+
+        if self._previous_status == self._target_status:
+            LOG.debug(
+                "[Task '%s'] Minion pool '%s' already in status '%s'. "
+                "Nothing to do." % (
+                    self._task_name, self._minion_pool_id,
+                    self._target_status))
+        else:
+            LOG.debug(
+                "[Task '%s'] Transitioning minion pool '%s' from status '%s' "
+                "to '%s'." % (
+                    self._task_name, self._minion_pool_id,
+                    self._previous_status, self._target_status))
+            self._set_minion_pool_status(
+                context, self._minion_pool_id, self._target_status)
+            self._add_minion_pool_event(
+                context,
+                "Pool status transitioned from '%s' to '%s'" % (
+                    self._previous_status, self._target_status))
+
+        return self._target_status
+
+    def revert(self, context, *args, **kwargs):
+        super(UpdateMinionPoolStatusTask, self).revert(*args, **kwargs)
+
+        minion_pool = db_api.get_minion_pool(
+            context, self._minion_pool_id, include_machines=False,
+            include_events=False, include_progress_updates=False)
+        if not minion_pool:
+            LOG.debug(
+                "[Task '%s'] Could not find pool with ID '%s' for status "
+                "reversion." % (self._task_name, self._minion_pool_id))
+            return
+
+        previous_status = self._previous_status
+        if self._status_to_revert_to:
+            LOG.debug(
+                "Forcibly reverting pool to status '%s' despite previous "
+                "status being '%s'",
+                self._status_to_revert_to, self._previous_status)
+            previous_status = self._status_to_revert_to
+        if minion_pool.status == previous_status:
+            LOG.debug(
+                "[Task '%s'] Minion pool '%s' is/was already reverted to "
+                "'%s'." % (
+                    self._task_name, self._minion_pool_id,
+                    previous_status))
+        else:
+            if minion_pool.status != self._target_status:
+                LOG.warn(
+                    "[Task %s] Minion pool '%s' is in status '%s', which is "
+                    "neither the previous status ('%s'), nor the newly-set "
+                    "status ('%s'). Reverting to '%s' anyway.",
+                    self._task_name, self._minion_pool_id, minion_pool.status,
+                    previous_status, self._target_status, previous_status)
+            LOG.debug(
+                "[Task '%s'] Reverting pool '%s' status from '%s' to "
+                "'%s'" % (
+                    self._task_name, self._minion_pool_id, minion_pool.status,
+                    previous_status))
+            self._set_minion_pool_status(
+                context, self._minion_pool_id, previous_status)
+            self._add_minion_pool_event(
+                context,
+                "Pool status reverted from '%s' to '%s'" % (
+                    minion_pool.status, previous_status))
+
+
+class BaseMinionManangerTask(
+        coriolis_taskflow_base.BaseRunWorkerTask,
+        MinionManagerTaskEventMixin):
+
+    """Base taskflow.Task implementation for Minion Mananger tasks.
+
+    Acts as a simple adapter between minion-pool-specific params and the
+    BaseRunWorkerTask.
+    """
+
+    default_provides = 'task_info'
+
+    def __init__(
+            self, minion_pool_id, minion_machine_id,
+            main_task_runner_type, **kwargs):
+        self._minion_pool_id = minion_pool_id
+        self._minion_machine_id = minion_machine_id
+
+        super(BaseMinionManangerTask, self).__init__(
+            self._get_task_name(minion_pool_id, minion_machine_id),
+            # TODO(aznashwan): passing the minion pool ID as the task ID is
+            # required to allow for the Minion pool event manager in the worker
+            # service to know what pool to emit events for.
+            minion_pool_id, minion_machine_id, main_task_runner_type, **kwargs)
+
+    @abc.abstractmethod
+    def _get_task_name(self, minion_pool_id, minion_machine_id):
+        raise NotImplementedError("No task name providable")
+
+    def execute(self, context, origin, destination, task_info):
+        LOG.info(
+            "Starting minion pool task '%s' (runner type '%s')",
+            self._task_name, self._main_task_runner_type)
+        res = super(BaseMinionManangerTask, self).execute(
+            context, origin, destination, task_info)
+        LOG.info(
+            "Completed minion pool task '%s' (runner type '%s')",
+            self._task_name, self._main_task_runner_type)
+        return res
+
+    def revert(self, context, origin, destination, task_info, **kwargs):
+        flow_failures = kwargs.get('flow_failures', {})
+        self._add_minion_pool_event(
+            context,
+            "Failure occurred for one or more operations on minion pool '%s'. "
+            "Please check the logs for additional details. Error messages "
+            "were:\n%s" % (
+                self._minion_pool_id,
+                self._get_error_str_for_flow_failures(
+                    flow_failures, full_tracebacks=False)),
+            level=constants.TASK_EVENT_ERROR)
+        super(BaseMinionManangerTask, self).revert(
+            context, origin, destination, task_info, **kwargs)
+
+
+class ValidateMinionPoolOptionsTask(BaseMinionManangerTask):
+
+    def __init__(
+            self, minion_pool_id, minion_machine_id, minion_pool_type,
+            **kwargs):
+        task_type = constants.TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_OPTIONS
+        if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE:
+            task_type = (
+                constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_OPTIONS)
+        super(ValidateMinionPoolOptionsTask, self).__init__(
+            minion_pool_id, minion_machine_id, task_type, **kwargs)
+
+    def _get_task_name(self, minion_pool_id, minion_machine_id):
+        return MINION_POOL_VALIDATION_TASK_NAME_FORMAT % minion_pool_id
+
+    def execute(self, context, origin, destination, task_info):
+        self._add_minion_pool_event(
+            context, "Validating minion pool options")
+        res = super(ValidateMinionPoolOptionsTask, self).execute(
+            context, origin, destination, task_info)
+        self._add_minion_pool_event(
+            context, "Successfully validated minion pool options")
+
+    def revert(self, context, origin, destination, task_info, **kwargs):
+        LOG.debug("[%s] Nothing to revert for validation", self._task_name)
+        super(ValidateMinionPoolOptionsTask, self).revert(
+            context, origin, destination, task_info, **kwargs)
+
+
+class AllocateSharedPoolResourcesTask(BaseMinionManangerTask):
+
+    def __init__(
+            self, minion_pool_id, minion_machine_id, minion_pool_type,
+            **kwargs):
+
+        resource_deployment_task_type = (
+            constants.TASK_TYPE_SET_UP_SOURCE_POOL_SHARED_RESOURCES)
+        resource_cleanup_task_type = (
+            constants.TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES)
+        if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE:
+            resource_deployment_task_type = (
+                constants.TASK_TYPE_SET_UP_DESTINATION_POOL_SHARED_RESOURCES)
+            resource_cleanup_task_type = (
+                constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES)
+        super(AllocateSharedPoolResourcesTask, self).__init__(
+            minion_pool_id, minion_machine_id, resource_deployment_task_type,
+            cleanup_task_runner_type=resource_cleanup_task_type, **kwargs)
+
+    def _get_task_name(self, minion_pool_id, minion_machine_id):
+        return MINION_POOL_ALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT % (
+            minion_pool_id)
+
+    def execute(self, context, origin, destination, task_info):
+        with minion_manager_utils.get_minion_pool_lock(
+                self._minion_pool_id, external=True):
+            minion_pool = db_api.get_minion_pool(
+                context, self._minion_pool_id)
+            if not minion_pool:
+                raise exception.InvalidMinionPoolSelection(
+                    "[Task '%s'] Minion pool '%s' doesn't exist in the DB. "
+                    "It cannot have shared resources deployed for it." % (
+                        self._task_name))
+            if minion_pool.shared_resources:
+                raise exception.InvalidMinionPoolState(
+                    "[Task '%s'] Minion pool already has shared resources "
+                    "defined for it. Cannot re-deploy shared resources. "
+                    "DB entry is: %s" % (
+                        self._task_name, minion_pool.shared_resources))
+
+        self._add_minion_pool_event(
+            context, "Deploying shared pool resources")
+        res = super(AllocateSharedPoolResourcesTask, self).execute(
+            context, origin, destination, task_info)
+        pool_shared_resources = res['pool_shared_resources']
+
+        updated_values = {
+            "shared_resources": pool_shared_resources}
+
+        self._add_minion_pool_event(
+            context, "Successfully deployed shared pool resources")
+        with minion_manager_utils.get_minion_pool_lock(
+                self._minion_pool_id, external=True):
+            db_api.update_minion_pool(
+                context, self._minion_pool_id, updated_values)
+
+        task_info['pool_shared_resources'] = res['pool_shared_resources']
+        return task_info
+
+    def revert(self, context, origin, destination, task_info, **kwargs):
+        if 'pool_shared_resources' not in task_info:
+            LOG.debug(
+                "[Task '%s'] Failed to find 'pool_shared_resources' in "
+                "provided task_info from original execution of allocation "
+                "task for pool '%s'. Defaulting to None.",
+                self._task_name, self._minion_pool_id)
+            task_info['pool_shared_resources'] = {}
+
+        super(AllocateSharedPoolResourcesTask, self).revert(
+            context, origin, destination, task_info, **kwargs)
+
+        with minion_manager_utils.get_minion_pool_lock(
+                self._minion_pool_id, external=True):
+            updated_values = {
+                "pool_shared_resources": None}
+            db_api.update_minion_pool(
+                context, self._minion_pool_id, updated_values)
+
+
+class DeallocateSharedPoolResourcesTask(BaseMinionManangerTask):
+
+    def __init__(
+            self, minion_pool_id, minion_machine_id, minion_pool_type,
+            **kwargs):
+
+        resource_deallocation_task = (
+            constants.TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES)
+        if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE:
+            resource_deallocation_task = (
+                constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES)
+        super(DeallocateSharedPoolResourcesTask, self).__init__(
+            minion_pool_id, minion_machine_id, resource_deallocation_task,
+            **kwargs)
+
+    def _get_task_name(self, minion_pool_id, minion_machine_id):
+        return MINION_POOL_DEALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT % (
+            minion_pool_id)
+
+    def execute(self, context, origin, destination, task_info):
+        self._add_minion_pool_event(
+            context, "Deallocating shared pool resources")
+        if 'pool_shared_resources' not in task_info:
+            raise exception.InvalidInput(
+                "[Task '%s'] No 'pool_shared_resources' provided in the "
+                "task_info." % self._task_name)
+        execution_info = {
+            "pool_environment_options": task_info.get(
+                'pool_environment_options', {}),
+            "pool_shared_resources": task_info['pool_shared_resources']}
+        res = super(DeallocateSharedPoolResourcesTask, self).execute(
+            context, origin, destination, execution_info)
+        if res:
+            LOG.warn(
+                "[Task '%s'] Pool '%s' shared resource deallocation task "
+                "returned non-void values: %s" % (
+                    self._task_name, self._minion_pool_id, res))
+        updated_values = {
+            "shared_resources": None}
+        db_api.update_minion_pool(
+            context, self._minion_pool_id, updated_values)
+        self._add_minion_pool_event(
+            context, "Successfully deallocated shared pool resources")
+        return task_info
+
+
+class AllocateMinionMachineTask(BaseMinionManangerTask):
+
+    def __init__(
+            self, minion_pool_id, minion_machine_id, minion_pool_type,
+            raise_on_cleanup_failure=True, allocate_to_action=None, **kwargs):
+        resource_deployment_task_type = (
+            constants.TASK_TYPE_CREATE_SOURCE_MINION_MACHINE)
+        resource_cleanup_task_type = (
+            constants.TASK_TYPE_DELETE_SOURCE_MINION_MACHINE)
+        if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE:
+            resource_deployment_task_type = (
+                constants.TASK_TYPE_CREATE_DESTINATION_MINION_MACHINE)
+            resource_cleanup_task_type = (
+                constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE)
+        self._allocate_to_action = allocate_to_action
+        self._raise_on_cleanup_failure = raise_on_cleanup_failure
+        super(AllocateMinionMachineTask, self).__init__(
+            minion_pool_id, minion_machine_id, resource_deployment_task_type,
+            cleanup_task_runner_type=resource_cleanup_task_type,
+            raise_on_cleanup_failure=raise_on_cleanup_failure, **kwargs)
+
+    def _get_task_name(self, minion_pool_id, minion_machine_id):
+        return MINION_POOL_ALLOCATE_MACHINE_TASK_NAME_FORMAT % (
+            minion_pool_id, minion_machine_id)
+
+    def execute(self, context, origin, destination, task_info):
+        minion_machine = self._get_minion_machine(
+            context, self._minion_machine_id, raise_if_not_found=False)
+        if minion_machine:
+            if minion_machine.allocation_status != (
+                    constants.MINION_MACHINE_STATUS_UNINITIALIZED):
+                raise exception.InvalidMinionMachineState(
+                    "Minion machine entry with ID '%s' already exists within "
+                    "the DB and it is in '%s' status instead of the expected "
+                    "'%s' status. Existing machine's properties are: %s" % (
+                        self._minion_machine_id, minion_machine.allocation_status,
+                        constants.MINION_MACHINE_STATUS_UNINITIALIZED,
+                        minion_machine.to_dict()))
+            if minion_machine.pool_id != self._minion_pool_id:
+                raise exception.InvalidMinionMachineState(
+                    "Minion machine entry with ID '%s' already exists within "
+                    "the DB but it belongs to a different minion pool ('%s') "
+                    "from the one requested by this task ('%s')." % (
+                        self._minion_machine_id, minion_machine.pool_id,
+                        self._minion_pool_id))
+            if self._allocate_to_action and (
+                    minion_machine.allocated_action and (
+                        self._allocate_to_action != (
+                            minion_machine.allocated_action))):
+                raise exception.InvalidMinionMachineState(
+                    "Minion machine entry with ID '%s' already exists in the "
+                    "DB but it is already allocated to a different action "
+                    "('%s') from the one requested by the task ('%s')." % (
+                        self._minion_machine_id,
+                        minion_machine.allocated_action,
+                        self._allocate_to_action))
+            LOG.info(
+                "[Task '%s'] Found existing entry in DB for minion machine "
+                "'%s'. Reusing that for deployment task.",
+                self._task_name, self._minion_machine_id)
+            self._set_minion_machine_allocation_status(
+                context, self._minion_pool_id, self._minion_machine_id,
+                constants.MINION_MACHINE_STATUS_ALLOCATING)
+        else:
+            minion_machine = models.MinionMachine()
+            minion_machine.id = self._minion_machine_id
+            minion_machine.pool_id = self._minion_pool_id
+            minion_machine.allocation_status = (
+                constants.MINION_MACHINE_STATUS_ALLOCATING)
+            minion_machine.power_status = (
+                constants.MINION_MACHINE_POWER_STATUS_UNINITIALIZED)
+            log_msg = (
+                "[Task '%s'] Adding new minion machine with ID '%s' "
+                "to the DB" % (self._task_name, self._minion_machine_id))
+            if self._allocate_to_action:
+                minion_machine.allocated_action = self._allocate_to_action
+                log_msg = "%s (allocated to action '%s')" % (
+                    log_msg, self._allocate_to_action)
+            LOG.info(log_msg)
+            db_api.add_minion_machine(context, minion_machine)
+
+        execution_info = {
+            "pool_environment_options": task_info["pool_environment_options"],
+            "pool_identifier": task_info["pool_identifier"],
+            "pool_shared_resources": task_info["pool_shared_resources"],
+            "pool_os_type": task_info["pool_os_type"]}
+
+        event_message = (
+            "Allocating minion machine with internal pool ID '%s'" % (
+                self._minion_machine_id))
+        if self._allocate_to_action:
+            event_message = (
+                "%s to be used for transfer action with ID '%s'" % (
+                    event_message, self._allocate_to_action))
+        self._add_minion_pool_event(
+            context, event_message)
+
+        try:
+            res = super(AllocateMinionMachineTask, self).execute(
+                context, origin, destination, execution_info)
+        except:
+            self._set_minion_machine_allocation_status(
+                context, self._minion_pool_id, self._minion_machine_id,
+                constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING)
+            raise
+
+        self._add_minion_pool_event(
+            context,
+            "Successfully allocated minion machine with internal pool "
+            "ID '%s'" % (self._minion_machine_id))
+
+        updated_values = {
+            "last_used_at": timeutils.utcnow(),
+            "allocation_status": constants.MINION_MACHINE_STATUS_AVAILABLE,
+            "power_status": constants.MINION_MACHINE_POWER_STATUS_POWERED_ON,
+            "connection_info": res['minion_connection_info'],
+            "provider_properties": res['minion_provider_properties'],
+            "backup_writer_connection_info": res[
+                "minion_backup_writer_connection_info"]}
+        if self._allocate_to_action:
+            updated_values["allocated_action"] = self._allocate_to_action
+            updated_values["allocation_status"] = (
+                constants.MINION_MACHINE_STATUS_IN_USE)
+        self._update_minion_machine(
+            context, self._minion_pool_id, self._minion_machine_id,
+            updated_values)
+
+        return task_info
+
+    def revert(self, context, origin, destination, task_info, **kwargs):
+
+        minion_provider_properties = None
+        task_info_minion_provider_properties = task_info.get(
+            'minion_provider_properties')
+
+        # check if the original result is a taskflow Failure object:
+        original_result = kwargs.get('result', {})
+        if isinstance(original_result, failure.Failure):
+            LOG.debug(
+                "[Task '%s'] Reversion for allocation Minion Machine '%s' "
+                "(pool '%s') received a failure as the original result. "
+                "Presuming the original execution failed and found the "
+                "following 'machine_properties' key in task info: %s",
+                self._task_name, self._minion_machine_id,
+                self._minion_pool_id, task_info_minion_provider_properties)
+            LOG.warn(
+                "[Task '%s'] Allocation failed for machine '%s'. Error "
+                "details were: %s",
+                self._task_name, self._minion_machine_id,
+                original_result.traceback_str)
+        # else, if it's a dict, fetch it:
+        elif isinstance(original_result, dict):
+            minion_provider_properties = original_result.get(
+                'minion_provider_properties', None)
+        else:
+            LOG.warn(
+                "[Task '%s'] Allocation task reversion for machine '%s' "
+                "of pool '%s' got an unexpected task result type (%s): %s",
+                    self._task_name, self._minion_machine_id,
+                    self._minion_pool_id, type(original_result),
+                    original_result)
+            minion_provider_properties = None
+
+        # default to any minion properties found in the task_info:
+        task_info_minion_provider_properties = task_info.get(
+            'minion_provider_properties')
+        if not minion_provider_properties:
+            LOG.debug(
+                "[Task '%s'] Reversion for Minion Machine '%s' (pool '%s')"
+                " did not return any 'minion_provider_properties' after "
+                "its initial execution. Defaulting to task_info value: %s",
+                self._task_name, self._minion_machine_id,
+                self._minion_pool_id,
+                task_info_minion_provider_properties)
+            minion_provider_properties = task_info_minion_provider_properties
+
+        # lastly, if the machine entry exists in the DB:
+        with minion_manager_utils.get_minion_pool_lock(
+            self._minion_pool_id, external=True):
+            machine_db_entry = (
+                db_api.get_minion_machine(context, self._minion_machine_id))
+            if machine_db_entry:
+                LOG.debug(
+                    "[Task %s] Removing minion machine entry with ID '%s' for "
+                    "minion pool '%s' from the DB as part of reversion of its "
+                    "allocation task. Machine properties at deletion time "
+                    "were: %s",
+                    self._task_name, self._minion_machine_id,
+                    self._minion_pool_id,
+                    machine_db_entry.to_dict())
+                if not minion_provider_properties and (
+                        machine_db_entry.provider_properties):
+                    minion_provider_properties = (
+                         machine_db_entry.provider_properties)
+                    LOG.debug(
+                        "[Task '%s'] Using minion provider properties of "
+                        "minion machine with ID '%s' from DB entry during the "
+                        "reversion of its allocation task. DB props are: %s",
+                        self._task_name, self._minion_machine_id,
+                        minion_provider_properties)
+
+            LOG.debug(
+                "[Task %s] Deleting minion machine with ID '%s' from the DB.",
+                self._task_name, self._minion_machine_id)
+            try:
+                db_api.delete_minion_machine(context, self._minion_machine_id)
+            except Exception as ex:
+                LOG.warn(
+                    "[Task '%s'] Failed to delete DB entry for minion machine "
+                    "'%s' following reversion of its allocation task. Error "
+                    "trace was: %s",
+                    self._task_name, self._minion_machine_id,
+                    utils.get_exception_details())
+
+        if not minion_provider_properties:
+            LOG.debug(
+                "[Task '%s'] Reversion for Minion Machine '%s' (pool '%s') "
+                "found no 'minion_provider_properties'. Presuming the machine "
+                "never got created on the cloud and skiping any deletion task",
+                self._task_name, self._minion_machine_id,
+                self._minion_pool_id)
+        else:
+            cleanup_info = copy.deepcopy(task_info)
+            cleanup_info['minion_provider_properties'] = (
+                minion_provider_properties)
+            try:
+                super(AllocateMinionMachineTask, self).revert(
+                    context, origin, destination, cleanup_info, **kwargs)
+            except Exception as ex:
+                log_msg = (
+                    "[Task '%s'] Exception occurred while attempting to revert "
+                    "deployment of minion machine with ID '%s' for pool '%s'." % (
+                        self._task_name, self._minion_machine_id,
+                        self._minion_pool_id))
+                if not self._raise_on_cleanup_failure:
+                    log_msg = (
+                        "%s Ignoring exception." % log_msg)
+                log_msg = (
+                    "%s Exception details were: %s" % (
+                        log_msg, utils.get_exception_details))
+                LOG.warn(log_msg)
+                if self._raise_on_cleanup_failure:
+                    raise
+
+
+class DeallocateMinionMachineTask(BaseMinionManangerTask):
+
+    def __init__(
+            self, minion_pool_id, minion_machine_id, minion_pool_type,
+            raise_on_cleanup_failure=True, **kwargs):
+        resource_deletion_task_type = (
+            constants.TASK_TYPE_DELETE_SOURCE_MINION_MACHINE)
+        self._raise_on_cleanup_failure = raise_on_cleanup_failure
+        if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE:
+            resource_deletion_task_type = (
+                constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE)
+        super(DeallocateMinionMachineTask, self).__init__(
+            minion_pool_id, minion_machine_id, resource_deletion_task_type,
+            raise_on_cleanup_failure=raise_on_cleanup_failure, **kwargs)
+
+    def _get_task_name(self, minion_pool_id, minion_machine_id):
+        return MINION_POOL_DEALLOCATE_MACHINE_TASK_NAME_FORMAT % (
+            minion_pool_id, minion_machine_id)
+
+    def execute(self, context, origin, destination, task_info):
+        machine = self._get_minion_machine(context, self._minion_machine_id)
+        if not machine:
+            LOG.info(
+                "[Task '%s'] Could not find machine with ID '%s' in the DB. "
+                "Presuming it was already deleted and returning early.",
+                self._task_name, self._minion_machine_id)
+            return task_info
+
+        self._add_minion_pool_event(
+            context,
+            "Deallocating minion machine with internal pool ID '%s'" % (
+                self._minion_machine_id))
+
+        if machine.provider_properties:
+            self._set_minion_machine_allocation_status(
+                context, self._minion_pool_id, self._minion_machine_id,
+                constants.MINION_MACHINE_STATUS_DEALLOCATING)
+            execution_info = {
+                "minion_provider_properties": machine.provider_properties}
+            try:
+                _ = super(DeallocateMinionMachineTask, self).execute(
+                    context, origin, destination, execution_info)
+            except Exception as ex:
+                base_msg = (
+                    "Exception occured while deallocating minion machine '%s' "
+                    "There might be leftover instance resources requiring "
+                    "manual cleanup" % self._minion_machine_id)
+                LOG.warn(
+                    "[Task '%s'] %s. Error was: %s",
+                    self._task_name, base_msg, utils.get_exception_details())
+                event_level = constants.TASK_EVENT_INFO
+                if self._raise_on_cleanup_failure:
+                    event_level = constants.TASK_EVENT_ERROR
+                self._add_minion_pool_event(
+                    context, base_msg, level=event_level)
+                if self._raise_on_cleanup_failure:
+                    raise
+        else:
+            self._add_minion_pool_event(
+                context,
+                "Minion machine with ID '%s' had no provider properties set. "
+                "Presuming it failed to deploy in the first place and simply "
+                "removing the machine's entry from the DB" % (
+                    self._minion_machine_id))
+
+        LOG.debug(
+            "[Task '%s'] Deleting minion machine with ID '%s' from the DB",
+            self._task_name, self._minion_machine_id)
+        with minion_manager_utils.get_minion_pool_lock(
+                self._minion_pool_id, external=True):
+            db_api.delete_minion_machine(context, self._minion_machine_id)
+
+        self._add_minion_pool_event(
+            context,
+            "Successfully deallocated minion machine with internal pool "
+            "ID '%s'" % (self._minion_machine_id))
+
+        return task_info
+
+
+class HealthcheckMinionMachineTask(BaseMinionManangerTask):
+    """ Task which healthchecks the given minion machine. """
+
+    def __init__(
+            self, minion_pool_id, minion_machine_id, minion_pool_type,
+            fail_on_error=False,
+            machine_status_on_success=constants.MINION_MACHINE_STATUS_AVAILABLE,
+            **kwargs):
+        self._fail_on_error = fail_on_error
+        self._machine_status_on_success = machine_status_on_success
+        resource_healthcheck_task = (
+            constants.TASK_TYPE_HEALTHCHECK_SOURCE_MINION)
+        if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE:
+            resource_healthcheck_task = (
+                constants.TASK_TYPE_HEALTHCHECK_DESTINATION_MINION)
+        super(HealthcheckMinionMachineTask, self).__init__(
+            minion_pool_id, minion_machine_id, resource_healthcheck_task,
+            **kwargs)
+
+    def execute(self, context, origin, destination, task_info):
+        res = {
+            "healthy": True,
+            "error": None}
+
+        machine = self._get_minion_machine(
+            context, self._minion_machine_id, raise_if_not_found=False)
+        if not machine:
+            LOG.info(
+                "[Task '%s'] Could not find machine with ID '%s' in the DB. "
+                "Presuming it was already deleted so healthcheck failed.",
+                self._task_name, self._minion_machine_id)
+            base_msg = (
+                "Could not find minion machine DB entry with ID '%s' for "
+                "healtcheck." % self._minion_machine_id)
+            self._add_minion_pool_event(
+                context,
+                "%s Reporting healthcheck as failed" % base_msg,
+                level=constants.TASK_EVENT_WARNING)
+
+            if self._fail_on_error:
+                raise exception.InvalidMinionMachineState(base_msg)
+            return {"healthy": False, "error": base_msg}
+
+        machine_error_statuses = [
+            constants.MINION_MACHINE_STATUS_ERROR,
+            constants.MINION_MACHINE_STATUS_POWER_ERROR,
+            constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING]
+        if machine.allocation_status in machine_error_statuses:
+            base_msg = (
+                "Minion Machine with ID '%s' is marked as '%s' in the DB." % (
+                    self._minion_machine_id, machine.allocation_status))
+            LOG.debug(
+                "[Task '%s'] %s" % (self._task_name, base_msg))
+            self._add_minion_pool_event(
+                context,
+                "%s Reporting healthcheck as failed" % base_msg,
+                level=constants.TASK_EVENT_WARNING)
+
+            if self._fail_on_error:
+                raise exception.InvalidMinionMachineState(base_msg)
+            return {"healthy": False, "error": base_msg}
+
+        self._add_minion_pool_event(
+            context,
+            "Healthchecking  minion machine with internal pool ID '%s'" % (
+                self._minion_machine_id))
+
+        execution_info = {
+            "minion_provider_properties": machine.provider_properties,
+            "minion_connection_info": machine.connection_info}
+        try:
+            _ = super(HealthcheckMinionMachineTask, self).execute(
+                context, origin, destination, execution_info)
+            self._add_minion_pool_event(
+                context,
+                "Successfully healtchecked minion machine with internal "
+                "pool ID '%s'" % self._minion_machine_id)
+            self._set_minion_machine_allocation_status(
+                context, self._minion_pool_id, self._minion_machine_id,
+                self._machine_status_on_success)
+        except Exception as ex:
+            self._add_minion_pool_event(
+                context,
+                "Healtcheck for machine with internal pool ID '%s' has "
+                "failed." % (self._minion_machine_id),
+                level=constants.TASK_EVENT_WARNING)
+            LOG.debug(
+                "[Task '%s'] Healtcheck failed for machine '%s' of pool '%s'. "
+                "Full trace was:\n%s", self._task_name,
+                self._minion_machine_id, self._minion_pool_id,
+                utils.get_exception_details())
+            self._set_minion_machine_allocation_status(
+                context, self._minion_pool_id, self._minion_machine_id,
+                constants.MINION_MACHINE_STATUS_ERROR)
+            if not self._fail_on_error:
+                res = {
+                    "healthy": False,
+                    "error": str(ex)}
+            else:
+                raise
+
+        return res
+
+    def _get_task_name(self, minion_pool_id, minion_machine_id):
+        return self.get_healthcheck_task_name(
+            minion_pool_id, minion_machine_id)
+
+    @classmethod
+    def get_healthcheck_task_name(cls, minion_pool_id, minion_machine_id):
+        return MINION_POOL_HEALTHCHECK_MACHINE_TASK_NAME_FORMAT % (
+            minion_pool_id, minion_machine_id)
+
+
+class MinionMachineHealtchcheckDecider(object):
+    """ A callable to green/redlight further execution based on the result. """
+
+    def __init__(
+            self, minion_pool_id, minion_machine_id,
+            on_successful_healthcheck=True):
+        self._minion_pool_id = minion_pool_id
+        self._minion_machine_id = minion_machine_id
+        self._on_success = on_successful_healthcheck
+
+    def __call__(self, history):
+        healthcheck_task_name = (
+            HealthcheckMinionMachineTask.get_healthcheck_task_name(
+                self._minion_pool_id, self._minion_machine_id))
+
+        if not history and healthcheck_task_name not in history:
+            LOG.warn(
+                "Could not find healthceck result for minion machine '%s' "
+                "of pool '%s' (task name '%s'). NOT grennlighting futher "
+                "tasks.", self._minion_machine_id, self._minion_pool_id,
+                healthcheck_task_name)
+            return False
+
+        healtcheck_result = history[healthcheck_task_name]
+        if healtcheck_result.get('healthy'):
+            LOG.debug(
+                "Healtcheck task '%s' confirmed worker health. Decider "
+                "returning %s", healthcheck_task_name,
+                self._on_success)
+            return self._on_success
+        else:
+            LOG.debug(
+                "Healtcheck task '%s' denied worker health. Decider "
+                "returning %s. Error mesage was: %s",
+                healthcheck_task_name, not self._on_success,
+                healtcheck_result.get('error'))
+            return not self._on_success
+
+
+class PowerOnMinionMachineTask(BaseMinionManangerTask):
+
+    def __init__(
+            self, minion_pool_id, minion_machine_id, minion_pool_type,
+            fail_on_error=True, **kwargs):
+        self._fail_on_error = fail_on_error
+        power_on_task_type = (
+            constants.TASK_TYPE_POWER_ON_SOURCE_MINION)
+        if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE:
+            power_on_task_type = (
+                constants.TASK_TYPE_POWER_ON_DESTINATION_MINION)
+        super(PowerOnMinionMachineTask, self).__init__(
+            minion_pool_id, minion_machine_id, power_on_task_type,
+            **kwargs)
+
+    def _get_task_name(self, minion_pool_id, minion_machine_id):
+        return MINION_POOL_POWER_ON_MACHINE_TASK_NAME_FORMAT % (
+            minion_pool_id, minion_machine_id)
+
+    def execute(self, context, origin, destination, task_info):
+        machine = self._get_minion_machine(
+            context, self._minion_machine_id, raise_if_not_found=True)
+
+        if machine.power_status == constants.MINION_MACHINE_POWER_STATUS_POWERED_ON:
+            LOG.debug(
+                "[Task '%s'] Minion machine with ID '%s' from pool '%s' is "
+                "already marked as powered on. Returning early." % (
+                    self._task_name, self._minion_machine_id,
+                    self._minion_pool_id))
+            return task_info
+
+        if machine.power_status != constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF:
+            raise exception.InvalidMinionMachineState(
+                "Minion machine with ID '%s' from pool '%s' is in '%s' state "
+                "instead of the expected '%s' required for it to be powered "
+                "on." % (
+                    self._minion_machine_id, self._minion_pool_id,
+                    machine.power_status,
+                    constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF))
+
+        execution_info = {
+            "minion_provider_properties": machine.provider_properties}
+        try:
+            self._set_minion_machine_power_status(
+                context, self._minion_pool_id,
+                self._minion_machine_id,
+                constants.MINION_MACHINE_POWER_STATUS_POWERING_ON)
+            _ = super(PowerOnMinionMachineTask, self).execute(
+                context, origin, destination, execution_info)
+            self._set_minion_machine_power_status(
+                context, self._minion_pool_id,
+                self._minion_machine_id,
+                constants.MINION_MACHINE_POWER_STATUS_POWERED_ON)
+            self._add_minion_pool_event(
+                context,
+                "Successfully powered on minion machine with internal pool "
+                "ID '%s'" % self._minion_machine_id)
+        except Exception as ex:
+            base_msg = (
+                "[Task '%s'] Exception occurred while powering on minion "
+                "machine with ID '%s' of pool '%s'." % (
+                    self._task_name, self._minion_machine_id,
+                    self._minion_pool_id))
+            LOG.warn(
+                "%s Error details were: %s" % (
+                    base_msg, utils.get_exception_details()))
+            self._add_minion_pool_event(
+                context,
+                "Exception occurred while powering on minion machine with "
+                "internal pool ID '%s'. The minion machine will be marked "
+                "as ERROR'd and automatically redeployed later" % (
+                    self._minion_machine_id),
+                level=constants.TASK_EVENT_ERROR)
+            self._set_minion_machine_allocation_status(
+                context, self._minion_pool_id, self._minion_machine_id,
+                constants.MINION_MACHINE_STATUS_POWER_ERROR)
+            if self._fail_on_error:
+                raise exception.CoriolisException(base_msg)
+
+        return task_info
+
+
+class PowerOffMinionMachineTask(BaseMinionManangerTask):
+
+    def __init__(
+            self, minion_pool_id, minion_machine_id, minion_pool_type,
+            fail_on_error=True,
+            status_once_powered_off=constants.MINION_MACHINE_STATUS_AVAILABLE,
+            **kwargs):
+        self._fail_on_error = fail_on_error
+        self._status_once_powered_off = status_once_powered_off
+        power_on_task_type = (
+            constants.TASK_TYPE_POWER_OFF_SOURCE_MINION)
+        if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE:
+            power_on_task_type = (
+                constants.TASK_TYPE_POWER_OFF_DESTINATION_MINION)
+        super(PowerOffMinionMachineTask, self).__init__(
+            minion_pool_id, minion_machine_id, power_on_task_type,
+            **kwargs)
+
+    def _get_task_name(self, minion_pool_id, minion_machine_id):
+        return MINION_POOL_POWER_OFF_MACHINE_TASK_NAME_FORMAT % (
+            minion_pool_id, minion_machine_id)
+
+    def execute(self, context, origin, destination, task_info):
+        machine = self._get_minion_machine(
+            context, self._minion_machine_id, raise_if_not_found=True)
+
+        if machine.power_status == (
+                constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF):
+            LOG.debug(
+                "[Task '%s'] Minion machine with ID '%s' from pool '%s' is "
+                "already marked as powered off. Returning early." % (
+                    self._task_name, self._minion_machine_id,
+                    self._minion_pool_id))
+            return task_info
+
+        execution_info = {
+            "minion_provider_properties": machine.provider_properties}
+        try:
+            self._set_minion_machine_power_status(
+                context, self._minion_pool_id,
+                self._minion_machine_id,
+                constants.MINION_MACHINE_POWER_STATUS_POWERING_OFF)
+            _ = super(PowerOffMinionMachineTask, self).execute(
+                context, origin, destination, execution_info)
+            self._set_minion_machine_power_status(
+                context, self._minion_pool_id,
+                self._minion_machine_id,
+                constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF)
+            self._set_minion_machine_allocation_status(
+                context, self._minion_pool_id, self._minion_machine_id,
+                self._status_once_powered_off)
+            self._add_minion_pool_event(
+                context,
+                "Successfully powered off minion machine with internal pool "
+                "ID '%s'" % self._minion_machine_id)
+        except Exception as ex:
+            base_msg = (
+                "[Task '%s'] Exception occurred while powering off minion "
+                "machine with ID '%s' of pool '%s'." % (
+                    self._task_name, self._minion_machine_id,
+                    self._minion_pool_id))
+            self._add_minion_pool_event(
+                context,
+                "Exception occurred while powering off minion machine with "
+                "internal pool ID '%s'. The minion machine will be marked "
+                "as ERROR'd and automatically redeployed later." % (
+                    self._minion_machine_id),
+                level=constants.TASK_EVENT_ERROR)
+            self._set_minion_machine_allocation_status(
+                context, self._minion_pool_id, self._minion_machine_id,
+                constants.MINION_MACHINE_STATUS_POWER_ERROR)
+            LOG.warn(
+                "%s Error details were: %s" % (
+                    base_msg, utils.get_exception_details()))
+            if self._fail_on_error:
+                raise exception.CoriolisException(base_msg)
+
+        return task_info

coriolis/minion_manager/rpc/utils.py

@@ -0,0 +1,47 @@
+# Copyright 2020 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import functools
+
+from oslo_concurrency import lockutils
+
+from coriolis import constants
+
+
+def get_minion_pool_lock(minion_pool_id, external=True):
+    return lockutils.lock(
+        constants.MINION_POOL_LOCK_NAME_FORMAT % minion_pool_id,
+        external=external)
+
+
+def minion_pool_synchronized(minion_pool_id, func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        @lockutils.synchronized(
+            constants.MINION_POOL_LOCK_NAME_FORMAT % minion_pool_id,
+            external=True)
+        def inner():
+            return func(*args, **kwargs)
+        return inner()
+    return wrapper
+
+
+def minion_pool_synchronized_op(func):
+    @functools.wraps(func)
+    def wrapper(self, ctxt, minion_pool_id, *args, **kwargs):
+        return minion_pool_synchronized(minion_pool_id, func)(
+            self, ctxt, minion_pool_id, *args, **kwargs)
+    return wrapper
+
+
+def minion_machine_synchronized(minion_pool_id, minion_machine_id, func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        @lockutils.synchronized(
+            constants.MINION_MACHINE_LOCK_NAME_FORMAT % (
+                minion_pool_id, minion_machine_id),
+            external=True)
+        def inner():
+            return func(*args, **kwargs)
+        return inner()
+    return wrapper

coriolis/minion_pool_tasks_executions/api.py

@@ -1,26 +0,0 @@
-# Copyright 2020 Cloudbase Solutions Srl
-# All Rights Reserved.
-
-from coriolis import utils
-from coriolis.conductor.rpc import client as rpc_client
-
-
-class API(object):
-    def __init__(self):
-        self._rpc_client = rpc_client.ConductorClient()
-
-    def list(self, ctxt, minion_pool_id, include_tasks=False):
-        return self._rpc_client.get_minion_pool_lifecycle_executions(
-            ctxt, minion_pool_id, include_tasks=include_tasks)
-
-    def get(self, ctxt, minion_pool_id, execution_id):
-        return self._rpc_client.get_minion_pool_lifecycle_execution(
-            ctxt, minion_pool_id, execution_id)
-
-    def cancel(self, ctxt, minion_pool_id, execution_id, force):
-        return self._rpc_client.cancel_minion_pool_lifecycle_execution(
-            ctxt, minion_pool_id, execution_id, force)
-
-    def delete(self, ctxt, minion_pool_id, execution_id):
-        return self._rpc_client.delete_minion_pool_lifecycle_execution(
-            ctxt, minion_pool_id, execution_id)

coriolis/minion_pools/api.py

@@ -2,21 +2,23 @@
 # All Rights Reserved.
 
 from coriolis import utils
-from coriolis.conductor.rpc import client as rpc_client
+from coriolis.minion_manager.rpc import client as rpc_client
 
 
 class API(object):
     def __init__(self):
-        self._rpc_client = rpc_client.ConductorClient()
+        self._rpc_client = rpc_client.MinionManagerClient()
 
     def create(
             self, ctxt, name, endpoint_id, pool_platform, pool_os_type,
             environment_options, minimum_minions, maximum_minions,
-            minion_max_idle_time, minion_retention_strategy, notes=None):
+            minion_max_idle_time, minion_retention_strategy, notes=None,
+            skip_allocation=False):
         return self._rpc_client.create_minion_pool(
             ctxt, name, endpoint_id, pool_platform, pool_os_type,
             environment_options, minimum_minions, maximum_minions,
-            minion_max_idle_time, minion_retention_strategy, notes=notes)
+            minion_max_idle_time, minion_retention_strategy, notes=notes,
+            skip_allocation=skip_allocation)
 
     def update(self, ctxt, minion_pool_id, updated_values):
         return self._rpc_client.update_minion_pool(
@@ -31,19 +33,14 @@ class API(object):
     def get_minion_pool(self, ctxt, minion_pool_id):
         return self._rpc_client.get_minion_pool(ctxt, minion_pool_id)
 
-    def set_up_shared_pool_resources(self, ctxt, minion_pool_id):
-        return self._rpc_client.set_up_shared_minion_pool_resources(
+    def allocate_minion_pool(self, ctxt, minion_pool_id):
+        return self._rpc_client.allocate_minion_pool(
             ctxt, minion_pool_id)
 
-    def tear_down_shared_pool_resources(
-            self, ctxt, minion_pool_id, force=False):
-        return self._rpc_client.tear_down_shared_minion_pool_resources(
-            ctxt, minion_pool_id, force=force)
-
-    def allocate_machines(self, ctxt, minion_pool_id):
-        return self._rpc_client.allocate_minion_pool_machines(
+    def refresh_minion_pool(self, ctxt, minion_pool_id):
+        return self._rpc_client.refresh_minion_pool(
             ctxt, minion_pool_id)
 
-    def deallocate_machines(self, ctxt, minion_pool_id, force=False):
-        return self._rpc_client.deallocate_minion_pool_machines(
+    def deallocate_minion_pool(self, ctxt, minion_pool_id, force=False):
+        return self._rpc_client.deallocate_minion_pool(
             ctxt, minion_pool_id, force=force)

coriolis/osmorphing/osmount/windows.py

@@ -20,26 +20,12 @@ class WindowsMountTools(base.BaseOSMountTools):
 
         host = connection_info["ip"]
         port = connection_info.get("port", 5986)
-        username = connection_info["username"]
-        password = connection_info.get("password")
-        cert_pem = connection_info.get("cert_pem")
-        cert_key_pem = connection_info.get("cert_key_pem")
-        url = "https://%s:%s/wsman" % (host, port)
-
-        LOG.info("Connection info: %s", str(connection_info))
-
-        LOG.info("Waiting for connectivity on host: %(host)s:%(port)s",
-                 {"host": host, "port": port})
-        utils.wait_for_port_connectivity(host, port)
-
         self._event_manager.progress_update(
             "Connecting to WinRM host: %(host)s:%(port)s" %
             {"host": host, "port": port})
 
-        conn = wsman.WSManConnection()
-        conn.connect(url=url, username=username, password=password,
-                     cert_pem=cert_pem, cert_key_pem=cert_key_pem)
-        self._conn = conn
+        self._conn = wsman.WSManConnection.from_connection_info(
+            connection_info)
 
     def get_connection(self):
         return self._conn

coriolis/policies/minion_pools.py

@@ -73,9 +73,9 @@ MINION_POOLS_DEFAULT_RULES = [
         ]
     ),
     policy.DocumentedRuleDefault(
-        get_minion_pools_policy_label('set_up_shared_resources'),
+        get_minion_pools_policy_label('allocate'),
         MINION_POOLS_DEFAULT_RULE,
-        "Set up shared minion pool resources",
+        "Allocate Minion Pool",
         [
             {
                 "path": "/minion_pools/{minion_pool_id}/actions",
@@ -84,9 +84,9 @@ MINION_POOLS_DEFAULT_RULES = [
         ]
     ),
     policy.DocumentedRuleDefault(
-        get_minion_pools_policy_label('tear_down_shared_resources'),
+        get_minion_pools_policy_label('refresh'),
         MINION_POOLS_DEFAULT_RULE,
-        "Tear down shared minion pool resources",
+        "Refresh Minion Pool",
         [
             {
                 "path": "/minion_pools/{minion_pool_id}/actions",
@@ -95,20 +95,9 @@ MINION_POOLS_DEFAULT_RULES = [
         ]
     ),
     policy.DocumentedRuleDefault(
-        get_minion_pools_policy_label('allocate_machines'),
+        get_minion_pools_policy_label('deallocate'),
         MINION_POOLS_DEFAULT_RULE,
-        "Allocate Minion Pool machines",
-        [
-            {
-                "path": "/minion_pools/{minion_pool_id}/actions",
-                "method": "POST"
-            }
-        ]
-    ),
-    policy.DocumentedRuleDefault(
-        get_minion_pools_policy_label('deallocate_machines'),
-        MINION_POOLS_DEFAULT_RULE,
-        "Deallocate Minion Pool machines",
+        "Deallocate Minion Pool",
         [
             {
                 "path": "/minion_pools/{minion_pool_id}/actions",

coriolis/providers/base.py

@@ -614,6 +614,12 @@ class _BaseMinionPoolProvider(
             self, ctxt, connection_info, minion_properties, volumes_info):
         pass
 
+    @abc.abstractmethod
+    def healthcheck_minion(
+            self, ctxt, environment_options, connection_info,
+            minion_properties, minion_connection_info):
+        pass
+
 
 class BaseSourceMinionPoolProvider(_BaseMinionPoolProvider):
 

coriolis/providers/replicator.py

@@ -434,10 +434,8 @@ class Replicator(object):
                 perc_step = perc_steps.get(devName)
                 if perc_step is None:
                     perc_step = self._event_manager.add_percentage_step(
-                        100,
-                        message_format=(
-                            "Chunking progress for disk %s (%.2f MB): "
-                            "{:.0f}%%") % (devName, dev_size))
+                        "Performing chunking for disk %s (total size %.2f MB)" % (
+                            devName, dev_size), 100)
                     perc_steps[devName] = perc_step
                 perc_done = vol["checksum-status"]["percentage"]
                 self._event_manager.set_percentage_step(
@@ -801,11 +799,11 @@ class Replicator(object):
             size = self._get_size_from_chunks(chunks)
 
             msg = (
-                "Disk replication progress for disk \"%s\" (device \"%s\" "
-                "written chunks: %.2f MB): {:.0f}%%") % (
+                "Replicating changed data for disk \"%s\" (device \"%s\", "
+                "written chunks: %.2f MB)") % (
                     volume["disk_id"], devName, size)
             perc_step = self._event_manager.add_percentage_step(
-                len(chunks), message_format=msg)
+                msg, len(chunks))
 
             total = 0
             with backup_writer.open("", volume['disk_id']) as destination:
@@ -829,8 +827,7 @@ class Replicator(object):
         size = self._cli.get_disk_size(disk)
 
         perc_step = self._event_manager.add_percentage_step(
-            size, message_format="Downloading disk /dev/%s : "
-            "{:.0f}%%" % disk)
+            "Downloading full disk /dev/%s" % disk, size)
 
         total = 0
         with self._cli._cli.get(diskUri, stream=True,
@@ -853,8 +850,8 @@ class Replicator(object):
             # create sparse file
             fp.truncate(size)
             perc_step = self._event_manager.add_percentage_step(
-                len(chunks), message_format="Disk download progress for "
-                "/dev/%s (%s MB): {:.0f}%%" % (disk, size_from_chunks))
+                "Downloading spart disk /dev/%s (%s MB)" % (
+                    disk, size_from_chunks), len(chunks))
             for chunk in chunks:
                 offset = int(chunk["offset"])
                 # seek to offset

coriolis/replica_cron/api.py

@@ -1,5 +1,6 @@
 # Copyright 2017 Cloudbase Solutions Srl
 # All Rights Reserved.
+
 from coriolis.conductor.rpc import client as rpc_client
 
 

coriolis/replica_cron/rpc/server.py

@@ -1,13 +1,16 @@
+# Copyright 2017 Cloudbase Solutions Srl
+# All Rights Reserved.
+
 import json
 
-from coriolis.conductor.rpc import client as rpc_client
+from oslo_log import log as logging
+from oslo_utils import timeutils
+
 from coriolis import context
 from coriolis import exception
 from coriolis import utils
-from coriolis.replica_cron import cron
-
-from oslo_log import log as logging
-from oslo_utils import timeutils
+from coriolis.conductor.rpc import client as rpc_client
+from coriolis.cron import cron
 
 LOG = logging.getLogger(__name__)
 

coriolis/scheduler/rpc/client.py

@@ -1,12 +1,22 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
 
-from oslo_config import cfg
+import random
+import time
+
 import oslo_messaging as messaging
+from oslo_config import cfg
+from oslo_log import log as logging
 
+from coriolis import constants
+from coriolis import exception
 from coriolis import rpc
+from coriolis import utils
+from coriolis.tasks import factory as tasks_factory
+
 
 VERSION = "1.0"
+LOG = logging.getLogger(__name__)
 
 scheduler_opts = [
     cfg.IntOpt("scheduler_rpc_timeout",
@@ -35,3 +45,140 @@ class SchedulerClient(rpc.BaseRPCClient):
         return self._call(
             ctxt, 'get_workers_for_specs', region_sets=region_sets,
             enabled=enabled, provider_requirements=provider_requirements)
+
+    def get_any_worker_service(
+            self, ctxt, random_choice=False, raise_if_none=True):
+        services = self.get_workers_for_specs(ctxt)
+        if not services:
+            if raise_if_none:
+                raise exception.NoWorkerServiceError()
+            return None
+        service = services[0]
+        if random_choice:
+            service = random.choice(services)
+        LOG.debug(
+            "Selected service with ID '%s' for any-worker request.",
+            service['id'])
+        return service
+
+    def get_worker_service_for_specs(
+            self, ctxt, provider_requirements=None, region_sets=None,
+            enabled=True, random_choice=False, raise_on_no_matches=True):
+        """Utility method which ensures at least one service matching
+        the provided requirements exists and is usable.
+        """
+        requirements_str = (
+            "enabled=%s; region_sets=%s; provider_requirements=%s" % (
+                enabled, region_sets, provider_requirements))
+        LOG.info(
+            "Requesting Worker Service from scheduler with the following "
+            "specifications: %s", requirements_str)
+        services = self.get_workers_for_specs(
+            ctxt, provider_requirements=provider_requirements,
+            region_sets=region_sets, enabled=enabled)
+        if not services:
+            if raise_on_no_matches:
+                raise exception.NoSuitableWorkerServiceError()
+            return None
+        LOG.debug(
+            "Was offered Worker Services with the following IDs for "
+            "requirements '%s': %s",
+            requirements_str, [s["id"] for s in services])
+
+        selected_service = services[0]
+        if random_choice:
+            selected_service = random.choice(services)
+
+        LOG.info(
+            "Was offered Worker Service with ID '%s' for requirements: %s",
+            selected_service['id'], requirements_str)
+        return selected_service
+
+    def get_worker_service_for_task(
+            self, ctxt, task, origin_endpoint, destination_endpoint,
+            retry_count=5, retry_period=2, random_choice=True):
+        """ Gets a worker service for the task with the given properties
+        and source/target endpoints.
+
+        :param task: Dict of the form: {
+            "id": "<task_id>",
+            "task_type": "<constants.TASK_TYPE_*>"}
+        :param origin_endpoint: Dict of the form {
+            "id": "<ID>",
+            "mapped_regions": ["List of mapped endpoint regions"]}
+        :param destination_endpoint: Same as origin_endpoint
+        """
+        LOG.debug(
+            "Compiling required Worker Service specs for task with "
+            "ID '%s' (type '%s') from endpoints '%s' to '%s'",
+            task['id'], task['task_type'], origin_endpoint['id'],
+            destination_endpoint['id'])
+        task_cls = tasks_factory.get_task_runner_class(
+            task['task_type'])
+
+        # determine required Coriolis regions based on the endpoints:
+        required_region_sets = []
+        origin_endpoint_region_ids = [
+            r['id'] for r in origin_endpoint['mapped_regions']]
+        destination_endpoint_region_ids = [
+            r['id'] for r in destination_endpoint['mapped_regions']]
+
+        required_platform = task_cls.get_required_platform()
+        if required_platform in (
+                constants.TASK_PLATFORM_SOURCE,
+                constants.TASK_PLATFORM_BILATERAL):
+            required_region_sets.append(origin_endpoint_region_ids)
+        if required_platform in (
+                constants.TASK_PLATFORM_DESTINATION,
+                constants.TASK_PLATFORM_BILATERAL):
+            required_region_sets.append(destination_endpoint_region_ids)
+
+        # determine provider requirements:
+        provider_requirements = {}
+        required_provider_types = task_cls.get_required_provider_types()
+        if constants.PROVIDER_PLATFORM_SOURCE in required_provider_types:
+            provider_requirements[origin_endpoint['type']] = (
+                required_provider_types[
+                    constants.PROVIDER_PLATFORM_SOURCE])
+        if constants.PROVIDER_PLATFORM_DESTINATION in required_provider_types:
+            provider_requirements[destination_endpoint['type']] = (
+                required_provider_types[
+                    constants.PROVIDER_PLATFORM_DESTINATION])
+
+        worker_service = None
+        for i in range(retry_count):
+            try:
+                LOG.debug(
+                    "Requesting Worker Service for task with ID '%s' (type "
+                    "'%s') from endpoints '%s' to '%s'", task['id'],
+                    task['task_type'], origin_endpoint['id'],
+                    destination_endpoint['id'])
+                worker_service = self.get_worker_service_for_specs(
+                    ctxt, provider_requirements=provider_requirements,
+                    region_sets=required_region_sets, enabled=True,
+                    random_choice=random_choice)
+                LOG.debug(
+                    "Scheduler has granted Worker Service '%s' for task with "
+                    "ID '%s' (type '%s') from endpoints '%s' to '%s'",
+                    worker_service['id'], task['id'], task['task_type'],
+                    origin_endpoint['id'], destination_endpoint['id'])
+                return worker_service
+            except Exception as ex:
+                LOG.warn(
+                    "Failed to schedule task with ID '%s' (attempt %d/%d). "
+                    "Waiting %d seconds and then retrying. Error was: %s",
+                    task['id'], i+1, retry_count, retry_period,
+                    utils.get_exception_details())
+                time.sleep(retry_period)
+
+        message = (
+            "Failed to schedule task %s after %d tries. This may indicate that"
+            " there are no Coriolis Worker services able to perform the task "
+            "on the platforms and in the Coriolis Regions required by the "
+            "selected source/destination Coriolis Endpoints. Please review"
+            " the Conductor and Scheduler logs for more exact details." % (
+                task['id'], retry_count))
+        # db_api.set_task_status(
+        #     ctxt, task.id, constants.TASK_STATUS_FAILED_TO_SCHEDULE,
+        #     exception_details=message)
+        raise exception.NoSuitableWorkerServiceError(message)

coriolis/scheduler/scheduler_utils.py

@@ -0,0 +1,67 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import random
+
+from oslo_log import log as logging
+
+from coriolis import constants
+from coriolis.db import api as db_api
+from coriolis import exception
+from coriolis.replica_cron.rpc import client as rpc_cron_client
+from coriolis.scheduler.rpc import client as rpc_scheduler_client
+from coriolis import utils
+from coriolis.worker.rpc import client as rpc_worker_client
+
+
+VERSION = "1.0"
+
+LOG = logging.getLogger(__name__)
+
+RPC_TOPIC_TO_CLIENT_CLASS_MAP = {
+    constants.WORKER_MAIN_MESSAGING_TOPIC: rpc_worker_client.WorkerClient,
+    constants.SCHEDULER_MAIN_MESSAGING_TOPIC: (
+        rpc_scheduler_client.SchedulerClient),
+    constants.REPLICA_CRON_MAIN_MESSAGING_TOPIC: (
+        rpc_cron_client.ReplicaCronClient)
+}
+
+
+def get_rpc_client_for_service(service, *client_args, **client_kwargs):
+    rpc_client_class = RPC_TOPIC_TO_CLIENT_CLASS_MAP.get(service.topic)
+    if not rpc_client_class:
+        raise exception.NotFound(
+            "No RPC client class for service with topic '%s'." % (
+                service.topic))
+
+    topic = service.topic
+    if service.topic == constants.WORKER_MAIN_MESSAGING_TOPIC:
+        # NOTE: coriolis.service.MessagingService-type services (such
+        # as the worker), always have a dedicated per-host queue
+        # which can be used to target the service:
+        topic = constants.SERVICE_MESSAGING_TOPIC_FORMAT % ({
+            "main_topic": constants.WORKER_MAIN_MESSAGING_TOPIC,
+            "host": service.host})
+
+    return rpc_client_class(*client_args, topic=topic, **client_kwargs)
+
+
+def get_any_worker_service(
+        scheduler_client, ctxt, random_choice=False, raw_dict=False):
+    services = scheduler_client.get_workers_for_specs(ctxt)
+    if not services:
+        raise exception.NoWorkerServiceError()
+    service = services[0]
+    if random_choice:
+        service = random.choice(services)
+    if raw_dict:
+        return service
+    return db_api.get_service(ctxt, service['id'])
+
+def get_worker_rpc_for_host(host, *client_args, **client_kwargs):
+    rpc_client_class = RPC_TOPIC_TO_CLIENT_CLASS_MAP[
+        constants.WORKER_MAIN_MESSAGING_TOPIC]
+    topic = constants.SERVICE_MESSAGING_TOPIC_FORMAT % ({
+        "main_topic": constants.WORKER_MAIN_MESSAGING_TOPIC,
+        "host": host})
+    return rpc_client_class(*client_args, topic=topic, **client_kwargs)

coriolis/taskflow/base.py

@@ -0,0 +1,260 @@
+# Copyright 2020 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+from taskflow import task as taskflow_tasks
+from taskflow.types import failure
+
+from coriolis import constants
+from coriolis import exception
+from coriolis import utils
+from coriolis.tasks import factory as tasks_factory
+from coriolis.scheduler.rpc import client as rpc_scheduler_client
+from coriolis.worker.rpc import client as rpc_worker_client
+
+
+TASK_RETURN_VALUE_FORMAT = "%s-result" % (
+        constants.TASK_LOCK_NAME_FORMAT)
+LOG = logging.getLogger()
+
+taskflow_opts = [
+    cfg.IntOpt("worker_task_execution_timeout",
+               default=3600,
+               help="Number of seconds until Coriolis tasks which are executed"
+                    "remotely on a Worker Service through taskflow timeout.")
+]
+
+CONF = cfg.CONF
+CONF.register_opts(taskflow_opts, 'taskflow')
+
+
+class BaseCoriolisTaskflowTask(taskflow_tasks.Task):
+    """ Base class for all TaskFlow tasks within Coriolis. """
+
+    def _get_error_str_for_flow_failures(
+            self, flow_failures, full_tracebacks=True):
+        if not flow_failures:
+            return "No flow failures provided."
+
+        if not flow_failures.items():
+            return "No flow failures present."
+
+        res = ""
+        for (task_id, task_failure) in flow_failures.items():
+            label = "Error message"
+            failure_str = task_failure.exception_str
+            if full_tracebacks:
+                label = "Traceback"
+                failure_str = task_failure.traceback_str
+            else:
+                failure_str = task_failure.exception_str
+                if isinstance(
+                        task_failure.exception,
+                        exception.TaskProcessException):
+                    # NOTE: TaskProcessException contains a full trace
+                    # from the worker service so we must split it:
+                    exception_lines = task_failure.exception_str.split('\n')
+                    if exception_lines:
+                        if len(exception_lines) > 2:
+                            failure_str = exception_lines[-2].strip()
+                        else:
+                            failure_str = exception_lines[-1].strip()
+            res = (
+                "%s %s for task '%s': %s\n" % (
+                    res, label, task_id, failure_str))
+        if res:
+            # remove extra newline:
+            res = res[:-1]
+
+        return res
+
+    def revert(self, *args, **kwargs):
+        result = kwargs.get('result')
+        if isinstance(result, failure.Failure):
+            # it means that this is the task which error'd out:
+            LOG.error(
+                "Taskflow task '%s' is reverting after errorring out with the "
+                "following trace: %s", self.name, result.traceback_str)
+        else:
+            # else the failures were from other tasks:
+            flow_failures = kwargs.get('flow_failures', {})
+            LOG.error(
+                "Taskflow task '%s' is reverting after the failure of one "
+                "or more other tasks (%s) Tracebacks were:\n%s" % (
+                    self.name, list(flow_failures.keys()),
+                    self._get_error_str_for_flow_failures(
+                        flow_failures, full_tracebacks=True)))
+
+
+class BaseRunWorkerTask(BaseCoriolisTaskflowTask):
+    """ Base taskflow.Task implementation for tasks which can be run
+    on the worker service.
+    This class can be seen as an "adapter" between the current
+    coriolis.tasks.TaskRunner classes and taskflow ones.
+
+    :param task_id: ID of the task. This value is declared as a returned value
+        from the task and can be set as a requirement for other tasks, thus
+        achieving a dependency system.
+    :param main_task_runner_class: constants.TASK_TYPE_* referencing the
+        main coriolis.tasks.TaskRunner class to be run on a worker service.
+    :param cleanup_task_runner_task: constants.TASK_TYPE_* referencing the
+        cleanup task to be run on reversion. No cleanup will be performed
+        during the task's reversion (apart from Worker Service deallocation)
+        otherwise.
+    """
+
+    def __init__(
+            self, task_name, task_id, task_instance, main_task_runner_type,
+            cleanup_task_runner_type=None, depends_on=None,
+            raise_on_cleanup_failure=False, **kwargs):
+        self._task_id = task_id
+        self._task_name = task_name
+        self._task_instance = task_instance
+        self._main_task_runner_type = main_task_runner_type
+        self._cleanup_task_runner_type = cleanup_task_runner_type
+        self._raise_on_cleanup_failure = raise_on_cleanup_failure
+        self._scheduler_client_instance = None
+
+        super(BaseRunWorkerTask, self).__init__(name=task_name, **kwargs)
+
+    @property
+    def _scheduler_client(self):
+        if not getattr(self, '_scheduler_client_instance', None):
+            self._scheduler_client_instance = (
+                rpc_scheduler_client.SchedulerClient())
+        return self._scheduler_client_instance
+
+    def _set_provides_for_dependencies(self, kwargs):
+        dep = TASK_RETURN_VALUE_FORMAT % self._task_name
+        if kwargs.get('provides') is not None:
+            kwargs['provides'].append(dep)
+        else:
+            kwargs['provides'] = [dep]
+
+    def _set_requires_for_dependencies(self, kwargs, depends_on):
+        dep_requirements = [
+            TASK_RETURN_VALUE_FORMAT % dep_id
+            for dep_id in depends_on]
+        if kwargs.get('requires') is not None:
+            kwargs['requires'].extend(dep_requirements)
+        elif dep_requirements:
+            kwargs['requires'] = dep_requirements
+        return kwargs
+
+    def _set_requires_for_task_info_fields(self, kwargs):
+        new_requires = kwargs.get('requires', [])
+        main_task_runner = tasks_factory.get_task_runner_class(
+            self._main_task_runner_type)
+        main_task_deps = main_task_runner.get_required_task_info_properties()
+        new_requires.extend(main_task_deps)
+        if self._cleanup_task_runner_type:
+            cleanup_task_runner = tasks_factory.get_task_runner_class(
+                self._cleanup_task_runner_type)
+            cleanup_task_deps = list(
+                set(
+                    cleanup_task_runner.get_required_task_info_properties(
+                        )).difference(
+                            main_task_runner.get_returned_task_info_properties()))
+            new_requires.extend(cleanup_task_deps)
+
+        kwargs['requires'] = new_requires
+        return kwargs
+
+    def _set_provides_for_task_info_fields(self, kwargs):
+        new_provides = kwargs.get('provides', [])
+        main_task_runner = tasks_factory.get_task_runner_class(
+            self._main_task_runner_type)
+        main_task_res = main_task_runner.get_returned_task_info_properties()
+        new_provides.extend(main_task_res)
+        if self._cleanup_task_runner_type:
+            cleanup_task_runner = tasks_factory.get_task_runner_class(
+                self._cleanup_task_runner_type)
+            cleanup_task_res = list(
+                set(
+                    cleanup_task_runner.get_returned_task_info_properties(
+                        )).difference(
+                            main_task_runner.get_returned_task_info_properties()))
+            new_provides.extend(cleanup_task_res)
+
+        kwargs['provides'] = new_provides
+        return kwargs
+
+    def _get_worker_service_rpc_for_task(
+            self, ctxt, task_id, task_type, origin, destination,
+            retry_count=5, retry_period=2,
+            rpc_timeout=CONF.taskflow.worker_task_execution_timeout):
+        task_info = {
+            "id": task_id,
+            "task_type": task_type}
+        worker_service = self._scheduler_client.get_worker_service_for_task(
+            ctxt, task_info, origin, destination, retry_count=retry_count,
+            retry_period=retry_period, random_choice=True)
+        LOG.debug(
+            "[Task '%s'] Was offered the following worker service for executing "
+            "Taskflow worker task '%s': %s",
+                self._task_name, task_id, worker_service['id'])
+
+        return rpc_worker_client.WorkerClient.from_service_definition(
+            worker_service, timeout=rpc_timeout)
+
+    def _execute_task(
+            self, ctxt, task_id, task_type, origin, destination, task_info):
+        worker_rpc = self._get_worker_service_rpc_for_task(
+            ctxt, self._task_id, task_type, origin, destination)
+
+        try:
+            LOG.debug(
+                "[Task '%s'] Starting to run task '%s' (type '%s') "
+                "on worker service." % (
+                    self._task_id, self._task_name, task_type))
+            res = worker_rpc.run_task(
+                ctxt, self._task_id, task_type, origin, destination,
+                self._task_instance, task_info)
+            LOG.debug(
+                "[Task '%s'] Taskflow worker task '%s' (type %s) has "
+                "successfully run and returned the following info: %s" % (
+                    self._task_name, task_id, task_type, res))
+            return res
+        except Exception as ex:
+            LOG.debug(
+                "[Task %s] Exception occurred while executing task '%s' "
+                "(type '%s') on the worker service: %s", self._task_name,
+                task_id, task_type, utils.get_exception_details())
+            raise
+
+    def execute(self, context, origin, destination, task_info):
+        res = self._execute_task(
+            context, self._task_id, self._main_task_runner_type, origin,
+            destination, task_info)
+        return res
+
+    def revert(self, context, origin, destination, task_info, **kwargs):
+        super(BaseRunWorkerTask, self).revert(
+            context, origin, destination, task_info, **kwargs)
+        if not self._cleanup_task_runner_type:
+            LOG.debug(
+                "Task '%s' (main type '%s') had no cleanup task runner "
+                "associated with it. Skipping any reversion logic",
+                self._task_name, self._main_task_runner_type)
+            return
+
+        try:
+            res = self._execute_task(
+                context, self._task_id, self._cleanup_task_runner_type, origin,
+                destination, task_info)
+        except Exception as ex:
+            LOG.warn(
+                "Task cleanup for '%s' (main task type '%s', cleanup task type"
+                "'%s') has failed with the following trace: %s",
+                self._task_name, self._main_task_runner_type,
+                self._cleanup_task_runner_type, utils.get_exception_details())
+            if self._raise_on_cleanup_failure:
+                raise
+            return
+
+        LOG.debug(
+            "Reversion of taskflow task '%s' (ID '%s') was successfully "
+            "executed using task runner '%s' with the following result: %s" % (
+                self._task_name, self._task_id, self._cleanup_task_runner_type,
+                res))

coriolis/taskflow/runner.py

@@ -0,0 +1,141 @@
+# Copyright 2020 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+# NOTE: we neeed to make sure eventlet is imported:
+import multiprocessing
+import sys
+from logging import handlers
+
+import eventlet  #noqa
+from oslo_config import cfg
+from oslo_log import log as logging
+from six.moves import queue
+from taskflow import engines
+from taskflow.types import notifier
+
+from coriolis import utils
+
+
+LOG = logging.getLogger(__name__)
+
+TASKFLOW_EXECUTION_ORDER_PARALLEL = 'parallel'
+TASKFLOW_EXECUTION_ORDER_SERIAL = 'serial'
+
+TASKFLOW_EXECUTOR_THREADED = "threaded"
+TASKFLOW_EXECUTOR_PROCESSES = "processes"
+TASKFLOW_EXECUTOR_GREENTHREADED = "greenthreaded"
+
+
+class TaskFlowRunner(object):
+
+    def __init__(
+            self, service_name,
+            execution_order=TASKFLOW_EXECUTION_ORDER_PARALLEL,
+            executor=TASKFLOW_EXECUTOR_THREADED,
+            max_workers=1):
+
+        self._service_name = service_name
+        self._execution_order = execution_order
+        self._executor = executor
+        self._max_workers = max_workers
+
+    def _log_flow_transition(self, state, details):
+        LOG.debug(
+            "[TaskFlowRunner(%s)] Flow '%s' (internal UUID '%s') transitioned"
+            " from '%s' state to '%s'",
+            self._service_name, details['flow_name'], details['flow_uuid'],
+            details['old_state'], state)
+
+    def _log_task_transition(self, state, details):
+        LOG.debug(
+            "[TaskFlowRunner(%s)] Task '%s' (internal UUID '%s') transitioned"
+            " from '%s' state to '%s'",
+            self._service_name, details['task_name'], details['task_uuid'],
+            details['old_state'], state)
+
+    def _setup_engine_for_flow(self, flow, store=None):
+        engine = engines.load(
+            flow, store, executor=self._executor,
+            engine=self._execution_order, max_workers=self._max_workers)
+        engine.notifier.register(
+            notifier.Notifier.ANY, self._log_flow_transition)
+        engine.atom_notifier.register(
+            notifier.Notifier.ANY, self._log_task_transition)
+        return engine
+
+    def _run_flow(self, flow, store=None):
+        LOG.debug("Ramping up to run flow with name '%s'", flow.name)
+        engine = self._setup_engine_for_flow(flow, store=store)
+
+        LOG.debug("Attempting to compile flow with name '%s'", flow.name)
+        engine.compile()
+
+        LOG.debug("Preparing to run flow with name '%s'", flow.name)
+        engine.prepare()
+
+        LOG.debug("Running flow with name '%s'", flow.name)
+        try:
+            engine.run()
+        except Exception as ex:
+            LOG.warn(
+                "Fatal error occurred while attempting to run flow '%s'. "
+                "Full trace was: %s", flow.name, utils.get_exception_details())
+            raise
+        LOG.info(
+            "Successfully ran flow with name '%s'. Statistics were: %s",
+            flow.name, engine.statistics)
+
+    def run_flow(self, flow, store=None):
+        self._run_flow(flow, store=store)
+
+    def _setup_task_process_logging(self, mp_log_q):
+        # Setting up logging and cfg, needed since this is a new process
+        cfg.CONF(sys.argv[1:], project='coriolis', version="1.0.0")
+        utils.setup_logging()
+
+        # Log events need to be handled in the parent process
+        log_root = logging.getLogger(None).logger
+        for handler in log_root.handlers:
+            log_root.removeHandler(handler)
+        log_root.addHandler(handlers.QueueHandler(mp_log_q))
+
+    def _run_flow_in_process(self, flow, mp_log_queue, store=None):
+        self._setup_task_process_logging(mp_log_queue)
+        self._run_flow(flow, store=store)
+
+    def _handle_mp_log_events(self, p, mp_log_q):
+        while True:
+            try:
+                record = mp_log_q.get(timeout=1)
+                if record is None:
+                    break
+                logger = logging.getLogger(record.name).logger
+                logger.handle(record)
+            except queue.Empty:
+                if not p.is_alive():
+                    break
+
+    def _spawn_process_flow(self, flow, store=None):
+        mp_ctx = multiprocessing.get_context('spawn')
+        mp_log_q = mp_ctx.Queue()
+        process = mp_ctx.Process(
+            target=self._run_flow_in_process,
+            args=(flow, mp_log_q, store))
+        LOG.debug("Starting new background process for flow '%s'", flow.name)
+        process.start()
+        LOG.debug(
+            "Sucessfully started background process for flow '%s' with "
+            "PID: '%d'", flow.name, process.pid)
+        eventlet.spawn(self._handle_mp_log_events, process, mp_log_q)
+
+    def run_flow_in_background(self, flow, store=None):
+        """ Starts the given flow in the background in a separate process.
+        Does NOT return/store any result.
+
+        All tasks must be "self-sufficient" and record their own results in
+        some fashion or another.
+        Care should be taken that any fields/attributes within the tasks
+        are thread/fork-safe.
+        The 'store' inputs should also only contain be thread-safe datatypes.
+        """
+        self._spawn_process_flow(flow, store=store)

coriolis/taskflow/utils.py

@@ -0,0 +1,20 @@
+# Copyright 2020 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from oslo_log import log as logging
+
+
+LOG = logging.getLogger(__name__)
+
+
+class DummyDecider(object):
+    """ A callable to decide execution in a pre-defined manner. """
+
+    def __init__(self, allow=True):
+        self._allow = allow
+
+    def __call__(self, history):
+        LOG.debug(
+            "Dummy decider returning '%s'. Provided task history was: %s",
+            self._allow, history)
+        return self._allow

coriolis/tasks/factory.py

@@ -128,7 +128,19 @@ _TASKS_MAP = {
     constants.TASK_TYPE_RELEASE_OSMORPHING_MINION:
         minion_pool_tasks.ReleaseOSMorphingMinionTask,
     constants.TASK_TYPE_COLLECT_OSMORPHING_INFO:
-        minion_pool_tasks.CollectOSMorphingInfoTask
+        minion_pool_tasks.CollectOSMorphingInfoTask,
+    constants.TASK_TYPE_HEALTHCHECK_SOURCE_MINION:
+        minion_pool_tasks.HealthcheckSourceMinionMachineTask,
+    constants.TASK_TYPE_HEALTHCHECK_DESTINATION_MINION:
+        minion_pool_tasks.HealthcheckDestinationMinionTask,
+    constants.TASK_TYPE_POWER_ON_SOURCE_MINION:
+        minion_pool_tasks.PowerOnSourceMinionTask,
+    constants.TASK_TYPE_POWER_OFF_SOURCE_MINION:
+        minion_pool_tasks.PowerOffSourceMinionTask,
+    constants.TASK_TYPE_POWER_ON_DESTINATION_MINION:
+        minion_pool_tasks.PowerOnDestinationMinionTask,
+    constants.TASK_TYPE_POWER_OFF_DESTINATION_MINION:
+        minion_pool_tasks.PowerOffDestinationMinionTask
 }
 
 

coriolis/tasks/minion_pool_tasks.py

@@ -14,18 +14,13 @@ LOG = logging.getLogger(__name__)
 
 
 SOURCE_MINION_TASK_INFO_FIELD_MAPPINGS = {
-    # NOTE: these redundancies are in place so as to have the
-    # 'Release*' task classes clear these fields after they run:
-    "source_minion_machine_id": "source_minion_machine_id",
-    "source_minion_provider_properties": "source_resources",
-    "source_minion_connection_info": "source_resources_connection_info"}
+    "origin_minion_provider_properties": "source_resources",
+    "origin_minion_connection_info": "source_resources_connection_info"}
 TARGET_MINION_TASK_INFO_FIELD_MAPPINGS = {
-    "target_minion_machine_id": "target_minion_machine_id",
-    "target_minion_provider_properties": "target_resources",
-    "target_minion_backup_writer_connection_info": (
+    "destination_minion_provider_properties": "target_resources",
+    "destination_minion_backup_writer_connection_info": (
         "target_resources_connection_info")}
 OSMOPRHING_MINION_TASK_INFO_FIELD_MAPPINGS = {
-    "osmorphing_minion_machine_id": "osmorphing_minion_machine_id",
     "osmorphing_minion_provider_properties": "os_morphing_resources",
     "osmorphing_minion_connection_info": "osmorphing_connection_info"}
 
@@ -477,7 +472,7 @@ class AttachVolumesToSourceMinionTask(_BaseAttachVolumesToTransferMinionTask):
 
     @classmethod
     def _get_minion_properties_task_info_field(cls):
-        return "source_minion_provider_properties"
+        return "origin_minion_provider_properties"
 
     @classmethod
     def get_volumes_info_from_task_info(cls, task_info):
@@ -508,7 +503,7 @@ class AttachVolumesToDestinationMinionTask(
 
     @classmethod
     def _get_minion_properties_task_info_field(cls):
-        return "target_minion_provider_properties"
+        return "destination_minion_provider_properties"
 
     @classmethod
     def _get_provider_disk_operation(cls, provider):
@@ -701,7 +696,7 @@ class ValidateSourceMinionCompatibilityTask(
 
     @classmethod
     def _get_minion_properties_task_info_field(cls):
-        return "source_minion_provider_properties"
+        return "origin_minion_provider_properties"
 
     @classmethod
     def _get_provider_pool_validation_operation(cls, provider):
@@ -721,7 +716,7 @@ class ValidateDestinationMinionCompatibilityTask(
 
     @classmethod
     def _get_minion_properties_task_info_field(cls):
-        return "target_minion_provider_properties"
+        return "destination_minion_provider_properties"
 
     @classmethod
     def _get_provider_pool_validation_operation(cls, provider):
@@ -865,3 +860,161 @@ class CollectOSMorphingInfoTask(base.TaskRunner):
 
         return {
             "osmorphing_info": result["osmorphing_info"]}
+
+
+class _BaseHealthcheckMinionMachineTask(base.TaskRunner):
+    """ Calls into the provider to healthcheck the minion machine. """
+
+    @classmethod
+    def get_required_platform(cls):
+        raise NotImplementedError(
+            "No minion healthcheck platform specified")
+
+    @classmethod
+    def get_required_task_info_properties(cls):
+        return ["minion_provider_properties", "minion_connection_info"]
+
+    @classmethod
+    def get_returned_task_info_properties(cls):
+        return []
+
+    @classmethod
+    def get_required_provider_types(cls):
+        return _get_required_minion_pool_provider_types_for_platform(
+            cls.get_required_platform())
+
+    def _run(self, ctxt, instance, origin, destination,
+             task_info, event_handler):
+
+        platform_to_target = None
+        required_platform = self.get_required_platform()
+        if required_platform == constants.TASK_PLATFORM_SOURCE:
+            platform_to_target = origin
+        elif required_platform == constants.TASK_PLATFORM_DESTINATION:
+            platform_to_target = destination
+        else:
+            raise NotImplementedError(
+                "Unknown minion healthcheck platform '%s'" % (
+                    required_platform))
+
+        connection_info = base.get_connection_info(ctxt, platform_to_target)
+        provider_type = self.get_required_provider_types()[
+            self.get_required_platform()][0]
+        provider = providers_factory.get_provider(
+            platform_to_target["type"], provider_type, event_handler)
+
+        minion_properties = task_info['minion_provider_properties']
+        minion_connection_info = base.unmarshal_migr_conn_info(
+            task_info['minion_connection_info'])
+
+        provider.healthcheck_minion(
+            ctxt, connection_info, minion_properties, minion_connection_info)
+
+        return {}
+
+
+class HealthcheckSourceMinionMachineTask(_BaseHealthcheckMinionMachineTask):
+
+    @classmethod
+    def get_required_platform(cls):
+        return constants.TASK_PLATFORM_SOURCE
+
+
+class HealthcheckDestinationMinionTask(_BaseHealthcheckMinionMachineTask):
+
+    @classmethod
+    def get_required_platform(cls):
+        return constants.TASK_PLATFORM_DESTINATION
+
+
+class _BasePowerCycleMinionTask(base.TaskRunner):
+
+    @classmethod
+    def get_required_platform(cls):
+        raise NotImplementedError(
+            "No minion power cycle platform specified")
+
+    @classmethod
+    def get_required_provider_types(cls):
+        return _get_required_minion_pool_provider_types_for_platform(
+            cls.get_required_platform())
+
+    @classmethod
+    def get_required_task_info_properties(cls):
+        return ["minion_provider_properties"]
+
+    @classmethod
+    def get_returned_task_info_properties(cls):
+        return []
+
+    @classmethod
+    def _get_minion_power_cycle_op(cls, provider):
+        raise NotImplementedError(
+            "No minion power cycle operation implemented.")
+
+    def _run(self, ctxt, instance, origin, destination,
+             task_info, event_handler):
+
+        platform_to_target = None
+        required_platform = self.get_required_platform()
+        if required_platform == constants.TASK_PLATFORM_SOURCE:
+            platform_to_target = origin
+        elif required_platform == constants.TASK_PLATFORM_DESTINATION:
+            platform_to_target = destination
+        else:
+            raise NotImplementedError(
+                "Unknown minion healthcheck platform '%s'" % (
+                    required_platform))
+
+        connection_info = base.get_connection_info(ctxt, platform_to_target)
+        provider_type = self.get_required_provider_types()[
+            self.get_required_platform()][0]
+        provider = providers_factory.get_provider(
+            platform_to_target["type"], provider_type, event_handler)
+        power_cycle_op = self._get_minion_power_cycle_op(provider)
+        minion_properties = task_info['minion_provider_properties']
+        power_cycle_op(ctxt, connection_info, minion_properties)
+
+        return {}
+
+
+class _BasePowerOnMinionTask(_BasePowerCycleMinionTask):
+
+    @classmethod
+    def _get_minion_power_cycle_op(cls, provider):
+        return provider.start_minion
+
+
+class PowerOnSourceMinionTask(_BasePowerOnMinionTask):
+
+    @classmethod
+    def get_required_platform(cls):
+        return constants.TASK_PLATFORM_SOURCE
+
+
+class PowerOnDestinationMinionTask(_BasePowerOnMinionTask):
+
+    @classmethod
+    def get_required_platform(cls):
+        return constants.TASK_PLATFORM_DESTINATION
+
+
+class _BasePowerOffMinionTask(_BasePowerCycleMinionTask):
+
+    @classmethod
+    def _get_minion_power_cycle_op(cls, provider):
+        return provider.shutdown_minion
+
+
+class PowerOffSourceMinionTask(_BasePowerOffMinionTask):
+
+    @classmethod
+    def get_required_platform(cls):
+        return constants.TASK_PLATFORM_SOURCE
+
+
+class PowerOffDestinationMinionTask(_BasePowerOffMinionTask):
+
+    @classmethod
+    def get_required_platform(cls):
+        return constants.TASK_PLATFORM_DESTINATION

coriolis/utils.py

@@ -11,7 +11,6 @@ import io
 import json
 import os
 import pickle
-import platform
 import re
 import socket
 import string
@@ -20,8 +19,6 @@ import sys
 import time
 import traceback
 import uuid
-import __main__ as main
-
 from io import StringIO
 
 import OpenSSL
@@ -29,6 +26,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
 
+import __main__ as main
 import netifaces
 import paramiko
 # NOTE(gsamfira): I am aware that this is not ideal, but pip

coriolis/worker/rpc/client.py

@@ -34,12 +34,35 @@ class WorkerClient(rpc.BaseRPCClient):
         super(WorkerClient, self).__init__(
             target, timeout=timeout)
 
+    @classmethod
+    def from_service_definition(
+            cls, service, timeout=None, topic_override=None):
+        if service.get('topic') != constants.WORKER_MAIN_MESSAGING_TOPIC:
+            raise ValueError(
+                "Unknown topic '%s' for worker service client. Only "
+                "acceptable value is '%s': %s" % (
+                    service.get('topic'),
+                    constants.WORKER_MAIN_MESSAGING_TOPIC,
+                    service))
+        topic = constants.WORKER_MAIN_MESSAGING_TOPIC
+        if topic_override:
+            topic = topic_override
+        return cls(
+            timeout=timeout, base_worker_topic=topic, host=service.get('host'))
+
     def begin_task(self, ctxt, task_id, task_type, origin, destination,
                    instance, task_info):
         self._cast(
             ctxt, 'exec_task', task_id=task_id, task_type=task_type,
             origin=origin, destination=destination, instance=instance,
-            task_info=task_info)
+            task_info=task_info, report_to_conductor=True)
+
+    def run_task(self, ctxt, task_id, task_type, origin, destination,
+                 instance, task_info):
+        return self._call(
+            ctxt, 'exec_task', task_id=task_id, task_type=task_type,
+            origin=origin, destination=destination, instance=instance,
+            task_info=task_info, report_to_conductor=False)
 
     def cancel_task(self, ctxt, task_id, process_id, force):
         return self._call(

coriolis/worker/rpc/server.py

@@ -21,6 +21,7 @@ from coriolis import constants
 from coriolis import context
 from coriolis import events
 from coriolis import exception
+from coriolis.minion_manager.rpc import client as rpc_minion_manager_client
 from coriolis.providers import factory as providers_factory
 from coriolis import schemas
 from coriolis import service
@@ -36,46 +37,21 @@ LOG = logging.getLogger(__name__)
 VERSION = "1.0"
 
 
-class _ConductorProviderEventHandler(events.BaseEventHandler):
-    def __init__(self, ctxt, task_id):
-        self._ctxt = ctxt
-        self._task_id = task_id
-        self._rpc_conductor_client = rpc_conductor_client.ConductorClient()
-
-    def add_task_progress_update(self, total_steps, message):
-        LOG.info("Progress update: %s", message)
-        self._rpc_conductor_client.add_task_progress_update(
-            self._ctxt, self._task_id, total_steps, message)
-
-    def update_task_progress_update(self, step, total_steps, message):
-        LOG.info("Progress update: %s", message)
-        self._rpc_conductor_client.update_task_progress_update(
-            self._ctxt, self._task_id, step, total_steps, message)
-
-    def get_task_progress_step(self):
-        return self._rpc_conductor_client.get_task_progress_step(
-            self._ctxt, self._task_id)
-
-    def info(self, message):
-        LOG.info(message)
-        self._rpc_conductor_client.task_event(
-            self._ctxt, self._task_id, constants.TASK_EVENT_INFO, message)
-
-    def warn(self, message):
-        LOG.warn(message)
-        self._rpc_conductor_client.task_event(
-            self._ctxt, self._task_id, constants.TASK_EVENT_WARNING, message)
-
-    def error(self, message):
-        LOG.error(message)
-        self._rpc_conductor_client.task_event(
-            self._ctxt, self._task_id, constants.TASK_EVENT_ERROR, message)
+# TODO(aznashwan): parametrize the event handler provided during task execution
+# to decouple what gets notified from the task running logic itself:
+def _get_event_handler_for_task_type(task_type, ctxt, task_object_id):
+    if task_type in constants.MINION_POOL_OPERATIONS_TASKS:
+        return rpc_minion_manager_client.MinionManagerPoolRpcEventHandler(
+            ctxt, task_object_id)
+    return rpc_conductor_client.ConductorTaskRpcEventHandler(
+        ctxt, task_object_id)
 
 
 class WorkerServerEndpoint(object):
     def __init__(self):
         self._server = utils.get_hostname()
         self._service_registration = self._register_worker_service()
+        self._rpc_conductor_client_instance = None
 
     @property
     def _rpc_conductor_client(self):
@@ -84,7 +60,10 @@ class WorkerServerEndpoint(object):
         # be invalidated. Considering this class both serves from a "main
         # process" as well as forking child processes, it is safest to
         # re-instantiate the client every time:
-        return rpc_conductor_client.ConductorClient()
+        if self._rpc_conductor_client_instance is None:
+            self._rpc_conductor_client_instance = (
+                rpc_conductor_client.ConductorClient())
+        return self._rpc_conductor_client_instance
 
     def _register_worker_service(self):
         host = utils.get_hostname()
@@ -97,8 +76,10 @@ class WorkerServerEndpoint(object):
         status = self.get_service_status(dummy_context)
         service_registration = (
             conductor_rpc_utils.check_create_registration_for_service(
-                self._rpc_conductor_client, dummy_context, host, binary,
-                constants.WORKER_MAIN_MESSAGING_TOPIC, enabled=True,
+                # NOTE: considering this only runs once on startup, we
+                # instantiate a fresh conductor client instance for it:
+                rpc_conductor_client.ConductorClient(), dummy_context, host,
+                binary, constants.WORKER_MAIN_MESSAGING_TOPIC, enabled=True,
                 providers=status['providers'], specs=status['specs']))
         LOG.info(
             "Worker service is successfully registered with the following "
@@ -151,6 +132,8 @@ class WorkerServerEndpoint(object):
                 "completed/error'd." % (
                     process_id, task_id))
             LOG.error(msg)
+            self._rpc_conductor_client.confirm_task_cancellation(
+                ctxt, task_id, msg)
 
     def _handle_mp_log_events(self, p, mp_log_q):
         while True:
@@ -198,7 +181,8 @@ class WorkerServerEndpoint(object):
         """ Returns a list of strings with paths on the worker with shared
         libraries needed by the source/destination providers.
         """
-        event_handler = _ConductorProviderEventHandler(ctxt, task_id)
+        event_handler = _get_event_handler_for_task_type(
+            task_type, ctxt, task_id)
         task_runner = task_runners_factory.get_task_runner_class(
             task_type)()
 
@@ -224,7 +208,7 @@ class WorkerServerEndpoint(object):
         return result
 
     def _exec_task_process(self, ctxt, task_id, task_type, origin, destination,
-                           instance, task_info):
+                           instance, task_info, report_to_conductor=True):
         mp_ctx = multiprocessing.get_context('spawn')
         mp_q = mp_ctx.Queue()
         mp_log_q = mp_ctx.Queue()
@@ -237,24 +221,26 @@ class WorkerServerEndpoint(object):
             ctxt, task_id, task_type, origin, destination)
 
         try:
-            LOG.debug(
-                "Attempting to set task host on Conductor for task '%s'.",
-                task_id)
-            self._rpc_conductor_client.set_task_host(
-                ctxt, task_id, self._server)
+            if report_to_conductor:
+                LOG.debug(
+                    "Attempting to set task host on Conductor for task '%s'.",
+                    task_id)
+                self._rpc_conductor_client.set_task_host(
+                    ctxt, task_id, self._server)
             LOG.debug(
                 "Attempting to start process for task with ID '%s'", task_id)
             self._start_process_with_custom_library_paths(
                 p, extra_library_paths)
             LOG.info("Task process started: %s", task_id)
+            if report_to_conductor:
+                LOG.debug(
+                    "Attempting to set task process on Conductor for task '%s'.",
+                    task_id)
+                self._rpc_conductor_client.set_task_process(
+                    ctxt, task_id, p.pid)
             LOG.debug(
-                "Attempting to set task process on Conductor for task '%s'.",
-                task_id)
-            self._rpc_conductor_client.set_task_process(
-                ctxt, task_id, p.pid)
-            LOG.debug(
-                "Successfully started and retported task process for task "
-                "with ID '%s'.", task_id)
+                "Successfully started and reported task process for task "
+                "with ID '%s' (PID %d)", task_id, p.pid)
         except (Exception, KeyboardInterrupt) as ex:
             LOG.debug(
                 "Exception occurred whilst setting host for task '%s'. Error "
@@ -290,39 +276,51 @@ class WorkerServerEndpoint(object):
         return result
 
     def exec_task(self, ctxt, task_id, task_type, origin, destination,
-                  instance, task_info):
+                  instance, task_info, report_to_conductor=True):
         try:
             task_result = self._exec_task_process(
                 ctxt, task_id, task_type, origin, destination,
-                instance, task_info)
+                instance, task_info, report_to_conductor=report_to_conductor)
 
             LOG.info(
                 "Output of completed %s task with ID %s: %s",
                 task_type, task_id,
                 utils.sanitize_task_info(task_result))
 
+            if not report_to_conductor:
+                return task_result
             self._rpc_conductor_client.task_completed(
                 ctxt, task_id, task_result)
         except exception.TaskProcessCanceledException as ex:
-            LOG.debug(
-                "Task with ID '%s' appears to have been cancelled. "
-                "Confirming cancellation to Conductor now. Error was: %s",
-                task_id, utils.get_exception_details())
-            LOG.exception(ex)
-            self._rpc_conductor_client.confirm_task_cancellation(
-                ctxt, task_id, str(ex))
+            if report_to_conductor:
+                LOG.debug(
+                    "Task with ID '%s' appears to have been cancelled. "
+                    "Confirming cancellation to Conductor now. Error was: %s",
+                    task_id, utils.get_exception_details())
+                LOG.exception(ex)
+                self._rpc_conductor_client.confirm_task_cancellation(
+                    ctxt, task_id, str(ex))
+            else:
+                raise
         except exception.NoSuitableWorkerServiceError as ex:
-            LOG.warn(
-                "A conductor-side scheduling error has occurred following the "
-                "completion of task '%s'. Ignoring. Error was: %s",
-                task_id, utils.get_exception_details())
+            if report_to_conductor:
+                LOG.warn(
+                    "A conductor-side scheduling error has occurred following "
+                    "the completion of task '%s'. Ignoring. Error was: %s",
+                    task_id, utils.get_exception_details())
+            else:
+                raise
         except Exception as ex:
-            LOG.debug(
-                "Task with ID '%s' has error'd out. Reporting error to "
-                "Conductor now. Error was: %s",
-                task_id, utils.get_exception_details())
-            LOG.exception(ex)
-            self._rpc_conductor_client.set_task_error(ctxt, task_id, str(ex))
+            if report_to_conductor:
+                LOG.debug(
+                    "Task with ID '%s' has error'd out. Reporting error to "
+                    "Conductor now. Error was: %s",
+                    task_id, utils.get_exception_details())
+                LOG.exception(ex)
+                self._rpc_conductor_client.set_task_error(
+                        ctxt, task_id, str(ex))
+            else:
+                raise
 
     def get_endpoint_instances(self, ctxt, platform_name, connection_info,
                                source_environment, marker, limit,
@@ -643,7 +641,8 @@ def _task_process(ctxt, task_id, task_type, origin, destination, instance,
 
         task_runner = task_runners_factory.get_task_runner_class(
             task_type)()
-        event_handler = _ConductorProviderEventHandler(ctxt, task_id)
+        event_handler = _get_event_handler_for_task_type(
+            task_type, ctxt, task_id)
 
         LOG.debug("Executing task: %(task_id)s, type: %(task_type)s, "
                   "origin: %(origin)s, destination: %(destination)s, "

coriolis/wsman.py

@@ -48,6 +48,41 @@ class WSManConnection(object):
             cert_pem=cert_pem,
             cert_key_pem=cert_key_pem)
 
+    @classmethod
+    def from_connection_info(cls, connection_info):
+        """ Returns a wsman.WSManConnection object for the provided conn info. """
+        if not isinstance(connection_info, dict):
+            raise ValueError(
+                "WSMan connection must be a dict. Got type '%s', value: %s" % (
+                    type(connection_info), connection_info))
+
+        required_keys = ["ip", "username", "password"]
+        missing = [key for key in required_keys if key not in connection_info]
+        if missing:
+            raise ValueError(
+                "The following keys were missing from WSMan connection info %s. "
+                "Got: %s" % (missing, connection_info))
+
+        host = connection_info["ip"]
+        port = connection_info.get("port", 5986)
+        username = connection_info["username"]
+        password = connection_info.get("password")
+        cert_pem = connection_info.get("cert_pem")
+        cert_key_pem = connection_info.get("cert_key_pem")
+        url = "https://%s:%s/wsman" % (host, port)
+
+        LOG.info("Connection info: %s", str(connection_info))
+
+        LOG.info("Waiting for connectivity on host: %(host)s:%(port)s",
+                 {"host": host, "port": port})
+        utils.wait_for_port_connectivity(host, port)
+
+        conn = cls()
+        conn.connect(url=url, username=username, password=password,
+                     cert_pem=cert_pem, cert_key_pem=cert_key_pem)
+
+        return conn
+
     def disconnect(self):
         self._protocol = None
 

requirements.txt

@@ -34,6 +34,7 @@ mysqlclient
 schedule
 strict-rfc3339
 sqlalchemy
+taskflow
 webob
 sshtunnel
-requests-unixsocket
+requests-unixsocket

setup.cfg

@@ -30,6 +30,7 @@ console_scripts =
     coriolis-worker = coriolis.cmd.worker:main
     coriolis-replica-cron = coriolis.cmd.replica_cron:main
     coriolis-scheduler= coriolis.cmd.scheduler:main
+    coriolis-minion-manager= coriolis.cmd.minion_manager:main
     coriolis-dbsync = coriolis.cmd.db_sync:main
 
 [wheel]