Merged in alexpilotti/coriolis/replica (pull request #15)

Adds Replica

.gitignore

@@ -37,3 +37,4 @@ nosetests.xml
 nova/tests/cover/*
 nova/vcsversion.py
 tools/conf/nova.conf*
+resources/write_data

coriolis/api/v1/migration_actions.py

@@ -16,7 +16,10 @@ class MigrationActionsController(api_wsgi.Controller):
     @api_wsgi.action('cancel')
     def _cancel(self, req, id, body):
         try:
-            self._migration_api.cancel(req.environ['coriolis.context'], id)
+            force = (body["cancel"] or {}).get("force", False)
+
+            self._migration_api.cancel(
+                req.environ['coriolis.context'], id, force)
             raise exc.HTTPNoContent()
         except exception.NotFound as ex:
             raise exc.HTTPNotFound(explanation=ex.msg)

coriolis/api/v1/migrations.py

@@ -3,6 +3,8 @@
 
 from webob import exc
 
+from oslo_log import log as logging
+
 from coriolis.api import wsgi as api_wsgi
 from coriolis.api.v1.views import migration_view
 from coriolis import constants
@@ -10,6 +12,8 @@ from coriolis import exception
 from coriolis.migrations import api
 from coriolis.providers import factory
 
+LOG = logging.getLogger(__name__)
+
 
 class MigrationController(api_wsgi.Controller):
     def __init__(self):
@@ -34,36 +38,52 @@ class MigrationController(api_wsgi.Controller):
             req, self._migration_api.get_migrations(
                 req.environ['coriolis.context'], include_tasks=True))
 
-    def _validate_create_body(self, body):
-        migration = body["migration"]
-
-        origin = migration["origin"]
-        destination = migration["destination"]
-
-        export_provider = factory.get_provider(
-            origin["type"], constants.PROVIDER_TYPE_EXPORT, None)
-        if not export_provider.validate_connection_info(
-                origin.get("connection_info", {})):
-            # TODO: use a decent exception
-            raise exception.CoriolisException("Invalid connection info")
-
-        import_provider = factory.get_provider(
-            destination["type"], constants.PROVIDER_TYPE_IMPORT, None)
-        if not import_provider.validate_connection_info(
-                destination.get("connection_info", {})):
-            # TODO: use a decent exception
-            raise exception.CoriolisException("Invalid connection info")
-
-        if not import_provider.validate_target_environment(
-                destination.get("target_environment", {})):
-            raise exception.CoriolisException("Invalid target environment")
-
-        return origin, destination, migration["instances"]
+    def _validate_migration_input(self, migration):
+        try:
+            origin = migration["origin"]
+            destination = migration["destination"]
+
+            export_provider = factory.get_provider(
+                origin["type"], constants.PROVIDER_TYPE_EXPORT, None)
+            export_provider.validate_connection_info(
+                origin.get("connection_info", {}))
+
+            import_provider = factory.get_provider(
+                destination["type"], constants.PROVIDER_TYPE_IMPORT, None)
+            import_provider.validate_connection_info(
+                destination.get("connection_info", {}))
+
+            import_provider.validate_target_environment(
+                destination.get("target_environment", {}))
+
+            return origin, destination, migration["instances"]
+        except Exception as ex:
+            LOG.exception(ex)
+            if hasattr(ex, "message"):
+                msg = ex.message
+            else:
+                msg = str(ex)
+            raise exception.InvalidInput(msg)
 
     def create(self, req, body):
-        origin, destination, instances = self._validate_create_body(body)
-        return migration_view.single(req, self._migration_api.start(
-            req.environ['coriolis.context'], origin, destination, instances))
+        # TODO: validate body
+        migration_body = body.get("migration", {})
+        context = req.environ['coriolis.context']
+
+        replica_id = migration_body.get("replica_id")
+        if replica_id:
+            clone_disks = migration_body.get("clone_disks", True)
+            force = migration_body.get("force", False)
+
+            migration = self._migration_api.deploy_replica_instances(
+                context, replica_id, clone_disks, force)
+        else:
+            origin, destination, instances = self._validate_migration_input(
+                migration_body)
+            migration = self._migration_api.migrate_instances(
+                context, origin, destination, instances)
+
+        return migration_view.single(req, migration)
 
     def delete(self, req, id):
         try:

coriolis/api/v1/replica_actions.py

@@ -0,0 +1,30 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from webob import exc
+
+from coriolis.api.v1.views import replica_tasks_execution_view
+from coriolis.api import wsgi as api_wsgi
+from coriolis import exception
+from coriolis.replicas import api
+
+
+class ReplicaActionsController(api_wsgi.Controller):
+    def __init__(self):
+        self._replica_api = api.API()
+        super(ReplicaActionsController, self).__init__()
+
+    @api_wsgi.action('delete-disks')
+    def _delete_disks(self, req, id, body):
+        try:
+            return replica_tasks_execution_view.single(
+                req, self._replica_api.delete_disks(
+                    req.environ['coriolis.context'], id))
+        except exception.NotFound as ex:
+            raise exc.HTTPNotFound(explanation=ex.msg)
+        except exception.InvalidParameterValue as ex:
+            raise exc.HTTPNotFound(explanation=ex.msg)
+
+
+def create_resource():
+    return api_wsgi.Resource(ReplicaActionsController())

coriolis/api/v1/replica_tasks_execution_actions.py

@@ -0,0 +1,31 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from webob import exc
+
+from coriolis.api import wsgi as api_wsgi
+from coriolis import exception
+from coriolis.replica_tasks_executions import api
+
+
+class ReplicaTasksExecutionActionsController(api_wsgi.Controller):
+    def __init__(self):
+        self._replica_tasks_execution_api = api.API()
+        super(ReplicaTasksExecutionActionsController, self).__init__()
+
+    @api_wsgi.action('cancel')
+    def _cancel(self, req, replica_id, id, body):
+        try:
+            force = (body["cancel"] or {}).get("force", False)
+
+            self._replica_tasks_execution_api.cancel(
+                req.environ['coriolis.context'], id, force)
+            raise exc.HTTPNoContent()
+        except exception.NotFound as ex:
+            raise exc.HTTPNotFound(explanation=ex.msg)
+        except exception.InvalidParameterValue as ex:
+            raise exc.HTTPNotFound(explanation=ex.msg)
+
+
+def create_resource():
+    return api_wsgi.Resource(ReplicaTasksExecutionActionsController())

coriolis/api/v1/replica_tasks_executions.py

@@ -0,0 +1,58 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from webob import exc
+
+from coriolis.api import wsgi as api_wsgi
+from coriolis.api.v1.views import replica_tasks_execution_view
+from coriolis import exception
+from coriolis.replica_tasks_executions import api
+
+
+class ReplicaTasksExecutionController(api_wsgi.Controller):
+    def __init__(self):
+        self._replica_tasks_execution_api = api.API()
+        super(ReplicaTasksExecutionController, self).__init__()
+
+    def show(self, req, replica_id, id):
+        execution = self._replica_tasks_execution_api.get_execution(
+            req.environ["coriolis.context"], id)
+        if not execution:
+            raise exc.HTTPNotFound()
+
+        return replica_tasks_execution_view.single(req, execution)
+
+    def index(self, req, replica_id):
+        return replica_tasks_execution_view.collection(
+            req, self._replica_tasks_execution_api.get_executions(
+                req.environ['coriolis.context'], replica_id,
+                include_tasks=False))
+
+    def detail(self, req, replica_id):
+        return replica_tasks_execution_view.collection(
+            req, self._replica_tasks_execution_api.get_executions(
+                req.environ['coriolis.context'], replica_id,
+                include_tasks=True))
+
+    def create(self, req, replica_id, body):
+        # TODO: validate body
+
+        execution_body = body.get("execution", {})
+        shutdown_instances = execution_body.get("shutdown_instances", False)
+
+        return replica_tasks_execution_view.single(
+            req, self._replica_tasks_execution_api.create(
+                req.environ['coriolis.context'], replica_id,
+                shutdown_instances))
+
+    def delete(self, req, replica_id, id):
+        try:
+            self._replica_tasks_execution_api.delete(
+                req.environ['coriolis.context'], id)
+            raise exc.HTTPNoContent()
+        except exception.NotFound as ex:
+            raise exc.HTTPNotFound(explanation=ex.msg)
+
+
+def create_resource():
+    return api_wsgi.Resource(ReplicaTasksExecutionController())

coriolis/api/v1/replicas.py

@@ -0,0 +1,86 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from webob import exc
+
+from oslo_log import log as logging
+
+from coriolis.api import wsgi as api_wsgi
+from coriolis.api.v1.views import replica_view
+from coriolis import constants
+from coriolis import exception
+from coriolis.replicas import api
+from coriolis.providers import factory
+
+LOG = logging.getLogger(__name__)
+
+
+class ReplicaController(api_wsgi.Controller):
+    def __init__(self):
+        self._replica_api = api.API()
+        super(ReplicaController, self).__init__()
+
+    def show(self, req, id):
+        replica = self._replica_api.get_replica(
+            req.environ["coriolis.context"], id)
+        if not replica:
+            raise exc.HTTPNotFound()
+
+        return replica_view.single(req, replica)
+
+    def index(self, req):
+        return replica_view.collection(
+            req, self._replica_api.get_replicas(
+                req.environ['coriolis.context'],
+                include_tasks_executions=False))
+
+    def detail(self, req):
+        return replica_view.collection(
+            req, self._replica_api.get_replicas(
+                req.environ['coriolis.context'],
+                include_tasks_executions=True))
+
+    def _validate_create_body(self, body):
+        try:
+            replica = body["replica"]
+
+            origin = replica["origin"]
+            destination = replica["destination"]
+
+            export_provider = factory.get_provider(
+                origin["type"], constants.PROVIDER_TYPE_EXPORT, None)
+            export_provider.validate_connection_info(
+                origin.get("connection_info", {}))
+
+            import_provider = factory.get_provider(
+                destination["type"], constants.PROVIDER_TYPE_IMPORT, None)
+            import_provider.validate_connection_info(
+                destination.get("connection_info", {}))
+
+            import_provider.validate_target_environment(
+                destination.get("target_environment", {}))
+
+            return origin, destination, replica["instances"]
+        except Exception as ex:
+            LOG.exception(ex)
+            if hasattr(ex, "message"):
+                msg = ex.message
+            else:
+                msg = str(ex)
+            raise exception.InvalidInput(msg)
+
+    def create(self, req, body):
+        origin, destination, instances = self._validate_create_body(body)
+        return replica_view.single(req, self._replica_api.create(
+            req.environ['coriolis.context'], origin, destination, instances))
+
+    def delete(self, req, id):
+        try:
+            self._replica_api.delete(req.environ['coriolis.context'], id)
+            raise exc.HTTPNoContent()
+        except exception.NotFound as ex:
+            raise exc.HTTPNotFound(explanation=ex.msg)
+
+
+def create_resource():
+    return api_wsgi.Resource(ReplicaController())

coriolis/api/v1/router.py

@@ -6,6 +6,10 @@ from oslo_log import log as logging
 from coriolis import api
 from coriolis.api.v1 import migrations
 from coriolis.api.v1 import migration_actions
+from coriolis.api.v1 import replica_actions
+from coriolis.api.v1 import replica_tasks_executions
+from coriolis.api.v1 import replica_tasks_execution_actions
+from coriolis.api.v1 import replicas
 
 LOG = logging.getLogger(__name__)
 
@@ -38,3 +42,37 @@ class APIRouter(api.APIRouter):
                        controller=self.resources['migration_actions'],
                        action='action',
                        conditions={'method': 'POST'})
+
+        self.resources['replicas'] = replicas.create_resource()
+        mapper.resource('replica', 'replicas',
+                        controller=self.resources['replicas'],
+                        collection={'detail': 'GET'},
+                        member={'action': 'POST'})
+
+        replica_actions_resource = replica_actions.create_resource()
+        self.resources['replica_actions'] = replica_actions_resource
+        migration_path = '/{project_id}/replicas/{id}'
+        mapper.connect('replica_actions',
+                       migration_path + '/actions',
+                       controller=self.resources['replica_actions'],
+                       action='action',
+                       conditions={'method': 'POST'})
+
+        self.resources['replica_tasks_executions'] = \
+            replica_tasks_executions.create_resource()
+        mapper.resource('execution', 'replicas/{replica_id}/executions',
+                        controller=self.resources['replica_tasks_executions'],
+                        collection={'detail': 'GET'},
+                        member={'action': 'POST'})
+
+        replica_tasks_execution_actions_resource = \
+            replica_tasks_execution_actions.create_resource()
+        self.resources['replica_tasks_execution_actions'] = \
+            replica_tasks_execution_actions_resource
+        migration_path = '/{project_id}/replicas/{replica_id}/executions/{id}'
+        mapper.connect('replica_tasks_execution_actions',
+                       migration_path + '/actions',
+                       controller=self.resources[
+                           'replica_tasks_execution_actions'],
+                       action='action',
+                       conditions={'method': 'POST'})

coriolis/api/v1/views/migration_view.py

@@ -3,6 +3,8 @@
 
 import itertools
 
+from coriolis.api.v1.views import replica_tasks_execution_view
+
 
 def _format_migration(req, migration, keys=None):
     def transform(key, value):
@@ -10,9 +12,20 @@ def _format_migration(req, migration, keys=None):
             return
         yield (key, value)
 
-    return dict(itertools.chain.from_iterable(
+    migration_dict = dict(itertools.chain.from_iterable(
         transform(k, v) for k, v in migration.items()))
 
+    # Migrations have a single tasks execution
+    execution = replica_tasks_execution_view.format_replica_tasks_execution(
+        req, migration_dict["executions"][0])
+
+    migration_dict["status"] = execution["status"]
+    tasks = execution.get("tasks")
+    if tasks:
+        migration_dict["tasks"] = tasks
+    del migration_dict["executions"]
+    return migration_dict
+
 
 def single(req, migration):
     return {"migration": _format_migration(req, migration)}

coriolis/api/v1/views/replica_tasks_execution_view.py

@@ -0,0 +1,45 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import itertools
+
+from coriolis import constants
+from coriolis import utils
+
+
+def _sort_tasks(tasks):
+    non_error_only_tasks = [t for t in tasks if
+                            t["depends_on"] or not t["on_error"]]
+    # Include error only tasks only if executed
+    error_only_tasks = [t for t in tasks if t["status"] !=
+                        constants.TASK_STATUS_ON_ERROR_ONLY and
+                        t not in non_error_only_tasks]
+
+    sorted_tasks = utils.topological_graph_sorting(
+        non_error_only_tasks, sort_key="task_type")
+    sorted_tasks += utils.topological_graph_sorting(
+        error_only_tasks, sort_key="task_type")
+    return sorted_tasks
+
+
+def format_replica_tasks_execution(req, execution, keys=None):
+    def transform(key, value):
+        if keys and key not in keys:
+            return
+        yield (key, value)
+
+    if "tasks" in execution:
+        execution["tasks"] = _sort_tasks(execution["tasks"])
+
+    return dict(itertools.chain.from_iterable(
+        transform(k, v) for k, v in execution.items()))
+
+
+def single(req, execution):
+    return {"execution": format_replica_tasks_execution(req, execution)}
+
+
+def collection(req, executions):
+    formatted_executions = [format_replica_tasks_execution(req, m)
+                            for m in executions]
+    return {'executions': formatted_executions}

coriolis/api/v1/views/replica_view.py

@@ -0,0 +1,24 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import itertools
+
+
+def _format_replica(req, replica, keys=None):
+    def transform(key, value):
+        if keys and key not in keys:
+            return
+        yield (key, value)
+
+    return dict(itertools.chain.from_iterable(
+        transform(k, v) for k, v in replica.items()))
+
+
+def single(req, replica):
+    return {"replica": _format_replica(req, replica)}
+
+
+def collection(req, replicas):
+    formatted_replicas = [_format_replica(req, m)
+                          for m in replicas]
+    return {'replicas': formatted_replicas}

coriolis/conductor/rpc/client.py

@@ -13,6 +13,56 @@ class ConductorClient(object):
         target = messaging.Target(topic='coriolis_conductor', version=VERSION)
         self._client = rpc.get_client(target)
 
+    def execute_replica_tasks(self, ctxt, replica_id,
+                              shutdown_instances=False):
+        return self._client.call(
+            ctxt, 'execute_replica_tasks', replica_id=replica_id,
+            shutdown_instances=shutdown_instances)
+
+    def get_replica_tasks_executions(self, ctxt, replica_id,
+                                     include_tasks=False):
+        return self._client.call(
+            ctxt, 'get_replica_tasks_executions',
+            replica_id=replica_id,
+            include_tasks=include_tasks)
+
+    def get_replica_tasks_execution(self, ctxt, execution_id):
+        return self._client.call(
+            ctxt, 'get_replica_tasks_execution',
+            execution_id=execution_id)
+
+    def delete_replica_tasks_execution(self, ctxt, execution_id):
+        return self._client.call(
+            ctxt, 'delete_replica_tasks_execution',
+            execution_id=execution_id)
+
+    def cancel_replica_tasks_execution(self, ctxt, execution_id, force):
+        return self._client.call(
+            ctxt, 'cancel_replica_tasks_execution',
+            execution_id=execution_id, force=force)
+
+    def create_instances_replica(self, ctxt, origin, destination, instances):
+        return self._client.call(
+            ctxt, 'create_instances_replica', origin=origin,
+            destination=destination, instances=instances)
+
+    def get_replicas(self, ctxt, include_tasks_executions=False):
+        return self._client.call(
+            ctxt, 'get_replicas',
+            include_tasks_executions=include_tasks_executions)
+
+    def get_replica(self, ctxt, replica_id):
+        return self._client.call(
+            ctxt, 'get_replica', replica_id=replica_id)
+
+    def delete_replica(self, ctxt, replica_id):
+        self._client.call(
+            ctxt, 'delete_replica', replica_id=replica_id)
+
+    def delete_replica_disks(self, ctxt, replica_id):
+        return self._client.call(
+            ctxt, 'delete_replica_disks', replica_id=replica_id)
+
     def get_migrations(self, ctxt, include_tasks=False):
         return self._client.call(ctxt, 'get_migrations',
                                  include_tasks=include_tasks)
@@ -21,18 +71,24 @@ class ConductorClient(object):
         return self._client.call(
             ctxt, 'get_migration', migration_id=migration_id)
 
-    def begin_migrate_instances(self, ctxt, origin, destination, instances):
+    def migrate_instances(self, ctxt, origin, destination, instances):
         return self._client.call(
             ctxt, 'migrate_instances', origin=origin, destination=destination,
             instances=instances)
 
+    def deploy_replica_instances(self, ctxt, replica_id, clone_disks=False,
+                                 force=False):
+        return self._client.call(
+            ctxt, 'deploy_replica_instances', replica_id=replica_id,
+            clone_disks=clone_disks, force=force)
+
     def delete_migration(self, ctxt, migration_id):
         self._client.call(
             ctxt, 'delete_migration', migration_id=migration_id)
 
-    def cancel_migration(self, ctxt, migration_id):
+    def cancel_migration(self, ctxt, migration_id, force):
         self._client.call(
-            ctxt, 'cancel_migration', migration_id=migration_id)
+            ctxt, 'cancel_migration', migration_id=migration_id, force=force)
 
     def set_task_host(self, ctxt, task_id, host, process_id):
         self._client.call(

coriolis/conductor/rpc/server.py

@@ -1,14 +1,17 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+import functools
 import uuid
 
+from oslo_concurrency import lockutils
 from oslo_log import log as logging
 
 from coriolis import constants
 from coriolis.db import api as db_api
 from coriolis.db.sqlalchemy import models
 from coriolis import exception
+from coriolis import utils
 from coriolis.worker.rpc import client as rpc_worker_client
 
 VERSION = "1.0"
@@ -16,52 +19,367 @@ VERSION = "1.0"
 LOG = logging.getLogger(__name__)
 
 
+def replica_synchronized(func):
+    @functools.wraps(func)
+    def wrapper(self, ctxt, replica_id, *args, **kwargs):
+        @lockutils.synchronized(replica_id)
+        def inner():
+            return func(self, ctxt, replica_id, *args, **kwargs)
+        return inner()
+    return wrapper
+
+
+def task_synchronized(func):
+    @functools.wraps(func)
+    def wrapper(self, ctxt, task_id, *args, **kwargs):
+        @lockutils.synchronized(task_id)
+        def inner():
+            return func(self, ctxt, task_id, *args, **kwargs)
+        return inner()
+    return wrapper
+
+
+def migration_synchronized(func):
+    @functools.wraps(func)
+    def wrapper(self, ctxt, migration_id, *args, **kwargs):
+        @lockutils.synchronized(migration_id)
+        def inner():
+            return func(self, ctxt, migration_id, *args, **kwargs)
+        return inner()
+    return wrapper
+
+
+def tasks_execution_synchronized(func):
+    @functools.wraps(func)
+    def wrapper(self, ctxt, execution_id, *args, **kwargs):
+        @lockutils.synchronized(execution_id)
+        def inner():
+            return func(self, ctxt, execution_id, *args, **kwargs)
+        return inner()
+    return wrapper
+
+
 class ConductorServerEndpoint(object):
     def __init__(self):
         self._rpc_worker_client = rpc_worker_client.WorkerClient()
 
+    @staticmethod
+    def _create_task(instance, task_type, execution, depends_on=None,
+                     on_error=False):
+        task = models.Task()
+        task.id = str(uuid.uuid4())
+        task.instance = instance
+        task.execution = execution
+        task.task_type = task_type
+        task.depends_on = depends_on
+        task.on_error = on_error
+
+        if not depends_on and on_error:
+            task.status = constants.TASK_STATUS_ON_ERROR_ONLY
+        else:
+            task.status = constants.TASK_STATUS_PENDING
+
+        return task
+
+    def _begin_tasks(self, ctxt, execution, task_info={}):
+        for task in execution.tasks:
+            if (not task.depends_on and
+                    task.status == constants.TASK_STATUS_PENDING):
+                self._rpc_worker_client.begin_task(
+                    ctxt, server=None,
+                    task_id=task.id,
+                    task_type=task.task_type,
+                    origin=execution.action.origin,
+                    destination=execution.action.destination,
+                    instance=task.instance,
+                    task_info=task_info.get(task.instance, {}))
+
+    @replica_synchronized
+    def execute_replica_tasks(self, ctxt, replica_id, shutdown_instances):
+        replica = self._get_replica(ctxt, replica_id)
+        self._check_replica_running_executions(ctxt, replica)
+        execution = models.TasksExecution()
+        execution.id = str(uuid.uuid4())
+        execution.status = constants.EXECUTION_STATUS_RUNNING
+        execution.action = replica
+
+        for instance in execution.action.instances:
+                depends_on = []
+                if shutdown_instances:
+                    shutdown_instance_task = self._create_task(
+                        instance, constants.TASK_TYPE_SHUTDOWN_INSTANCE,
+                        execution)
+                    depends_on = [shutdown_instance_task.id]
+
+                get_instance_info_task = self._create_task(
+                    instance, constants.TASK_TYPE_GET_INSTANCE_INFO,
+                    execution, depends_on=depends_on)
+
+                deploy_replica_disks_task = self._create_task(
+                    instance, constants.TASK_TYPE_DEPLOY_REPLICA_DISKS,
+                    execution, depends_on=[get_instance_info_task.id])
+
+                deploy_replica_source_resources_task = self._create_task(
+                    instance,
+                    constants.TASK_TYPE_DEPLOY_REPLICA_SOURCE_RESOURCES,
+                    execution, depends_on=[deploy_replica_disks_task.id])
+
+                deploy_replica_target_resources_task = self._create_task(
+                    instance,
+                    constants.TASK_TYPE_DEPLOY_REPLICA_TARGET_RESOURCES,
+                    execution, depends_on=[deploy_replica_disks_task.id])
+
+                replicate_disks_task = self._create_task(
+                    instance, constants.TASK_TYPE_REPLICATE_DISKS,
+                    execution, depends_on=[
+                        deploy_replica_source_resources_task.id,
+                        deploy_replica_target_resources_task.id])
+
+                self._create_task(
+                    instance,
+                    constants.TASK_TYPE_DELETE_REPLICA_SOURCE_RESOURCES,
+                    execution, depends_on=[replicate_disks_task.id],
+                    on_error=True)
+
+                self._create_task(
+                    instance,
+                    constants.TASK_TYPE_DELETE_REPLICA_TARGET_RESOURCES,
+                    execution, depends_on=[replicate_disks_task.id],
+                    on_error=True)
+
+        db_api.add_replica_tasks_execution(ctxt, execution)
+        LOG.info("Replica tasks execution created: %s", execution.id)
+
+        self._begin_tasks(ctxt, execution, replica.info)
+        return self.get_replica_tasks_execution(ctxt, execution.id)
+
+    @replica_synchronized
+    def get_replica_tasks_executions(self, ctxt, replica_id,
+                                     include_tasks=False):
+        return db_api.get_replica_tasks_executions(
+            ctxt, replica_id, include_tasks)
+
+    @tasks_execution_synchronized
+    def get_replica_tasks_execution(self, ctxt, execution_id):
+        return self._get_replica_tasks_execution(
+            ctxt, execution_id)
+
+    @tasks_execution_synchronized
+    def delete_replica_tasks_execution(self, ctxt, execution_id):
+        execution = self._get_replica_tasks_execution(
+            ctxt, execution_id)
+        if execution.status == constants.EXECUTION_STATUS_RUNNING:
+            raise exception.InvalidMigrationState(
+                "Cannot delete a running replica tasks execution")
+        db_api.delete_replica_tasks_execution(ctxt, execution_id)
+
+    @tasks_execution_synchronized
+    def cancel_replica_tasks_execution(self, ctxt, execution_id, force):
+        execution = self._get_replica_tasks_execution(
+            ctxt, execution_id)
+        if execution.status != constants.EXECUTION_STATUS_RUNNING:
+            raise exception.InvalidReplicaState(
+                "The replica tasks execution is not running")
+        self._cancel_tasks_execution(ctxt, execution, force)
+
+    def _get_replica_tasks_execution(self, ctxt, execution_id):
+        execution = db_api.get_replica_tasks_execution(
+            ctxt, execution_id)
+        if not execution:
+            raise exception.NotFound("Tasks execution not found")
+        return execution
+
+    def get_replicas(self, ctxt, include_tasks_executions=False):
+        return db_api.get_replicas(ctxt, include_tasks_executions)
+
+    @replica_synchronized
+    def get_replica(self, ctxt, replica_id):
+        return self._get_replica(ctxt, replica_id)
+
+    @replica_synchronized
+    def delete_replica(self, ctxt, replica_id):
+        replica = self._get_replica(ctxt, replica_id)
+        self._check_replica_running_executions(ctxt, replica)
+        db_api.delete_replica(ctxt, replica_id)
+
+    @replica_synchronized
+    def delete_replica_disks(self, ctxt, replica_id):
+        replica = self._get_replica(ctxt, replica_id)
+        self._check_replica_running_executions(ctxt, replica)
+
+        execution = models.TasksExecution()
+        execution.id = str(uuid.uuid4())
+        execution.status = constants.EXECUTION_STATUS_RUNNING
+        execution.action = replica
+
+        has_tasks = False
+        for instance in replica.instances:
+            if (instance in replica.instances and
+                    "volumes_info" in replica.info[instance]):
+                self._create_task(
+                    instance, constants.TASK_TYPE_DELETE_REPLICA_DISKS,
+                    execution)
+                has_tasks = True
+
+        if not has_tasks:
+            raise exception.InvalidReplicaState(
+                "This replica does not have volumes information for any "
+                "instance. Ensure that the replica has been executed "
+                "successfully priorly")
+
+        db_api.add_replica_tasks_execution(ctxt, execution)
+        LOG.info("Replica tasks execution created: %s", execution.id)
+
+        self._begin_tasks(ctxt, execution, replica.info)
+        return self.get_replica_tasks_execution(ctxt, execution.id)
+
+    def create_instances_replica(self, ctxt, origin, destination, instances):
+        replica = models.Replica()
+        replica.id = str(uuid.uuid4())
+        replica.origin = origin
+        replica.destination = destination
+        replica.instances = instances
+        replica.executions = []
+        replica.info = {}
+
+        db_api.add_replica(ctxt, replica)
+        LOG.info("Replica created: %s", replica.id)
+        return self.get_replica(ctxt, replica.id)
+
+    def _get_replica(self, ctxt, replica_id):
+        replica = db_api.get_replica(ctxt, replica_id)
+        if not replica:
+            raise exception.NotFound("Replica not found")
+        return replica
+
     def get_migrations(self, ctxt, include_tasks):
         return db_api.get_migrations(ctxt, include_tasks)
 
+    @migration_synchronized
     def get_migration(self, ctxt, migration_id):
-        return self._get_migration(ctxt, migration_id)
+        # the default serialization mechanism enforces a max_depth of 3
+        return utils.to_dict(self._get_migration(ctxt, migration_id))
+
+    @staticmethod
+    def _check_running_replica_migrations(ctxt, replica_id):
+        migrations = db_api.get_replica_migrations(ctxt, replica_id)
+        if [m.id for m in migrations if m.executions[0].status ==
+                constants.EXECUTION_STATUS_RUNNING]:
+            raise exception.InvalidReplicaState(
+                "This replica is currently being migrated")
+
+    @staticmethod
+    def _check_running_executions(action):
+        if [e for e in action.executions
+                if e.status == constants.EXECUTION_STATUS_RUNNING]:
+            raise exception.InvalidActionTasksExecutionState(
+                "Another tasks execution is in progress")
+
+    def _check_replica_running_executions(self, ctxt, replica):
+        self._check_running_executions(replica)
+        self._check_running_replica_migrations(ctxt, replica.id)
+
+    @staticmethod
+    def _check_valid_replica_tasks_execution(replica, force=False):
+        sorted_executions = sorted(
+            replica.executions, key=lambda e: e.number, reverse=True)
+
+        if (force and sorted_executions[0].status !=
+                constants.EXECUTION_STATUS_COMPLETED):
+            raise exception.InvalidReplicaState(
+                "The last replica tasks execution was not successful. "
+                "Perform a forced migration if you wish to perform a "
+                "migration without a successful last replica execution")
+        elif not [e for e in sorted_executions
+                  if e.status == constants.EXECUTION_STATUS_COMPLETED]:
+            raise exception.InvalidReplicaState(
+                "A replica must have been executed succesfully in order "
+                "to be migrated")
+
+    @replica_synchronized
+    def deploy_replica_instances(self, ctxt, replica_id, clone_disks, force):
+        replica = self._get_replica(ctxt, replica_id)
+        self._check_replica_running_executions(ctxt, replica)
+        self._check_valid_replica_tasks_execution(replica, force)
+
+        for instance, info in replica.info.items():
+            if not info.get("volumes_info"):
+                raise exception.InvalidReplicaState(
+                    "The replica doesn't contain volumes information for "
+                    "instance: %s. If replicated disks are deleted, the "
+                    "replica needs to be executed anew before a migration can "
+                    "occur" % instance)
+
+        instances = replica.instances
+
+        migration = models.Migration()
+        migration.id = str(uuid.uuid4())
+        migration.origin = replica.origin
+        migration.destination = replica.destination
+        migration.instances = instances
+        migration.replica = replica
+        migration.info = replica.info
+
+        for instance in instances:
+            migration.info[instance]["clone_disks"] = clone_disks
+
+        execution = models.TasksExecution()
+        migration.executions = [execution]
+        execution.status = constants.EXECUTION_STATUS_RUNNING
+        execution.number = 1
+
+        for instance in instances:
+            create_snapshot_task = self._create_task(
+                instance, constants.TASK_TYPE_CREATE_REPLICA_DISK_SNAPSHOTS,
+                execution)
+
+            deploy_replica_task = self._create_task(
+                instance, constants.TASK_TYPE_DEPLOY_REPLICA_INSTANCE,
+                execution, [create_snapshot_task.id])
+
+            self._create_task(
+                instance, constants.TASK_TYPE_DELETE_REPLICA_DISK_SNAPSHOTS,
+                execution, [deploy_replica_task.id],
+                on_error=clone_disks)
+
+            if not clone_disks:
+                self._create_task(
+                    instance,
+                    constants.TASK_TYPE_RESTORE_REPLICA_DISK_SNAPSHOTS,
+                    execution, on_error=True)
+
+        db_api.add_migration(ctxt, migration)
+        LOG.info("Migration created: %s", migration.id)
+
+        self._begin_tasks(ctxt, execution, migration.info)
+
+        return self.get_migration(ctxt, migration.id)
 
     def migrate_instances(self, ctxt, origin, destination, instances):
         migration = models.Migration()
         migration.id = str(uuid.uuid4())
-        migration.status = constants.MIGRATION_STATUS_RUNNING
         migration.origin = origin
         migration.destination = destination
+        execution = models.TasksExecution()
+        execution.status = constants.EXECUTION_STATUS_RUNNING
+        execution.number = 1
+        migration.executions = [execution]
+        migration.instances = instances
+        migration.info = {}
 
         for instance in instances:
-            task_export = models.Task()
-            task_export.id = str(uuid.uuid4())
-            task_export.migration = migration
-            task_export.instance = instance
-            task_export.status = constants.TASK_STATUS_PENDING
-            task_export.task_type = constants.TASK_TYPE_EXPORT_INSTANCE
-
-            task_import = models.Task()
-            task_import.id = str(uuid.uuid4())
-            task_import.migration = migration
-            task_import.instance = instance
-            task_import.status = constants.TASK_STATUS_PENDING
-            task_import.task_type = constants.TASK_TYPE_IMPORT_INSTANCE
-            task_import.depends_on = [task_export.id]
+
+            task_export = self._create_task(
+                instance, constants.TASK_TYPE_EXPORT_INSTANCE, execution)
+
+            self._create_task(
+                instance, constants.TASK_TYPE_IMPORT_INSTANCE,
+                execution, depends_on=[task_export.id])
 
         db_api.add_migration(ctxt, migration)
         LOG.info("Migration created: %s", migration.id)
 
-        for task in migration.tasks:
-            if not task.depends_on:
-                self._rpc_worker_client.begin_task(
-                    ctxt, server=None,
-                    task_id=task.id,
-                    task_type=task.task_type,
-                    origin=migration.origin,
-                    destination=migration.destination,
-                    instance=task.instance,
-                    task_info=None)
+        self._begin_tasks(ctxt, execution)
 
         return self.get_migration(ctxt, migration.id)
 
@@ -71,75 +389,177 @@ class ConductorServerEndpoint(object):
             raise exception.NotFound("Migration not found")
         return migration
 
+    @migration_synchronized
     def delete_migration(self, ctxt, migration_id):
         migration = self._get_migration(ctxt, migration_id)
-        if migration.status == constants.MIGRATION_STATUS_RUNNING:
+        execution = migration.executions[0]
+        if execution.status == constants.EXECUTION_STATUS_RUNNING:
             raise exception.InvalidMigrationState(
                 "Cannot delete a running migration")
         db_api.delete_migration(ctxt, migration_id)
 
-    def cancel_migration(self, ctxt, migration_id):
+    @migration_synchronized
+    def cancel_migration(self, ctxt, migration_id, force):
         migration = self._get_migration(ctxt, migration_id)
-        if migration.status != constants.MIGRATION_STATUS_RUNNING:
+        execution = migration.executions[0]
+        if execution.status != constants.EXECUTION_STATUS_RUNNING:
             raise exception.InvalidMigrationState(
                 "The migration is not running")
-
-        for task in migration.tasks:
-            if task.status in [constants.TASK_STATUS_PENDING,
-                               constants.TASK_STATUS_RUNNING]:
-                if task.status == constants.TASK_STATUS_RUNNING:
-                    self._rpc_worker_client.cancel_task(
-                        ctxt, task.host, task.process_id)
+        execution = migration.executions[0]
+        self._cancel_tasks_execution(ctxt, execution, force)
+
+    def _cancel_tasks_execution(self, ctxt, execution, force=False):
+        has_running_tasks = False
+        for task in execution.tasks:
+            if task.status == constants.TASK_STATUS_RUNNING:
+                self._rpc_worker_client.cancel_task(
+                    ctxt, task.host, task.process_id, force)
+                has_running_tasks = True
+            elif (task.status == constants.TASK_STATUS_PENDING and
+                    not task.on_error):
                 db_api.set_task_status(
                     ctxt, task.id, constants.TASK_STATUS_CANCELED)
 
-        db_api.set_migration_status(
-            ctxt, migration_id, constants.MIGRATION_STATUS_ERROR)
-
+        if not has_running_tasks:
+            for task in execution.tasks:
+                if task.status in [constants.TASK_STATUS_PENDING,
+                                   constants.TASK_STATUS_ON_ERROR_ONLY]:
+                    if task.on_error:
+                        action = db_api.get_action(ctxt, execution.action_id)
+                        task_info = action.info.get(task.instance, {})
+
+                        self._rpc_worker_client.begin_task(
+                            ctxt, server=None,
+                            task_id=task.id,
+                            task_type=task.task_type,
+                            origin=action.origin,
+                            destination=action.destination,
+                            instance=task.instance,
+                            task_info=task_info)
+
+                        has_running_tasks = True
+
+        if not has_running_tasks:
+            db_api.set_execution_status(
+                ctxt, execution.id, constants.EXECUTION_STATUS_ERROR)
+
+    @task_synchronized
     def set_task_host(self, ctxt, task_id, host, process_id):
         db_api.set_task_host(ctxt, task_id, host, process_id)
         db_api.set_task_status(
             ctxt, task_id, constants.TASK_STATUS_RUNNING)
 
-    def _start_pending_tasks(self, ctxt, migration, parent_task, task_info):
-        has_pending_tasks = False
-        for task in migration.tasks:
-            if (task.depends_on and parent_task.id in task.depends_on and
-                    task.status == constants.TASK_STATUS_PENDING):
-                has_pending_tasks = True
-                # instance imports needs to be executed on the same host
-                server = None
-                if task.task_type == constants.TASK_TYPE_IMPORT_INSTANCE:
-                    server = parent_task.host
-
-                self._rpc_worker_client.begin_task(
-                    ctxt, server=server,
-                    task_id=task.id,
-                    task_type=task.task_type,
-                    origin=migration.origin,
-                    destination=migration.destination,
-                    instance=task.instance,
-                    task_info=task_info)
-        return has_pending_tasks
-
+    def _start_pending_tasks(self, ctxt, execution, parent_task, task_info):
+        for task in execution.tasks:
+            if task.status == constants.TASK_STATUS_PENDING:
+                if task.depends_on and parent_task.id in task.depends_on:
+                    start_task = True
+                    for depend_task_id in task.depends_on:
+                        if depend_task_id != parent_task.id:
+                            depend_task = db_api.get_task(ctxt, depend_task_id)
+                            if (depend_task.status !=
+                                    constants.TASK_STATUS_COMPLETED):
+                                start_task = False
+                                break
+                    if start_task:
+                        # instance imports need to be executed on the same host
+                        server = None
+                        if (task.task_type ==
+                                constants.TASK_TYPE_IMPORT_INSTANCE):
+                            server = parent_task.host
+
+                        action = execution.action
+                        self._rpc_worker_client.begin_task(
+                            ctxt, server=server,
+                            task_id=task.id,
+                            task_type=task.task_type,
+                            origin=action.origin,
+                            destination=action.destination,
+                            instance=task.instance,
+                            task_info=task_info)
+
+    def _update_replica_volumes_info(self, ctxt, migration_id, instance,
+                                     updated_task_info):
+        migration = db_api.get_migration(ctxt, migration_id)
+        replica_id = migration.replica_id
+
+        with lockutils.lock(replica_id):
+            LOG.debug(
+                "Updating volume_info in replica due to snapshot "
+                "restore during migration. replica id: %s", replica_id)
+            db_api.set_transfer_action_info(
+                ctxt, replica_id, instance,
+                updated_task_info)
+
+    def _handle_post_task_actions(self, ctxt, task, execution, task_info):
+        task_type = task.task_type
+        updated_task_info = None
+
+        if task_type == constants.TASK_TYPE_RESTORE_REPLICA_DISK_SNAPSHOTS:
+            # When restoring a snapshot in some import providers (OpenStack),
+            # a new volume_id is generated. This needs to be updated in the
+            # Replica instance as well.
+            volumes_info = task_info.get("volumes_info")
+            if volumes_info:
+                updated_task_info = {"volumes_info": volumes_info}
+        elif task_type == constants.TASK_TYPE_DELETE_REPLICA_DISK_SNAPSHOTS:
+
+            if not task_info.get("clone_disks"):
+                # The migration completed. If the replica is executed again,
+                # new volumes need to be deployed in place of the migrated
+                # ones.
+                updated_task_info = {"volumes_info": None}
+
+        if updated_task_info:
+            self._update_replica_volumes_info(
+                ctxt, execution.action_id, task.instance,
+                updated_task_info)
+
+    @task_synchronized
     def task_completed(self, ctxt, task_id, task_info):
         LOG.info("Task completed: %s", task_id)
 
         db_api.set_task_status(
             ctxt, task_id, constants.TASK_STATUS_COMPLETED)
 
-        task = db_api.get_task(
-            ctxt, task_id, include_migration_tasks=True)
-
-        migration = task.migration
-        has_pending_tasks = self._start_pending_tasks(ctxt, migration, task,
-                                                      task_info)
-
-        if not has_pending_tasks:
-            LOG.info("Migration completed: %s", migration.id)
-            db_api.set_migration_status(
-                ctxt, migration.id, constants.MIGRATION_STATUS_COMPLETED)
-
+        task = db_api.get_task(ctxt, task_id)
+        execution = db_api.get_tasks_execution(ctxt, task.execution_id)
+
+        action_id = execution.action_id
+        with lockutils.lock(action_id):
+            LOG.info("Setting instance %(instance)s "
+                     "action info: %(task_info)s",
+                     {"instance": task.instance, "task_info": task_info})
+            updated_task_info = db_api.set_transfer_action_info(
+                ctxt, action_id, task.instance, task_info)
+
+            self._handle_post_task_actions(
+                ctxt, task, execution, updated_task_info)
+
+            if execution.status == constants.EXECUTION_STATUS_RUNNING:
+                self._start_pending_tasks(
+                    ctxt, execution, task, updated_task_info)
+
+                if not [t for t in execution.tasks
+                        if t.status in [constants.TASK_STATUS_RUNNING,
+                                        constants.TASK_STATUS_PENDING]]:
+                    # The execution is in error status if there's one or more
+                    # tasks in error or canceled status
+                    if [t for t in execution.tasks
+                            if t.status in [constants.TASK_STATUS_ERROR,
+                                            constants.TASK_STATUS_CANCELED]]:
+                        execution_status = constants.EXECUTION_STATUS_ERROR
+                    else:
+                        execution_status = constants.EXECUTION_STATUS_COMPLETED
+
+                    LOG.info("Tasks execution %(execution_id)s completed "
+                             "with status: %(status)s",
+                             {"execution_id": execution.id,
+                              "status": execution_status})
+                    db_api.set_execution_status(
+                        ctxt, execution.id, execution_status)
+
+    @task_synchronized
     def set_task_error(self, ctxt, task_id, exception_details):
         LOG.error("Task error: %(task_id)s - %(ex)s",
                   {"task_id": task_id, "ex": exception_details})
@@ -147,23 +567,18 @@ class ConductorServerEndpoint(object):
         db_api.set_task_status(
             ctxt, task_id, constants.TASK_STATUS_ERROR, exception_details)
 
-        task = db_api.get_task(
-            ctxt, task_id, include_migration_tasks=True)
-        migration = task.migration
-
-        for task in migration.tasks:
-            if task.status == constants.TASK_STATUS_PENDING:
-                db_api.set_task_status(
-                    ctxt, task.id, constants.TASK_STATUS_CANCELED)
+        task = db_api.get_task(ctxt, task_id)
+        execution = db_api.get_tasks_execution(ctxt, task.execution_id)
 
-        LOG.error("Migration failed: %s", migration.id)
-        db_api.set_migration_status(
-            ctxt, migration.id, constants.MIGRATION_STATUS_ERROR)
+        with lockutils.lock(execution.action_id):
+            self._cancel_tasks_execution(ctxt, execution)
 
+    @task_synchronized
     def task_event(self, ctxt, task_id, level, message):
         LOG.info("Task event: %s", task_id)
         db_api.add_task_event(ctxt, task_id, level, message)
 
+    @task_synchronized
     def task_progress_update(self, ctxt, task_id, current_step, total_steps,
                              message):
         LOG.info("Task progress update: %s", task_id)

coriolis/constants.py

@@ -1,19 +1,35 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
 
-MIGRATION_STATUS_RUNNING = "RUNNING"
-MIGRATION_STATUS_COMPLETED = "COMPLETED"
-MIGRATION_STATUS_ERROR = "ERROR"
+EXECUTION_STATUS_RUNNING = "RUNNING"
+EXECUTION_STATUS_COMPLETED = "COMPLETED"
+EXECUTION_STATUS_ERROR = "ERROR"
 
 TASK_STATUS_PENDING = "PENDING"
 TASK_STATUS_RUNNING = "RUNNING"
 TASK_STATUS_COMPLETED = "COMPLETED"
 TASK_STATUS_ERROR = "ERROR"
 TASK_STATUS_CANCELED = "CANCELED"
+TASK_STATUS_ON_ERROR_ONLY = "EXECUTE_ON_ERROR_ONLY"
 
 TASK_TYPE_EXPORT_INSTANCE = "EXPORT_INSTANCE"
 TASK_TYPE_IMPORT_INSTANCE = "IMPORT_INSTANCE"
 
+TASK_TYPE_GET_INSTANCE_INFO = "GET_INSTANCE_INFO"
+TASK_TYPE_DEPLOY_REPLICA_DISKS = "DEPLOY_REPLICA_DISKS"
+TASK_TYPE_DELETE_REPLICA_DISKS = "DELETE_REPLICA_DISKS"
+TASK_TYPE_REPLICATE_DISKS = "REPLICATE_DISKS"
+TASK_TYPE_DEPLOY_REPLICA_SOURCE_RESOURCES = "DEPLOY_REPLICA_SOURCE_RESOURCES"
+TASK_TYPE_DELETE_REPLICA_SOURCE_RESOURCES = "DELETE_REPLICA_SOURCE_RESOURCES"
+TASK_TYPE_DEPLOY_REPLICA_TARGET_RESOURCES = "DEPLOY_REPLICA_TARGET_RESOURCES"
+TASK_TYPE_DELETE_REPLICA_TARGET_RESOURCES = "DELETE_REPLICA_TARGET_RESOURCES"
+TASK_TYPE_SHUTDOWN_INSTANCE = "SHUTDOWN_INSTANCE"
+TASK_TYPE_DEPLOY_REPLICA_INSTANCE = "DEPLOY_REPLICA_INSTANCE"
+TASK_TYPE_CREATE_REPLICA_DISK_SNAPSHOTS = "CREATE_REPLICA_DISK_SNAPSHOTS"
+TASK_TYPE_DELETE_REPLICA_DISK_SNAPSHOTS = "DELETE_REPLICA_DISK_SNAPSHOTS"
+TASK_TYPE_RESTORE_REPLICA_DISK_SNAPSHOTS = "RESTORE_REPLICA_DISK_SNAPSHOTS"
+
+
 PROVIDER_TYPE_IMPORT = 1
 PROVIDER_TYPE_EXPORT = 2
 
@@ -46,3 +62,5 @@ OS_TYPE_LINUX = "linux"
 OS_TYPE_OS_X = "osx"
 OS_TYPE_SOLARIS = "solaris"
 OS_TYPE_WINDOWS = "windows"
+
+TMP_DIRS_KEY = "__tmp_dirs"

coriolis/data_transfer.py

@@ -0,0 +1,44 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import struct
+import zlib
+
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+def encode_data(msg_id, path, offset, content, compress=True):
+    inflated_content = (path.encode() + b'\0' +
+                        struct.pack("<Q", offset) +
+                        content)
+
+    data_len_inflated = len(inflated_content)
+
+    if compress:
+        data_content = zlib.compress(inflated_content)
+        data_len = len(data_content)
+
+        compression_saving = 100.0 * (1 - float(data_len) / data_len_inflated)
+        LOG.debug("Compression space saving: {:.02f}%".format(
+            compression_saving))
+
+        if data_len >= data_len_inflated:
+            # No advantage in sending the compressed data
+            LOG.debug("Ignoring compression, not worth")
+            compress = False
+
+    if not compress:
+        data_len = data_len_inflated
+        data_len_inflated = 0
+        data_content = inflated_content
+
+    return (struct.pack("<I", msg_id) +
+            struct.pack("<I", data_len) +
+            struct.pack("<I", data_len_inflated) +
+            data_content)
+
+
+def encode_eod(msg_id):
+    return struct.pack("<I", msg_id) + struct.pack("<I", 0)

coriolis/db/api.py

@@ -5,6 +5,7 @@ from oslo_config import cfg
 from oslo_db import api as db_api
 from oslo_db import options as db_options
 from oslo_db.sqlalchemy import enginefacade
+from sqlalchemy import func
 from sqlalchemy import orm
 
 from coriolis.db.sqlalchemy import models
@@ -57,18 +58,134 @@ def _soft_delete_aware_query(context, *args, **kwargs):
     return query
 
 
+@enginefacade.reader
+def get_replica_tasks_executions(context, replica_id, include_tasks=False):
+    q = _soft_delete_aware_query(context, models.TasksExecution)
+    q = q.join(models.Replica)
+    if include_tasks:
+        q = _get_tasks_with_details_options(q)
+    return q.filter(
+        models.Replica.project_id == context.tenant,
+        models.Replica.id == replica_id).all()
+
+
+@enginefacade.reader
+def get_replica_tasks_execution(context, execution_id):
+    q = _soft_delete_aware_query(context, models.TasksExecution).join(
+        models.Replica)
+    q = _get_tasks_with_details_options(q)
+    return q.filter(
+        models.Replica.project_id == context.tenant,
+        models.TasksExecution.id == execution_id).first()
+
+
+@enginefacade.writer
+def add_replica_tasks_execution(context, execution):
+    if execution.action.project_id != context.tenant:
+        raise exception.NotAuthorized()
+
+    # include deleted records
+    max_number = _model_query(
+        context, func.max(models.TasksExecution.number)).filter_by(
+            action_id=execution.action.id).first()[0] or 0
+    execution.number = max_number + 1
+
+    context.session.add(execution)
+
+
+@enginefacade.writer
+def delete_replica_tasks_execution(context, execution_id):
+    q = _soft_delete_aware_query(context, models.TasksExecution).filter(
+        models.TasksExecution.id == execution_id)
+    if not q.join(models.Replica).filter(
+            models.Replica.project_id == context.tenant).first():
+        raise exception.NotAuthorized()
+    count = q.soft_delete()
+    if count == 0:
+        raise exception.NotFound("0 entries were soft deleted")
+
+
+def _get_replica_with_tasks_executions_options(q):
+    return q.options(orm.joinedload(models.Replica.executions))
+
+
+@enginefacade.reader
+def get_replicas(context, include_tasks_executions=False):
+    q = _soft_delete_aware_query(context, models.Replica)
+    if include_tasks_executions:
+        q = _get_replica_with_tasks_executions_options(q)
+    return q.filter(
+        models.Replica.project_id == context.tenant).all()
+
+
+@enginefacade.reader
+def get_replica(context, replica_id):
+    q = _soft_delete_aware_query(context, models.Replica)
+    q = _get_replica_with_tasks_executions_options(q)
+    return q.filter(
+        models.Replica.project_id == context.tenant,
+        models.Replica.id == replica_id).first()
+
+
+@enginefacade.writer
+def add_replica(context, replica):
+    replica.user_id = context.user
+    replica.project_id = context.tenant
+    context.session.add(replica)
+
+
+@enginefacade.writer
+def _delete_transfer_action(context, cls, id):
+    count = _soft_delete_aware_query(context, cls).filter_by(
+        project_id=context.tenant, base_id=id).soft_delete()
+    if count == 0:
+        raise exception.NotFound("0 entries were soft deleted")
+
+    _soft_delete_aware_query(context, models.TasksExecution).filter_by(
+        action_id=id).soft_delete()
+
+
+@enginefacade.writer
+def delete_replica(context, replica_id):
+    _delete_transfer_action(context, models.Replica, replica_id)
+
+
+@enginefacade.reader
+def get_replica_migrations(context, replica_id):
+    q = _soft_delete_aware_query(context, models.Migration)
+    q = q.join("replica")
+    q = q.options(orm.joinedload("executions"))
+    return q.filter(
+        models.Migration.project_id == context.tenant,
+        models.Replica.id == replica_id).all()
+
+
 @enginefacade.reader
 def get_migrations(context, include_tasks=False):
     q = _soft_delete_aware_query(context, models.Migration)
     if include_tasks:
         q = _get_migration_task_query_options(q)
+    else:
+        q = q.options(orm.joinedload("executions"))
     return q.filter_by(project_id=context.tenant).all()
 
 
+def _get_tasks_with_details_options(query):
+    return query.options(
+        orm.joinedload("tasks").
+        joinedload("progress_updates")).options(
+            orm.joinedload("tasks").
+            joinedload("events"))
+
+
 def _get_migration_task_query_options(query):
     return query.options(
-        orm.joinedload("tasks").joinedload("progress_updates")).options(
-            orm.joinedload("tasks").joinedload("events"))
+        orm.joinedload("executions").
+        joinedload("tasks").
+        joinedload("progress_updates")).options(
+            orm.joinedload("executions").
+            joinedload("tasks").
+            joinedload("events"))
 
 
 @enginefacade.reader
@@ -87,20 +204,65 @@ def add_migration(context, migration):
 
 @enginefacade.writer
 def delete_migration(context, migration_id):
-    count = _soft_delete_aware_query(context, models.Migration).filter_by(
-        project_id=context.tenant, id=migration_id).soft_delete()
-    if count == 0:
-        raise exception.NotFound("0 entries were soft deleted")
+    _delete_transfer_action(context, models.Migration, migration_id)
 
 
 @enginefacade.writer
-def set_migration_status(context, migration_id, status):
-    migration = _soft_delete_aware_query(context, models.Migration).filter_by(
-        project_id=context.tenant, id=migration_id).first()
-    if not migration:
-        raise exception.NotFound("Migration not found: %s" % migration_id)
+def set_execution_status(context, execution_id, status):
+    execution = _soft_delete_aware_query(
+        context, models.TasksExecution).join(
+            models.TasksExecution.action).filter(
+                models.BaseTransferAction.project_id == context.tenant,
+                models.TasksExecution.id == execution_id).first()
+    if not execution:
+        raise exception.NotFound(
+            "Tasks execution not found: %s" % execution_id)
 
-    migration.status = status
+    execution.status = status
+
+
+@enginefacade.reader
+def get_action(context, action_id):
+    action = _soft_delete_aware_query(
+        context, models.BaseTransferAction).filter(
+            models.BaseTransferAction.project_id == context.tenant,
+            models.BaseTransferAction.base_id == action_id).first()
+    if not action:
+        raise exception.NotFound(
+            "Transfer action not found: %s" % action_id)
+    return action
+
+
+@enginefacade.writer
+def set_transfer_action_info(context, action_id, instance, instance_info):
+    action = get_action(context, action_id)
+
+    # Copy is needed, otherwise sqlalchemy won't save the changes
+    action_info = action.info.copy()
+    if instance in action_info:
+        instance_info_old = action_info[instance].copy()
+        instance_info_old.update(instance_info)
+        action_info[instance] = instance_info_old
+    else:
+        action_info[instance] = instance_info
+    action.info = action_info
+
+    return action_info[instance]
+
+
+@enginefacade.reader
+def get_tasks_execution(context, execution_id):
+    q = _soft_delete_aware_query(context, models.TasksExecution)
+    q = q.join(models.BaseTransferAction)
+    q = q.options(orm.joinedload("action"))
+    q = q.options(orm.joinedload("tasks"))
+    execution = q.filter(
+        models.BaseTransferAction.project_id == context.tenant,
+        models.TasksExecution.id == execution_id).first()
+    if not execution:
+        raise exception.NotFound(
+            "Tasks execution not found: %s" % execution_id)
+    return execution
 
 
 def _get_task(context, task_id):
@@ -126,13 +288,9 @@ def set_task_host(context, task_id, host, process_id):
 
 
 @enginefacade.reader
-def get_task(context, task_id, include_migration_tasks=False):
-    join_options = orm.joinedload("migration")
-    if include_migration_tasks:
-        join_options = join_options.joinedload("tasks")
-
-    return _soft_delete_aware_query(context, models.Task).options(
-        join_options).filter_by(id=task_id).first()
+def get_task(context, task_id):
+    q = _soft_delete_aware_query(context, models.Task)
+    return q.filter_by(id=task_id).first()
 
 
 @enginefacade.writer
@@ -144,12 +302,22 @@ def add_task_event(context, task_id, level, message):
     context.session.add(task_event)
 
 
+def _get_progress_update(context, task_id, current_step):
+    q = _soft_delete_aware_query(context, models.TaskProgressUpdate)
+    return q.filter(
+        models.TaskProgressUpdate.task_id == task_id,
+        models.TaskProgressUpdate.current_step == current_step).first()
+
+
 @enginefacade.writer
 def add_task_progress_update(context, task_id, current_step, total_steps,
                              message):
-    task_progress_update = models.TaskProgressUpdate()
+    task_progress_update = _get_progress_update(context, task_id, current_step)
+    if not task_progress_update:
+        task_progress_update = models.TaskProgressUpdate()
+        context.session.add(task_progress_update)
+
     task_progress_update.task_id = task_id
     task_progress_update.current_step = current_step
     task_progress_update.total_steps = total_steps
     task_progress_update.message = message
-    context.session.add(task_progress_update)

coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py

@@ -10,9 +10,9 @@ def upgrade(migrate_engine):
     meta = sqlalchemy.MetaData()
     meta.bind = migrate_engine
 
-    migration = sqlalchemy.Table(
-        'migration', meta,
-        sqlalchemy.Column("id", sqlalchemy.String(36), primary_key=True,
+    base_transfer_action = sqlalchemy.Table(
+        'base_transfer_action', meta,
+        sqlalchemy.Column("base_id", sqlalchemy.String(36), primary_key=True,
                           default=lambda: str(uuid.uuid4())),
         sqlalchemy.Column('created_at', sqlalchemy.DateTime),
         sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
@@ -24,7 +24,22 @@ def upgrade(migrate_engine):
         sqlalchemy.Column("origin", sqlalchemy.Text, nullable=False),
         sqlalchemy.Column("destination", sqlalchemy.Text,
                           nullable=False),
-        sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False),
+        sqlalchemy.Column("instances", sqlalchemy.Text, nullable=False),
+        sqlalchemy.Column("type", sqlalchemy.String(50), nullable=False),
+        sqlalchemy.Column("info", sqlalchemy.Text, nullable=False),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'
+    )
+
+    migration = sqlalchemy.Table(
+        'migration', meta,
+        sqlalchemy.Column("id", sqlalchemy.String(36),
+                          sqlalchemy.ForeignKey(
+                              'base_transfer_action.base_id'),
+                          primary_key=True),
+        sqlalchemy.Column("replica_id", sqlalchemy.String(36),
+                          sqlalchemy.ForeignKey(
+                              'replica.id'), nullable=True),
         mysql_engine='InnoDB',
         mysql_charset='utf8'
     )
@@ -37,8 +52,9 @@ def upgrade(migrate_engine):
         sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
         sqlalchemy.Column('deleted_at', sqlalchemy.DateTime),
         sqlalchemy.Column('deleted', sqlalchemy.String(36)),
-        sqlalchemy.Column("migration_id", sqlalchemy.String(36),
-                          sqlalchemy.ForeignKey('migration.id'),
+        sqlalchemy.Column("execution_id", sqlalchemy.String(36),
+                          sqlalchemy.ForeignKey(
+                              'tasks_execution.id'),
                           nullable=False),
         sqlalchemy.Column("instance", sqlalchemy.String(1024), nullable=False),
         sqlalchemy.Column("host", sqlalchemy.String(1024), nullable=True),
@@ -48,6 +64,25 @@ def upgrade(migrate_engine):
                           nullable=False),
         sqlalchemy.Column("exception_details", sqlalchemy.Text, nullable=True),
         sqlalchemy.Column("depends_on", sqlalchemy.Text, nullable=True),
+        sqlalchemy.Column("on_error", sqlalchemy.Boolean, nullable=True),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'
+    )
+
+    tasks_execution = sqlalchemy.Table(
+        'tasks_execution', meta,
+        sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True,
+                          default=lambda: str(uuid.uuid4())),
+        sqlalchemy.Column('created_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('deleted_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('deleted', sqlalchemy.String(36)),
+        sqlalchemy.Column("action_id", sqlalchemy.String(36),
+                          sqlalchemy.ForeignKey(
+                              'base_transfer_action.base_id'),
+                          nullable=False),
+        sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False),
+        sqlalchemy.Column("number", sqlalchemy.Integer, nullable=False),
         mysql_engine='InnoDB',
         mysql_charset='utf8'
     )
@@ -87,8 +122,21 @@ def upgrade(migrate_engine):
         mysql_charset='utf8'
     )
 
+    replica = sqlalchemy.Table(
+        'replica', meta,
+        sqlalchemy.Column("id", sqlalchemy.String(36),
+                          sqlalchemy.ForeignKey(
+                              'base_transfer_action.base_id'),
+                          primary_key=True),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'
+    )
+
     tables = (
+        base_transfer_action,
+        replica,
         migration,
+        tasks_execution,
         task,
         task_progress_update,
         task_events,

coriolis/db/sqlalchemy/models.py

@@ -49,9 +49,9 @@ class Task(BASE, models.TimestampMixin, models.SoftDeleteMixin,
     id = sqlalchemy.Column(sqlalchemy.String(36),
                            default=lambda: str(uuid.uuid4()),
                            primary_key=True)
-    migration_id = sqlalchemy.Column(sqlalchemy.String(36),
-                                     sqlalchemy.ForeignKey('migration.id'),
-                                     nullable=False)
+    execution_id = sqlalchemy.Column(
+        sqlalchemy.String(36),
+        sqlalchemy.ForeignKey('tasks_execution.id'), nullable=False)
     instance = sqlalchemy.Column(sqlalchemy.String(1024), nullable=False)
     host = sqlalchemy.Column(sqlalchemy.String(1024), nullable=True)
     process_id = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
@@ -59,24 +59,85 @@ class Task(BASE, models.TimestampMixin, models.SoftDeleteMixin,
     task_type = sqlalchemy.Column(sqlalchemy.String(100), nullable=False)
     exception_details = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
     depends_on = sqlalchemy.Column(types.List, nullable=True)
+    on_error = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False)
+    # TODO: Add soft delete filter
     events = orm.relationship(TaskEvent, cascade="all,delete",
                               backref=orm.backref('task'))
+    # TODO: Add soft delete filter
     progress_updates = orm.relationship(TaskProgressUpdate,
                                         cascade="all,delete",
                                         backref=orm.backref('task'))
 
 
-class Migration(BASE, models.TimestampMixin, models.ModelBase,
-                models.SoftDeleteMixin):
-    __tablename__ = 'migration'
+class TasksExecution(BASE, models.TimestampMixin, models.ModelBase,
+                     models.SoftDeleteMixin):
+    __tablename__ = 'tasks_execution'
 
     id = sqlalchemy.Column(sqlalchemy.String(36),
                            default=lambda: str(uuid.uuid4()),
                            primary_key=True)
+    action_id = sqlalchemy.Column(
+        sqlalchemy.String(36),
+        sqlalchemy.ForeignKey('base_transfer_action.base_id'), nullable=False)
+    # TODO: Add soft delete filter
+    tasks = orm.relationship(Task, cascade="all,delete",
+                             backref=orm.backref('execution'))
+    status = sqlalchemy.Column(sqlalchemy.String(100), nullable=False)
+    number = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
+
+
+class BaseTransferAction(BASE, models.TimestampMixin, models.ModelBase,
+                         models.SoftDeleteMixin):
+    __tablename__ = 'base_transfer_action'
+
+    base_id = sqlalchemy.Column(sqlalchemy.String(36),
+                                default=lambda: str(uuid.uuid4()),
+                                primary_key=True)
     user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
     project_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
     origin = sqlalchemy.Column(types.Json, nullable=False)
     destination = sqlalchemy.Column(types.Json, nullable=False)
-    status = sqlalchemy.Column(sqlalchemy.String(100), nullable=False)
-    tasks = orm.relationship(Task, cascade="all,delete",
-                             backref=orm.backref('migration'))
+    type = sqlalchemy.Column(sqlalchemy.String(50))
+    executions = orm.relationship(TasksExecution, cascade="all,delete",
+                                  backref=orm.backref('action'),
+                                  primaryjoin="and_(BaseTransferAction."
+                                  "base_id==TasksExecution.action_id, "
+                                  "TasksExecution.deleted=='0')")
+    instances = sqlalchemy.Column(types.List, nullable=False)
+    info = sqlalchemy.Column(types.Json, nullable=False)
+
+    __mapper_args__ = {
+        'polymorphic_identity': 'base_transfer_action',
+        'polymorphic_on': type,
+    }
+
+
+class Replica(BaseTransferAction):
+    __tablename__ = 'replica'
+
+    id = sqlalchemy.Column(
+        sqlalchemy.String(36),
+        sqlalchemy.ForeignKey(
+            'base_transfer_action.base_id'), primary_key=True)
+
+    __mapper_args__ = {
+        'polymorphic_identity': 'replica',
+    }
+
+
+class Migration(BaseTransferAction):
+    __tablename__ = 'migration'
+
+    id = sqlalchemy.Column(
+        sqlalchemy.String(36),
+        sqlalchemy.ForeignKey(
+            'base_transfer_action.base_id'), primary_key=True)
+    replica_id = sqlalchemy.Column(
+        sqlalchemy.String(36),
+        sqlalchemy.ForeignKey('replica.id'), nullable=True)
+    replica = orm.relationship(
+        Replica, backref=orm.backref("migrations"), foreign_keys=[replica_id])
+
+    __mapper_args__ = {
+        'polymorphic_identity': 'migration',
+    }

coriolis/events.py

@@ -2,6 +2,11 @@
 # All Rights Reserved.
 
 import abc
+import collections
+
+
+_PercStepData = collections.namedtuple(
+    "_PercStepData", "last_value max_value perc_threshold message_format")
 
 
 class EventManager(object):
@@ -11,10 +16,33 @@ class EventManager(object):
         self._event_handler = event_handler
         self._current_step = 0
         self._total_steps = None
+        self._percentage_steps = {}
 
     def set_total_progress_steps(self, total_steps):
         self._total_steps = total_steps
 
+    def add_percentage_step(self, max_value, perc_threshold=1,
+                            message_format="{:.0f}%"):
+        self._current_step += 1
+        self._percentage_steps[self._current_step] = _PercStepData(
+            0, max_value, perc_threshold, message_format)
+        return self._current_step
+
+    def set_percentage_step(self, step, value):
+        step_data = self._percentage_steps[step]
+
+        old_perc = (step_data.last_value * 100 / step_data.max_value //
+                    step_data.perc_threshold * step_data.perc_threshold)
+        perc = (value * 100 / step_data.max_value //
+                step_data.perc_threshold * step_data.perc_threshold)
+
+        if perc > old_perc and self._event_handler:
+            self._event_handler.progress_update(
+                step, self._total_steps, step_data.message_format.format(perc))
+            self._percentage_steps[step] = _PercStepData(
+                value, step_data.max_value, step_data.perc_threshold,
+                step_data.message_format)
+
     def progress_update(self, message):
         self._current_step += 1
         if self._event_handler:

coriolis/exception.py

@@ -166,10 +166,18 @@ class InvalidConfigurationValue(Invalid):
                 'configuration option "%(option)s"')
 
 
+class InvalidActionTasksExecutionState(Invalid):
+    message = _("Invalid tasks execution state: %(reason)s")
+
+
 class InvalidMigrationState(Invalid):
     message = _("Invalid migration state: %(reason)s")
 
 
+class InvalidReplicaState(Invalid):
+    message = _("Invalid replica state: %(reason)s")
+
+
 class ServiceUnavailable(Invalid):
     message = _("Service is unavailable at this time.")
 

coriolis/migrations/api.py

@@ -8,15 +8,20 @@ class API(object):
     def __init__(self):
         self._rpc_client = rpc_client.ConductorClient()
 
-    def start(self, ctxt, origin, destination, instances):
-        return self._rpc_client.begin_migrate_instances(
+    def migrate_instances(self, ctxt, origin, destination, instances):
+        return self._rpc_client.migrate_instances(
             ctxt, origin, destination, instances)
 
+    def deploy_replica_instances(self, ctxt, replica_id, clone_disks=False,
+                                 force=False):
+        return self._rpc_client.deploy_replica_instances(
+            ctxt, replica_id, clone_disks, force)
+
     def delete(self, ctxt, migration_id):
         self._rpc_client.delete_migration(ctxt, migration_id)
 
-    def cancel(self, ctxt, migration_id):
-        self._rpc_client.cancel_migration(ctxt, migration_id)
+    def cancel(self, ctxt, migration_id, force):
+        self._rpc_client.cancel_migration(ctxt, migration_id, force)
 
     def get_migrations(self, ctxt, include_tasks=False):
         return self._rpc_client.get_migrations(ctxt, include_tasks)

coriolis/providers/azure/__init__.py

@@ -166,14 +166,12 @@ class ImportProvider(BaseImportProvider):
         """ Validates the provided connection information. """
         LOG.info("Validating connection info: %s", connection_info)
 
-        if not super(ImportProvider, self).validate_connection_info(
-                connection_info):
-            return False
+        super(ImportProvider, self).validate_connection_info(connection_info)
 
         # NOTE: considering we cannot check the validity of the credentials per
         # se if a secret href is provided here, we simply return on the spot:
         if "secret_ref" in connection_info:
-            return True
+            return
 
         try:
             # NOTE: attempt to register to a provider to ensure credentials
@@ -184,12 +182,8 @@ class ImportProvider(BaseImportProvider):
         except (KeyError, azure_exceptions.CloudError,
                 azexceptions.AzureOperationException) as ex:
 
-            LOG.info(
-                "Invalid or incomplete Azure credentials provided: %s\n%s",
-                connection_info, ex)
-            return False
-        else:
-            return True
+            raise exception.InvalidInput(
+                "Invalid or incomplete Azure credentials provided")
 
     def _get_cloud_credentials(self, connection_info):
         """ returns the msrestazure.azure_active_directory.Credentials

coriolis/providers/base.py

@@ -22,13 +22,7 @@ class BaseProvider(object):
         """ Checks the provided connection info and raises an exception
         if it is invalid.
         """
-        try:
-            schemas.validate_value(
-                connection_info, self.connection_info_schema)
-        except:
-            return False
-
-        return True
+        schemas.validate_value(connection_info, self.connection_info_schema)
 
 
 class BaseImportProvider(BaseProvider):
@@ -51,7 +45,6 @@ class BaseImportProvider(BaseProvider):
 
         return True
 
-
     @abc.abstractmethod
     def import_instance(self, ctxt, connection_info, target_environment,
                         instance_name, export_info):
@@ -62,6 +55,50 @@ class BaseImportProvider(BaseProvider):
         pass
 
 
+class BaseReplicaImportProvider(BaseImportProvider):
+    __metaclass__ = abc.ABCMeta
+
+    @abc.abstractmethod
+    def deploy_replica_instance(self, ctxt, connection_info,
+                                target_environment, instance_name, export_info,
+                                volumes_info, clone_disks):
+        pass
+
+    @abc.abstractmethod
+    def deploy_replica_disks(self, ctxt, connection_info, target_environment,
+                             instance_name, export_info, volumes_info):
+        pass
+
+    @abc.abstractmethod
+    def deploy_replica_target_resources(self, ctxt, connection_info,
+                                        target_environment, volumes_info):
+        pass
+
+    @abc.abstractmethod
+    def delete_replica_target_resources(self, ctxt, connection_info,
+                                        migr_resources_dict):
+        pass
+
+    @abc.abstractmethod
+    def delete_replica_disks(self, ctxt, connection_info, volumes_info):
+        pass
+
+    @abc.abstractmethod
+    def create_replica_disk_snapshots(self, ctxt, connection_info,
+                                      volumes_info):
+        pass
+
+    @abc.abstractmethod
+    def delete_replica_disk_snapshots(self, ctxt, connection_info,
+                                      volumes_info):
+        pass
+
+    @abc.abstractmethod
+    def restore_replica_disk_snapshots(self, ctxt, connection_info,
+                                       volumes_info):
+        pass
+
+
 class BaseExportProvider(BaseProvider):
     __metaclass__ = abc.ABCMeta
 
@@ -72,3 +109,30 @@ class BaseExportProvider(BaseProvider):
         to the provided export directory path using the given connection info.
         """
         pass
+
+
+class BaseReplicaExportProvider(BaseExportProvider):
+    __metaclass__ = abc.ABCMeta
+
+    @abc.abstractmethod
+    def get_replica_instance_info(self, ctxt, connection_info, instance_name):
+        pass
+
+    @abc.abstractmethod
+    def deploy_replica_source_resources(self, ctxt, connection_info):
+        pass
+
+    @abc.abstractmethod
+    def delete_replica_source_resources(self, ctxt, connection_info,
+                                        migr_resources_dict):
+        pass
+
+    @abc.abstractmethod
+    def replicate_disks(self, ctxt, connection_info, instance_name,
+                        source_conn_info, target_conn_info, volumes_info,
+                        incremental):
+        pass
+
+    @abc.abstractmethod
+    def shutdown_instance(self, ctxt, connection_info, instance_name):
+        pass

coriolis/providers/openstack/__init__.py

@@ -87,6 +87,9 @@ MIGR_USER_DATA = (
 MIGR_GUEST_USERNAME = 'cloudbase'
 MIGR_GUEST_USERNAME_WINDOWS = "admin"
 
+VOLUME_NAME_FORMAT = "%(instance_name)s %(num)s"
+REPLICA_VOLUME_NAME_FORMAT = "Coriolis Replica - %(instance_name)s %(num)s"
+
 LOG = logging.getLogger(__name__)
 
 
@@ -121,16 +124,96 @@ def _wait_for_instance(nova, instance, expected_status='ACTIVE'):
 
 
 @utils.retry_on_error()
-def _wait_for_volume(nova, volume, expected_status='in-use'):
-    volume = nova.volumes.findall(id=volume.id)[0]
+def _find_volume(cinder, volume_id):
+    volumes = cinder.volumes.findall(id=volume_id)
+    if volumes:
+        return volumes[0]
+
+
+@utils.retry_on_error()
+def _extend_volume(cinder, volume_id, new_size):
+    volume_size_gb = math.ceil(new_size / units.Gi)
+    cinder.volumes.extend(volume_id, volume_size_gb)
+
+
+@utils.retry_on_error()
+def _get_volume_from_snapshot(cinder, snapshot_id):
+    snapshot = cinder.volume_snapshots.get(snapshot_id)
+    return cinder.volumes.get(snapshot.volume_id)
+
+
+@utils.retry_on_error()
+def _create_volume(cinder, size, name, image_ref=None, snapshot_id=None):
+    if snapshot_id:
+        volume_size_gb = None
+    else:
+        volume_size_gb = math.ceil(size / units.Gi)
+    return cinder.volumes.create(
+        size=volume_size_gb,
+        name=name,
+        imageRef=image_ref,
+        snapshot_id=snapshot_id)
+
+
+@utils.retry_on_error()
+def _wait_for_volume(cinder, volume_id, expected_status='available'):
+    volumes = cinder.volumes.findall(id=volume_id)
+    if not volumes:
+        raise exception.CoriolisException("Volume not found")
+    volume = volumes[0]
+
     while volume.status not in [expected_status, 'error']:
         time.sleep(2)
-        volume = nova.volumes.get(volume.id)
+        volume = cinder.volumes.get(volume.id)
     if volume.status != expected_status:
         raise exception.CoriolisException(
             "Volume is in status: %s" % volume.status)
 
 
+@utils.retry_on_error()
+def _delete_volume(cinder, volume_id):
+    volumes = cinder.volumes.findall(id=volume_id)
+    for volume in volumes:
+        volume.delete()
+
+
+@utils.retry_on_error()
+def _create_volume_snapshot(cinder, volume_id, name):
+    return cinder.volume_snapshots.create(volume_id, name=name)
+
+
+@utils.retry_on_error()
+def _wait_for_volume_snapshot(cinder, snapshot_id,
+                              expected_status='available'):
+    snapshots = cinder.volume_snapshots.findall(id=snapshot_id)
+
+    if not snapshots:
+        if expected_status == 'deleted':
+            return
+        raise exception.CoriolisException("Volume snapshot not found")
+    snapshot = snapshots[0]
+
+    while snapshot.status not in [expected_status, 'error']:
+        time.sleep(2)
+        if expected_status == 'deleted':
+            snapshots = cinder.volume_snapshots.findall(id=snapshot_id)
+            if not snapshots:
+                return
+            snapshot = snapshots[0]
+        else:
+            snapshot = cinder.volume_snapshots.get(snapshot.id)
+    if snapshot.status != expected_status:
+        raise exception.CoriolisException(
+            "Volume snapshot is in status: %s" % snapshot.status)
+
+
+@utils.retry_on_error()
+def _delete_volume_snapshot(cinder, snapshot_id):
+    snapshots = cinder.volume_snapshots.findall(id=snapshot_id)
+    for snapshot in snapshots:
+        return cinder.volume_snapshots.delete(snapshot.id)
+
+
 class _MigrationResources(object):
     def __init__(self, nova, neutron, keypair, instance, port,
                  floating_ip, guest_port, sec_group, username, password, k):
@@ -146,6 +229,53 @@ class _MigrationResources(object):
         self._username = username
         self._password = password
 
+    def get_resources_dict(self):
+        return {
+            "instance_id": self._instance.id,
+            "keypair_name": self._keypair.name,
+            "port_id": self._port["id"],
+            "floating_ip_id": self._floating_ip.id,
+            "secgroup_id": self._sec_group.id,
+        }
+
+    @classmethod
+    @utils.retry_on_error()
+    def from_resources_dict(cls, nova, neutron, resources_dict):
+        instance_id = resources_dict["instance_id"]
+        keypair_name = resources_dict["keypair_name"]
+        floating_ip_id = resources_dict["floating_ip_id"]
+        secgroup_id = resources_dict["secgroup_id"]
+        port_id = resources_dict["port_id"]
+
+        instance = None
+        instances = nova.servers.findall(id=instance_id)
+        if instances:
+            instance = instances[0]
+
+        keypair = None
+        keypairs = nova.keypairs.findall(name=keypair_name)
+        if keypairs:
+            keypair = keypairs[0]
+
+        floating_ip = None
+        floating_ips = nova.floating_ips.findall(id=floating_ip_id)
+        if floating_ips:
+            floating_ip = floating_ips[0]
+
+        sec_group = None
+        sec_groups = nova.security_groups.findall(id=secgroup_id)
+        if sec_groups:
+            sec_group = sec_groups[0]
+
+        port = None
+        ports = neutron.list_ports(id=port_id)["ports"]
+        if ports:
+            port = ports[0]
+
+        return cls(
+            nova, neutron, keypair, instance, port, floating_ip, None,
+            sec_group, None, None, None)
+
     def get_guest_connection_info(self):
         return {
             "ip": self._floating_ip.ip,
@@ -188,7 +318,7 @@ class _MigrationResources(object):
             self._keypair = None
 
 
-class ImportProvider(base.BaseImportProvider):
+class ImportProvider(base.BaseReplicaImportProvider):
 
     connection_info_schema = schemas.get_schema(
         __name__, schemas.PROVIDER_CONNECTION_INFO_SCHEMA_NAME)
@@ -276,7 +406,7 @@ class ImportProvider(base.BaseImportProvider):
             os.close(fd)
             os.remove(key_path)
 
-    @utils.retry_on_error()
+    @utils.retry_on_error(max_attempts=10, sleep_seconds=30)
     def _deploy_migration_resources(self, nova, glance, neutron,
                                     os_type, migr_image_name, migr_flavor_name,
                                     migr_network_name, migr_fip_pool_name):
@@ -381,25 +511,29 @@ class ImportProvider(base.BaseImportProvider):
             raise
 
     @utils.retry_on_error()
-    def _attach_volume(self, nova, instance, volume, volume_dev=None):
-        nova.volumes.create_server_volume(
-            instance.id, volume.id, volume_dev)
-        _wait_for_volume(nova, volume, 'in-use')
+    def _attach_volume(self, nova, cinder, instance, volume_id,
+                       volume_dev=None):
+        # volume can be either a Volume object or an id
+        volume = nova.volumes.create_server_volume(
+            instance.id, volume_id, volume_dev)
+        _wait_for_volume(cinder, volume.id, 'in-use')
+        return volume
 
     def _get_import_config(self, target_environment, os_type):
         config = collections.namedtuple(
-            "glance_upload",
-            "target_disk_format",
-            "container_format",
-            "hypervisor_type",
-            "fip_pool_name",
-            "network_map",
-            "keypair_name",
-            "migr_image_name",
-            "migr_flavor_name",
-            "migr_fip_pool_name",
-            "migr_network_name",
-            "flavor_name")
+            "ImportConfig",
+            ["glance_upload",
+             "target_disk_format",
+             "container_format",
+             "hypervisor_type",
+             "fip_pool_name",
+             "network_map",
+             "keypair_name",
+             "migr_image_name",
+             "migr_flavor_name",
+             "migr_fip_pool_name",
+             "migr_network_name",
+             "flavor_name"])
 
         config.glance_upload = target_environment.get(
             "glance_upload", CONF.openstack_migration_provider.glance_upload)
@@ -450,8 +584,115 @@ class ImportProvider(base.BaseImportProvider):
 
         return config
 
-    def import_instance(self, ctxt, connection_info, target_environment,
-                        instance_name, export_info):
+    def _create_images_and_volumes(self, glance, nova, cinder, config,
+                                   disks_info):
+        if not config.glance_upload:
+            raise exception.CoriolisException(
+                "Glance upload is currently required for migrations")
+
+        images = []
+        volumes = []
+
+        for disk_info in disks_info:
+            disk_path = disk_info["path"]
+            disk_file_info = utils.get_disk_info(disk_path)
+
+            # if config.target_disk_format == disk_file_info["format"]:
+            #    target_disk_path = disk_path
+            # else:
+            #    target_disk_path = (
+            #        "%s.%s" % (os.path.splitext(disk_path)[0],
+            #                   config.target_disk_format))
+            #    utils.convert_disk_format(disk_path, target_disk_path,
+            #                              config.target_disk_format)
+
+            self._event_manager.progress_update(
+                "Uploading Glance image")
+
+            disk_format = disk_file_info["format"]
+            image = self._create_image(
+                glance, _get_unique_name(),
+                disk_path, disk_format,
+                config.container_format,
+                config.hypervisor_type)
+            images.append(image)
+
+            self._event_manager.progress_update(
+                "Waiting for Glance image to become active")
+            _wait_for_image(nova, image.id)
+
+            virtual_disk_size = disk_file_info["virtual-size"]
+            if disk_format != constants.DISK_FORMAT_RAW:
+                virtual_disk_size += DISK_HEADER_SIZE
+
+            self._event_manager.progress_update(
+                "Creating Cinder volume")
+
+            volume = _create_volume(
+                cinder, virtual_disk_size, _get_unique_name(), image.id)
+            volumes.append(volume)
+
+        return images, volumes
+
+    def _create_neutron_ports(self, neutron, config, nics_info):
+        ports = []
+
+        for nic_info in nics_info:
+            origin_network_name = nic_info.get("network_name")
+            if not origin_network_name:
+                self._warn("Origin network name not provided for for nic: "
+                           "%s, skipping", nic_info.get("name"))
+                continue
+
+            network_name = config.network_map.get(origin_network_name)
+            if not network_name:
+                raise exception.CoriolisException(
+                    "Network not mapped in network_map: %s" %
+                    origin_network_name)
+
+            ports.append(self._create_neutron_port(
+                neutron, network_name, nic_info.get("mac_address")))
+
+        return ports
+
+    @utils.retry_on_error()
+    def _get_replica_volumes(self, cinder, volumes_info):
+        volumes = []
+        for volume_id in [v["volume_id"] for v in volumes_info]:
+            volumes.append(cinder.volumes.get(volume_id))
+        return volumes
+
+    @utils.retry_on_error()
+    def _rename_volumes(self, cinder, volumes, instance_name):
+        for i, volume in enumerate(volumes):
+            new_volume_name = VOLUME_NAME_FORMAT % {
+                "instance_name": instance_name, "num": i + 1}
+            cinder.volumes.update(volume.id, name=new_volume_name)
+
+    @utils.retry_on_error()
+    def _set_bootable_volumes(self, cinder, volumes):
+        # TODO: check if just setting the first volume as bootable is enough
+        for volume in volumes:
+            if not volume.bootable or volume.bootable == 'false':
+                cinder.volumes.set_bootable(volume, True)
+
+    def _create_volumes_from_replica_snapshots(self, cinder, volumes_info):
+        volumes = []
+        for volume_info in volumes_info:
+            snapshot_id = volume_info["volume_snapshot_id"]
+            volume_name = _get_unique_name()
+
+            self._event_manager.progress_update(
+                "Creating Cinder volume from snapshot")
+
+            volume = _create_volume(
+                cinder, None, volume_name, snapshot_id=snapshot_id)
+            volumes.append(volume)
+        return volumes
+
+    def _deploy_instance(self, ctxt, connection_info, target_environment,
+                         instance_name, export_info, volumes_info=None,
+                         clone_disks=False):
         session = keystone.create_keystone_session(ctxt, connection_info)
 
         glance_api_version = connection_info.get("image_api_version",
@@ -467,55 +708,23 @@ class ImportProvider(base.BaseImportProvider):
 
         config = self._get_import_config(target_environment, os_type)
 
-        disks_info = export_info["devices"]["disks"]
-
         images = []
         volumes = []
         ports = []
 
         try:
-            if config.glance_upload:
-                for disk_info in disks_info:
-                    disk_path = disk_info["path"]
-                    disk_file_info = utils.get_disk_info(disk_path)
-
-                    # if config.target_disk_format == disk_file_info["format"]:
-                    #    target_disk_path = disk_path
-                    # else:
-                    #    target_disk_path = (
-                    #        "%s.%s" % (os.path.splitext(disk_path)[0],
-                    #                   config.target_disk_format))
-                    #    utils.convert_disk_format(disk_path, target_disk_path,
-                    #                              config.target_disk_format)
-
-                    self._event_manager.progress_update(
-                        "Uploading Glance image")
-
-                    disk_format = disk_file_info["format"]
-                    image = self._create_image(
-                        glance, _get_unique_name(),
-                        disk_path, disk_format,
-                        config.container_format,
-                        config.hypervisor_type)
-                    images.append(image)
-
-                    self._event_manager.progress_update(
-                        "Waiting for Glance image to become active")
-                    _wait_for_image(nova, image.id)
-
-                    virtual_disk_size = disk_file_info["virtual-size"]
-                    if disk_format != constants.DISK_FORMAT_RAW:
-                        virtual_disk_size += DISK_HEADER_SIZE
-
-                    self._event_manager.progress_update(
-                        "Creating Cinder volume")
-
-                    volume_size_gb = math.ceil(virtual_disk_size / units.Gi)
-                    volume = nova.volumes.create(
-                        size=volume_size_gb,
-                        display_name=_get_unique_name(),
-                        imageRef=image.id)
-                    volumes.append(volume)
+            if not volumes_info:
+                # Migration
+                disks_info = export_info["devices"]["disks"]
+                images, volumes = self._create_images_and_volumes(
+                    glance, nova, cinder, config, disks_info)
+            else:
+                # Migration from replica
+                if not clone_disks:
+                    volumes = self._get_replica_volumes(cinder, volumes_info)
+                else:
+                    volumes = self._create_volumes_from_replica_snapshots(
+                        cinder, volumes_info)
 
             migr_resources = self._deploy_migration_resources(
                 nova, glance, neutron, os_type, config.migr_image_name,
@@ -526,13 +735,13 @@ class ImportProvider(base.BaseImportProvider):
 
             try:
                 for i, volume in enumerate(volumes):
-                    _wait_for_volume(nova, volume, 'available')
+                    _wait_for_volume(cinder, volume.id)
 
                     self._event_manager.progress_update(
                         "Attaching volume to worker instance")
 
-                    self._attach_volume(nova, migr_resources.get_instance(),
-                                        volume)
+                    self._attach_volume(
+                        nova, cinder, migr_resources.get_instance(), volume.id)
 
                     conn_info = migr_resources.get_guest_connection_info()
 
@@ -553,44 +762,31 @@ class ImportProvider(base.BaseImportProvider):
                 migr_resources.delete()
 
             self._event_manager.progress_update("Renaming volumes")
+            self._rename_volumes(cinder, volumes, instance_name)
 
-            for i, volume in enumerate(volumes):
-                new_volume_name = "%s %s" % (instance_name, i + 1)
-                cinder.volumes.update(volume.id, name=new_volume_name)
-
-            for nic_info in nics_info:
-                self._event_manager.progress_update(
-                    "Creating Neutron port for migrated instance")
-
-                origin_network_name = nic_info.get("network_name")
-                if not origin_network_name:
-                    self._warn("Origin network name not provided for for nic: "
-                               "%s, skipping", nic_info.get("name"))
-                    continue
-
-                network_name = config.network_map.get(origin_network_name)
-                if not network_name:
-                    raise exception.CoriolisException(
-                        "Network not mapped in network_map: %s" %
-                        origin_network_name)
+            self._event_manager.progress_update(
+                "Ensuring volumes are bootable")
+            self._set_bootable_volumes(cinder, volumes)
 
-                ports.append(self._create_neutron_port(
-                    neutron, network_name, nic_info.get("mac_address")))
+            self._event_manager.progress_update(
+                "Creating Neutron ports for migrated instance")
+            ports = self._create_neutron_ports(neutron, config, nics_info)
 
             self._event_manager.progress_update(
                 "Creating migrated instance")
-
             self._create_target_instance(
                 nova, config.flavor_name, instance_name,
                 config.keypair_name, ports, volumes)
-        except Exception:
-            self._event_manager.progress_update("Deleting volumes")
-            for volume in volumes:
-                @utils.ignore_exceptions
-                @utils.retry_on_error()
-                def _del_volume():
-                    volume.delete()
-                _del_volume()
+        except:
+            if not volumes_info or clone_disks:
+                # Don't remove replica volumes
+                self._event_manager.progress_update("Deleting volumes")
+                for volume in volumes:
+                    @utils.ignore_exceptions
+                    @utils.retry_on_error()
+                    def _del_volume():
+                        volume.delete()
+                    _del_volume()
             self._event_manager.progress_update("Deleting Neutron ports")
             for port in ports:
                 @utils.ignore_exceptions
@@ -608,6 +804,11 @@ class ImportProvider(base.BaseImportProvider):
                     image.delete()
                 _del_image()
 
+    def import_instance(self, ctxt, connection_info, target_environment,
+                        instance_name, export_info):
+        self._deploy_instance(ctxt, connection_info, target_environment,
+                              instance_name, export_info)
+
     def _get_osmorphing_hypervisor_type(self, hypervisor_type):
         if (hypervisor_type and
                 hypervisor_type.lower() == constants.HYPERVISOR_QEMU):
@@ -615,7 +816,7 @@ class ImportProvider(base.BaseImportProvider):
         elif hypervisor_type:
             return hypervisor_type.lower()
 
-    @utils.retry_on_error()
+    @utils.retry_on_error(max_attempts=10, sleep_seconds=30)
     def _create_target_instance(self, nova, flavor_name, instance_name,
                                 keypair_name, ports, volumes):
         flavor = nova.flavors.find(name=flavor_name)
@@ -646,6 +847,256 @@ class ImportProvider(base.BaseImportProvider):
                 nova.servers.delete(instance)
             raise
 
+    def deploy_replica_instance(self, ctxt, connection_info,
+                                target_environment, instance_name, export_info,
+                                volumes_info, clone_disks):
+        self._deploy_instance(ctxt, connection_info, target_environment,
+                              instance_name, export_info, volumes_info,
+                              clone_disks)
+
+    def _update_existing_disk_volumes(self, cinder, disks_info, volumes_info):
+        for disk_info in disks_info:
+            disk_id = disk_info["id"]
+
+            vi = [v for v in volumes_info
+                  if v["disk_id"] == disk_id and v.get("volume_id")]
+            if vi:
+                volume_info = vi[0]
+                volume_id = volume_info["volume_id"]
+
+                volume = _find_volume(cinder, volume_id)
+                if volume:
+                    virtual_disk_size_gb = math.ceil(
+                        disk_info["size_bytes"] / units.Gi)
+
+                    if virtual_disk_size_gb > volume.size:
+                        LOG.info(
+                            "Extending volume %(volume_id)s. "
+                            "Current size: %(curr_size)s GB, "
+                            "Requested size: %(requested_size)s GB",
+                            {"volume_id": volume_id,
+                             "curr_size": virtual_disk_size_gb,
+                             "requested_size": volume.size})
+                        self._event_manager.progress_update("Extending volume")
+                        _extend_volume(
+                            cinder, volume_id, virtual_disk_size_gb * units.Gi)
+                    elif virtual_disk_size_gb < volume.size:
+                        LOG.warning(
+                            "Cannot shrink volume %(volume_id)s. "
+                            "Current size: %(curr_size)s GB, "
+                            "Requested size: %(requested_size)s GB",
+                            {"volume_id": volume_id,
+                             "curr_size": volume.size,
+                             "requested_size": virtual_disk_size_gb})
+                else:
+                    volumes_info.remove(volume_info)
+
+        return volumes_info
+
+    def _delete_removed_disk_volumes(self, cinder, disks_info, volumes_info):
+        for volume_info in volumes_info:
+            if volume_info["disk_id"] not in [
+                    d["id"] for d in disks_info if d["id"]]:
+
+                volume_id = volume_info["volume_id"]
+                volume = _find_volume(cinder, volume_id)
+                if volume:
+                    self._event_manager.progress_update("Deleting volume")
+                    _delete_volume(cinder, volume_id)
+                volumes_info.remove(volume_info)
+        return volumes_info
+
+    def _create_new_disk_volumes(self, cinder, disks_info, volumes_info,
+                                 instance_name):
+        try:
+            new_volumes = []
+            for i, disk_info in enumerate(disks_info):
+                disk_id = disk_info["id"]
+                virtual_disk_size = disk_info["size_bytes"]
+
+                if not [v for v in volumes_info if v["disk_id"] == disk_id]:
+                    self._event_manager.progress_update(
+                        "Creating volume")
+
+                    volume_name = REPLICA_VOLUME_NAME_FORMAT % {
+                        "instance_name": instance_name, "num": i + 1}
+                    volume = _create_volume(
+                        cinder, virtual_disk_size, volume_name)
+
+                    new_volumes.append(volume)
+                    volumes_info.append({
+                        "volume_id": volume.id,
+                        "disk_id": disk_id})
+                else:
+                    self._event_manager.progress_update(
+                        "Using previously deployed volume")
+
+            for volume in new_volumes:
+                _wait_for_volume(cinder, volume.id)
+
+            return volumes_info
+        except:
+            for volume in new_volumes:
+                _delete_volume(cinder, volume.id)
+            raise
+
+    def deploy_replica_disks(self, ctxt, connection_info, target_environment,
+                             instance_name, export_info, volumes_info):
+        session = keystone.create_keystone_session(ctxt, connection_info)
+
+        cinder = cinder_client.Client(CINDER_API_VERSION, session=session)
+
+        disks_info = export_info["devices"]["disks"]
+
+        volumes_info = self._update_existing_disk_volumes(
+            cinder, disks_info, volumes_info)
+
+        volumes_info = self._delete_removed_disk_volumes(
+            cinder, disks_info, volumes_info)
+
+        volumes_info = self._create_new_disk_volumes(
+            cinder, disks_info, volumes_info, instance_name)
+
+        return volumes_info
+
+    def deploy_replica_target_resources(self, ctxt, connection_info,
+                                        target_environment, volumes_info):
+        session = keystone.create_keystone_session(ctxt, connection_info)
+
+        glance_api_version = connection_info.get("image_api_version",
+                                                 GLANCE_API_VERSION)
+        nova = nova_client.Client(NOVA_API_VERSION, session=session)
+        glance = glance_client.Client(glance_api_version, session=session)
+        neutron = neutron_client.Client(NEUTRON_API_VERSION, session=session)
+        cinder = cinder_client.Client(CINDER_API_VERSION, session=session)
+
+        # Data migration uses a Linux guest binary
+        os_type = constants.OS_TYPE_LINUX
+
+        config = self._get_import_config(target_environment, os_type)
+
+        migr_resources = self._deploy_migration_resources(
+            nova, glance, neutron, os_type, config.migr_image_name,
+            config.migr_flavor_name, config.migr_network_name,
+            config.migr_fip_pool_name)
+
+        try:
+            for i, volume_info in enumerate(volumes_info):
+                self._event_manager.progress_update(
+                    "Attaching volume to worker instance")
+
+                volume_id = volume_info["volume_id"]
+                ret_volume = self._attach_volume(
+                    nova, cinder, migr_resources.get_instance(), volume_id)
+                volume_info["volume_dev"] = ret_volume.device
+
+            return {
+                "migr_resources": migr_resources.get_resources_dict(),
+                "volumes_info": volumes_info,
+                "connection_info": migr_resources.get_guest_connection_info(),
+            }
+        except:
+            self._event_manager.progress_update(
+                "Removing worker instance resources")
+            migr_resources.delete()
+            raise
+
+    def delete_replica_target_resources(self, ctxt, connection_info,
+                                        migr_resources_dict):
+        session = keystone.create_keystone_session(ctxt, connection_info)
+
+        nova = nova_client.Client(NOVA_API_VERSION, session=session)
+        neutron = neutron_client.Client(NEUTRON_API_VERSION, session=session)
+
+        migr_resources = _MigrationResources.from_resources_dict(
+            nova, neutron, migr_resources_dict)
+        self._event_manager.progress_update(
+            "Removing worker instance resources")
+        migr_resources.delete()
+
+    def delete_replica_disks(self, ctxt, connection_info, volumes_info):
+        session = keystone.create_keystone_session(ctxt, connection_info)
+
+        cinder = cinder_client.Client(CINDER_API_VERSION, session=session)
+
+        self._event_manager.progress_update(
+            "Removing replica disk volumes")
+        for volume_info in volumes_info:
+            _delete_volume(cinder, volume_info["volume_id"])
+
+    def create_replica_disk_snapshots(self, ctxt, connection_info,
+                                      volumes_info):
+        session = keystone.create_keystone_session(ctxt, connection_info)
+
+        cinder = cinder_client.Client(CINDER_API_VERSION, session=session)
+
+        snapshots = []
+        self._event_manager.progress_update(
+            "Creating replica disk snapshots")
+        for volume_info in volumes_info:
+            snapshot = _create_volume_snapshot(
+                cinder, volume_info["volume_id"], _get_unique_name())
+            snapshots.append(snapshot)
+            volume_info["volume_snapshot_id"] = snapshot.id
+
+        for snapshot in snapshots:
+            _wait_for_volume_snapshot(cinder, snapshot.id)
+
+        return volumes_info
+
+    def delete_replica_disk_snapshots(self, ctxt, connection_info,
+                                      volumes_info):
+        session = keystone.create_keystone_session(ctxt, connection_info)
+
+        cinder = cinder_client.Client(CINDER_API_VERSION, session=session)
+
+        self._event_manager.progress_update(
+            "Removing replica disk snapshots")
+        for volume_info in volumes_info:
+            snapshot_id = volume_info.get("volume_snapshot_id")
+            if snapshot_id:
+                _delete_volume_snapshot(cinder, snapshot_id)
+                _wait_for_volume_snapshot(cinder, snapshot_id, 'deleted')
+                volume_info["volume_snapshot_id"] = None
+
+    def restore_replica_disk_snapshots(self, ctxt, connection_info,
+                                       volumes_info):
+        session = keystone.create_keystone_session(ctxt, connection_info)
+
+        cinder = cinder_client.Client(CINDER_API_VERSION, session=session)
+
+        self._event_manager.progress_update(
+            "Restoring replica disk snapshots")
+
+        new_volumes = []
+        try:
+            for volume_info in volumes_info:
+                snapshot_id = volume_info.get("volume_snapshot_id")
+                if snapshot_id:
+                    original_volume = _get_volume_from_snapshot(
+                        cinder, snapshot_id)
+
+                    volume_name = original_volume.name
+                    volume = _create_volume(
+                        cinder, None, volume_name, snapshot_id=snapshot_id)
+                    new_volumes.append((volume_info, snapshot_id, volume))
+
+            for volume_info, snapshot_id, volume in new_volumes:
+                old_volume_id = volume_info["volume_id"]
+                _wait_for_volume(cinder, volume.id)
+                _delete_volume_snapshot(cinder, snapshot_id)
+                _wait_for_volume_snapshot(cinder, snapshot_id, 'deleted')
+                _delete_volume(cinder, old_volume_id)
+
+                volume_info["volume_id"] = volume.id
+                volume_info["volume_snapshot_id"] = None
+        except:
+            for _, _, volume in new_volumes:
+                _delete_volume(cinder, volume.id)
+            raise
+
+        return volumes_info
+
 
 class ExportProvider(base.BaseExportProvider):
     _OS_DISTRO_MAP = {

coriolis/providers/openstack/schemas/connection_info_schema.json

@@ -1,50 +1,64 @@
 {
   "$schema": "http://cloudbase.it/coriolis/schemas/openstack_connection#",
-  "type": "object",
-  "properties": {
-    "secret_ref": {
-      "type": "string"
-    },
-    "identity_api_version": {
-      "type": "integer"
-    },
-    "username": {
-      "type": "string"
-    },
-    "password": {
-      "type": "string"
-    },
-    "project_name": {
-      "type": "string"
-    },
-    "user_domain_name": {
-      "type": "string"
-    },
-    "project_domain_name": {
-      "type": "string"
-    },
-    "auth_url": {
-      "type": "string"
-    },
-    "allow_untrusted": {
-      "type": "boolean",
-      "default": false
-    }
-  },
   "oneOf": [
     {
-      "required": ["secret_ref"]
-    },
-    {
+      "type": "object",
+      "properties": {
+        "identity_api_version": {
+          "type": "integer",
+          "minimum": 2,
+          "maximum": 3
+        },
+        "username": {
+          "type": "string"
+        },
+        "password": {
+          "type": "string"
+        },
+        "project_name": {
+          "type": "string"
+        },
+        "user_domain_name": {
+          "type": "string"
+        },
+        "project_domain_name": {
+          "type": "string"
+        },
+        "auth_url": {
+          "type": "string"
+        },
+        "allow_untrusted": {
+          "type": "boolean",
+          "default": false
+        }
+      },
       "required": [
-        "identity_api_version",
         "username",
         "password",
         "project_name",
-        "user_domain_name",
-        "project_domain_name",
         "auth_url"
-      ]
+      ],
+      "additionalProperties": false
+    },
+    {
+      "type": "object",
+      "properties": {
+        "secret_ref": {
+          "type": "string",
+          "format": "uri"
+        }
+      },
+      "required": ["secret_ref"],
+      "additionalProperties": false
+    },
+    {
+      "type": "object",
+      "properties": {
+      },
+      "additionalProperties": false
+    },
+    {
+      "type": "null"
     }
   ]
 }

coriolis/providers/openstack/schemas/target_environment_schema.json

@@ -1,50 +1,51 @@
 {
   "$schema": "http://cloudbase.it/coriolis/schemas/openstack_target_environment#",
-  "type": "object",
-  "properties": {
-    "secret_ref": {
-      "type": "string"
-    },
-    "network_map": {
+  "oneOf": [
+    {
       "type": "object",
       "properties": {
-        "VM Network Local": {
+        "network_map": {
+          "type": "object"
+        },
+        "glance_upload": {
+          "type": "boolean"
+        },
+        "disk_format": {
+          "type": "string"
+        },
+        "container_format": {
+          "type": "string"
+        },
+        "hypervisor_type": {
+          "type": "string"
+        },
+        "migr_image_name": {
+          "type": "string"
+        },
+        "migr_image_name_map": {
+          "type": "object"
+        },
+        "migr_flavor_name": {
+          "type": "string"
+        },
+        "flavor_name": {
+          "type": "string"
+        },
+        "fip_pool_name": {
+          "type": "string"
+        },
+        "migr_fip_pool_name": {
           "type": "string"
         },
-        "VM Network": {
+        "keypair_name": {
           "type": "string"
         }
       },
-      "required": [
-        "VM Network Local",
-        "VM Network"
-      ]
-    },
-    "flavor_name": {
-      "type": "string"
-    },
-    "fip_pool_name": {
-      "type": "string"
-    },
-    "migr_fip_pool_name": {
-      "type": "string"
-    },
-    "keypair_name": {
-      "type": "string"
-    }
-  },
-  "oneOf": [
-    {
-      "required": ["secret_ref"]
-    },
-    {
       "required": [
         "network_map",
-        "flavor_name",
-        "fip_pool_name",
-        "migr_fip_pool_name",
-        "keypair_name"
-      ]
+        "flavor_name"
+      ],
+      "additionalProperties": false
     }
   ]
 }

coriolis/providers/vmware_vsphere/__init__.py

@@ -1,10 +1,13 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+import abc
 import contextlib
 import os
 import re
+import struct
 import sys
+import threading
 import time
 from urllib import request
 import uuid
@@ -12,10 +15,13 @@ import uuid
 import eventlet
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_utils import units
+import paramiko
 from pyVim import connect
 from pyVmomi import vim
 
 from coriolis import constants
+from coriolis import data_transfer
 from coriolis import exception
 from coriolis.providers import base
 from coriolis.providers.vmware_vsphere import guestid
@@ -34,8 +40,163 @@ CONF.register_opts(vmware_vsphere_opts, 'vmware_vsphere')
 
 LOG = logging.getLogger(__name__)
 
+vixdisklib.init()
 
-class ExportProvider(base.BaseExportProvider):
+
+class _BaseBackupWriter(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def _open(self):
+        pass
+
+    @contextlib.contextmanager
+    def open(self, path, disk_id):
+        self._path = path
+        self._disk_id = disk_id
+        self._open()
+        try:
+            yield self
+        finally:
+            self.close()
+
+    @abc.abstractmethod
+    def seek(self, pos):
+        pass
+
+    @abc.abstractmethod
+    def truncate(self, size):
+        pass
+
+    @abc.abstractmethod
+    def write(self, data):
+        pass
+
+    @abc.abstractmethod
+    def close(self):
+        pass
+
+
+class _FileBackupWriter(_BaseBackupWriter):
+    def _open(self):
+        # Create file if it doesnt exist
+        open(self._path, 'ab+').close()
+        self._file = open(self._path, 'rb+')
+
+    def seek(self, pos):
+        self._file.seek(pos)
+
+    def truncate(self, size):
+        self._file.truncate(size)
+
+    def write(self, data):
+        self._file.write(data)
+
+    def close(self):
+        self._file.close()
+
+
+class _SSHBackupWriter(_BaseBackupWriter):
+    def __init__(self, ip, port, username, pkey, password, volumes_info):
+        self._ip = ip
+        self._port = port
+        self._username = username
+        self._pkey = pkey
+        self._password = password
+        self._volumes_info = volumes_info
+        self._ssh = None
+
+    @contextlib.contextmanager
+    def open(self, path, disk_id):
+        self._path = path
+        self._disk_id = disk_id
+        self._open()
+        try:
+            yield self
+            # Don't send a message via ssh on exception
+            self.close()
+        except:
+            self._ssh.close()
+            raise
+
+    @utils.retry_on_error()
+    def _connect_ssh(self):
+        LOG.info("Connecting to SSH host: %(ip)s:%(port)s" %
+                 {"ip": self._ip, "port": self._port})
+        self._ssh = paramiko.SSHClient()
+        self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        self._ssh.connect(
+            hostname=self._ip,
+            port=self._port,
+            username=self._username,
+            pkey=self._pkey,
+            password=self._password)
+
+    @utils.retry_on_error()
+    def _copy_helper_cmd(self):
+        sftp = self._ssh.open_sftp()
+        local_path = os.path.join(
+            utils.get_resources_dir(), 'write_data')
+        sftp.put(local_path, 'write_data')
+        sftp.close()
+
+    @utils.retry_on_error()
+    def _exec_helper_cmd(self):
+        self._msg_id = 0
+        self._offset = 0
+        self._stdin, self._stdout, self._stderr = self._ssh.exec_command(
+            "chmod +x write_data && sudo ./write_data")
+
+    def _encode_data(self, content):
+        path = [v for v in self._volumes_info
+                if v["disk_id"] == self._disk_id][0]["volume_dev"]
+
+        msg = data_transfer.encode_data(
+            self._msg_id, path, self._offset, content)
+
+        LOG.debug(
+            "Guest path: %(path)s, offset: %(offset)d, content len: "
+            "%(content_len)d, msg len: %(msg_len)d",
+            {"path": path, "offset": self._offset, "content_len": len(content),
+             "msg_len": len(msg)})
+        return msg
+
+    def _encode_eod(self):
+        msg = data_transfer.encode_eod(self._msg_id)
+        LOG.debug("EOD message len: %d", len(msg))
+        return msg
+
+    @utils.retry_on_error()
+    def _send_msg(self, data):
+        self._msg_id += 1
+        self._stdin.write(data)
+        self._stdin.flush()
+        out_msg_id = self._stdout.read(4)
+
+    def _open(self):
+        self._connect_ssh()
+        self._copy_helper_cmd()
+        self._exec_helper_cmd()
+
+    def seek(self, pos):
+        self._offset = pos
+
+    def truncate(self, size):
+        pass
+
+    def write(self, data):
+        self._send_msg(self._encode_data(data))
+        self._offset += len(data)
+
+    def close(self):
+        self._send_msg(self._encode_eod())
+        ret_val = self._stdout.channel.recv_exit_status()
+        if ret_val:
+            raise exception.CoriolisException(
+                "An exception occurred while writing data on target. "
+                "Error code: %s" % ret_val)
+        self._ssh.close()
+
+
+class ExportProvider(base.BaseReplicaExportProvider):
 
     connection_info_schema = schemas.get_schema(
         __name__, schemas.PROVIDER_CONNECTION_INFO_SCHEMA_NAME)
@@ -53,16 +214,57 @@ class ExportProvider(base.BaseExportProvider):
         if task.info.state == vim.TaskInfo.State.error:
             raise exception.CoriolisException(task.info.error.msg)
 
+    @staticmethod
+    def _keep_alive_vmware_conn(si, exit_event):
+        try:
+            while True:
+                LOG.debug("VMware connection keep alive")
+                si.CurrentTime()
+                if exit_event.wait(60):
+                    return
+        finally:
+            LOG.debug("Exiting VMware connection keep alive thread")
+
     @utils.retry_on_error()
-    def _connect(self, host, username, password, port, context):
+    @contextlib.contextmanager
+    def _connect(self, connection_info):
+        host = connection_info["host"]
+        port = connection_info.get("port", 443)
+        username = connection_info["username"]
+        password = connection_info["password"]
+        allow_untrusted = connection_info.get("allow_untrusted", False)
+
+        # pyVmomi locks otherwise
+        sys.modules['socket'] = eventlet.patcher.original('socket')
+        ssl = eventlet.patcher.original('ssl')
+
+        context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        if allow_untrusted:
+            context.verify_mode = ssl.CERT_NONE
+
         LOG.info("Connecting to: %s:%s" % (host, port))
-        return connect.SmartConnect(
+        si = connect.SmartConnect(
             host=host,
             user=username,
             pwd=password,
             port=port,
             sslContext=context)
 
+        thread = None
+        try:
+            thread_exit_event = threading.Event()
+            thread = threading.Thread(
+                target=self._keep_alive_vmware_conn,
+                args=(si, thread_exit_event))
+            thread.start()
+
+            yield context, si
+        finally:
+            connect.Disconnect(si)
+            if thread:
+                thread_exit_event.set()
+                thread.join()
+
     def _wait_for_vm_status(self, vm, status, max_wait=120):
         i = 0
         while i < max_wait and vm.runtime.powerState != status:
@@ -84,7 +286,7 @@ class ExportProvider(base.BaseExportProvider):
             else:
                 container = container.childEntity[0].vmFolder
 
-        LOG.debug("VM path items:", path_items)
+        LOG.debug("VM path items: %s", path_items)
         for i, path_item in enumerate(path_items):
             l = [o for o in container.childEntity if o.name == path_item]
             if not l:
@@ -177,7 +379,9 @@ class ExportProvider(base.BaseExportProvider):
             disks.append({'size_bytes': device.capacityInBytes,
                           'unit_number': device.unitNumber,
                           'id': device.key,
-                          'controller_id': device.controllerKey})
+                          'controller_id': device.controllerKey,
+                          'path': device.backing.fileName,
+                          'format': constants.DISK_FORMAT_VMDK})
 
         cdroms = []
         devices = [d for d in vm.config.hardware.device if
@@ -190,7 +394,8 @@ class ExportProvider(base.BaseExportProvider):
         devices = [d for d in vm.config.hardware.device if
                    isinstance(d, vim.vm.device.VirtualFloppy)]
         for device in devices:
-            floppies.append({'unit_number': device.unitNumber, 'id': device.key,
+            floppies.append({'unit_number': device.unitNumber,
+                             'id': device.key,
                              'controller_id': device.controllerKey})
 
         nics = []
@@ -327,17 +532,18 @@ class ExportProvider(base.BaseExportProvider):
     @contextlib.contextmanager
     def _take_temp_vm_snapshot(self, vm, snapshot_name, memory=False,
                                quiesce=True):
-        self._event_manager.progress_update("Creating backup snapshot")
+        self._event_manager.progress_update("Creating snapshot")
         snapshot = self._take_vm_snapshot(vm, snapshot_name, memory, quiesce)
         try:
             yield snapshot
         finally:
-            self._event_manager.progress_update("Removing backup snapshot")
+            self._event_manager.progress_update("Removing snapshot")
             self._remove_vm_snapshot(snapshot)
 
     @utils.retry_on_error()
     def _backup_snapshot_disks(self, snapshot, export_path, connection_info,
-                               context, disk_paths):
+                               context, disk_paths, backup_writer,
+                               incremental):
         vm = snapshot.vm
         vmx_spec = "moref=%s" % vm._GetMoId()
         snapshot_ref = snapshot._GetMoId()
@@ -349,37 +555,68 @@ class ExportProvider(base.BaseExportProvider):
                                       vmx_spec, snapshot_ref) as conn:
             for disk in [d for d in snapshot.config.hardware.device
                          if isinstance(d, vim.vm.device.VirtualDisk)]:
+                change_id = '*'
 
                 l = [d for d in disk_paths if d['id'] == disk.key]
                 if l:
                     disk_path = l[0]
-                    change_id = disk_path["change_id"]
+                    if incremental:
+                        change_id = disk_path["change_id"]
                     path = disk_path["path"]
-                    disk_path['change_id'] = disk.backing.changeId
                 else:
-                    change_id = '*'
                     path = os.path.join(export_path, "disk-%s.raw" % disk.key)
-                    disk_paths.append({
+                    disk_path = {
                         'path': path,
                         'id': disk.key,
-                        'format': constants.DISK_FORMAT_RAW,
-                        'change_id': disk.backing.changeId})
+                        'format': constants.DISK_FORMAT_RAW}
+                    disk_paths.append(disk_path)
 
-                LOG.debug("CBT change id: %s", change_id)
+                LOG.info("CBT change id: %s", change_id)
                 changed_disk_areas = vm.QueryChangedDiskAreas(
-                    snapshot_ref, disk.key, pos, change_id)
+                    snapshot, disk.key, pos, change_id)
 
                 backup_disk_path = disk.backing.fileName
+
+                disk_size = changed_disk_areas.length
+                changed_area_size = sum(
+                    [x.length for x in changed_disk_areas.changedArea])
+
+                if not changed_area_size:
+                    self._event_manager.progress_update(
+                        "No blocks to replicate for disk: %s" %
+                        backup_disk_path)
+                    continue
+
+                if change_id == '*':
+                    self._event_manager.progress_update(
+                        "Performing full CBT replica for disk: {path}. "
+                        "Disk size: {disk_size:,}. Written blocks size: "
+                        "{changed_area_size:,}".format(
+                            path=backup_disk_path,
+                            disk_size=disk_size,
+                            changed_area_size=changed_area_size))
+                else:
+                    self._event_manager.progress_update(
+                        "Performing incremental CBT replica for disk: {path}. "
+                        "Disk size: {disk_size:,}. Changed blocks size: "
+                        "{changed_area_size:,}".format(
+                            path=backup_disk_path,
+                            disk_size=disk_size,
+                            changed_area_size=changed_area_size))
+
                 with vixdisklib.open(
                         conn, backup_disk_path) as disk_handle:
 
-                    # Create file if it doesn't exist
-                    open(path, "ab").close()
-
-                    with open(path, "rb+") as f:
+                    with backup_writer.open(path, disk.key) as f:
                         # Create a sparse file
                         f.truncate(disk.capacityInBytes)
 
+                        total_written_bytes = 0
+                        perc_step = self._event_manager.add_percentage_step(
+                            changed_area_size,
+                            message_format="Disk %s replica progress: "
+                            "{:.0f}%%" % backup_disk_path)
+
                         for area in changed_disk_areas.changedArea:
                             start_sector = area.start // sector_size
                             num_sectors = area.length // sector_size
@@ -391,8 +628,12 @@ class ExportProvider(base.BaseExportProvider):
                                 curr_num_sectors = min(
                                     num_sectors - i, max_sectors_per_read)
 
-                                buf = vixdisklib.get_buffer(
-                                    curr_num_sectors * sector_size)
+                                buf_size = curr_num_sectors * sector_size
+                                buf = vixdisklib.get_buffer(buf_size)
+
+                                LOG.debug(
+                                    "Read start sector: %s, num sectors: %s" %
+                                    (start_sector + i, curr_num_sectors))
 
                                 vixdisklib.read(
                                     disk_handle, start_sector + i,
@@ -401,71 +642,53 @@ class ExportProvider(base.BaseExportProvider):
 
                                 f.write(buf.raw)
 
+                                total_written_bytes += buf_size
+                                self._event_manager.set_percentage_step(
+                                    perc_step, total_written_bytes)
+
+                disk_path['change_id'] = disk.backing.changeId
+
     def _backup_disks(self, vm, export_path, connection_info, context):
         if not vm.config.changeTrackingEnabled:
             raise exception.CoriolisException("Change Tracking not enabled")
 
         disk_paths = []
-        vixdisklib.init()
-        try:
-            LOG.info("First backup pass")
-            snapshot_name = str(uuid.uuid4())
-            with self._take_temp_vm_snapshot(vm, snapshot_name) as snapshot:
-                self._backup_snapshot_disks(
-                    snapshot, export_path, connection_info, context,
-                    disk_paths)
 
-            self._shutdown_vm(vm)
+        LOG.info("First backup pass")
+        snapshot_name = str(uuid.uuid4())
+        with self._take_temp_vm_snapshot(vm, snapshot_name) as snapshot:
+            self._backup_snapshot_disks(
+                snapshot, export_path, connection_info, context,
+                disk_paths, _FileBackupWriter(), incremental=False)
 
-            LOG.info("Second backup pass")
-            snapshot_name = str(uuid.uuid4())
-            with self._take_temp_vm_snapshot(vm, snapshot_name) as snapshot:
-                self._backup_snapshot_disks(
-                    snapshot, export_path, connection_info, context,
-                    disk_paths)
+        self._shutdown_vm(vm)
 
-            return disk_paths
-        finally:
-            vixdisklib.exit()
+        LOG.info("Second backup pass")
+        snapshot_name = str(uuid.uuid4())
+        with self._take_temp_vm_snapshot(vm, snapshot_name) as snapshot:
+            self._backup_snapshot_disks(
+                snapshot, export_path, connection_info, context,
+                disk_paths, _FileBackupWriter(), incremental=True)
+
+        return disk_paths
 
     def export_instance(self, ctxt, connection_info, instance_name,
                         export_path):
-        host = connection_info["host"]
-        port = connection_info.get("port", 443)
-        username = connection_info["username"]
-        password = connection_info["password"]
-        allow_untrusted = connection_info.get("allow_untrusted", False)
-
-        backup_disks = False
-
-        # pyVmomi locks otherwise
-        sys.modules['socket'] = eventlet.patcher.original('socket')
-        ssl = eventlet.patcher.original('ssl')
-
-        context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
-        if allow_untrusted:
-            context.verify_mode = ssl.CERT_NONE
-
-        self._event_manager.set_total_progress_steps(4)
-
         self._event_manager.progress_update("Connecting to vSphere host")
-        si = self._connect(host, username, password, port, context)
-        try:
+        with self._connect(connection_info) as (context, si):
             self._event_manager.progress_update(
                 "Retrieving virtual machine data")
             vm_info, vm = self._get_vm_info(si, instance_name)
-            self._event_manager.progress_update("Exporting disks")
 
             # Take advantage of CBT if available
             backup_disks = vm.config.changeTrackingEnabled
 
+            self._event_manager.progress_update("Exporting disks")
             if backup_disks:
                 disk_paths = self._backup_disks(
                     vm, export_path, connection_info, context)
             else:
                 disk_paths = self._export_disks(vm, export_path, context)
-        finally:
-            connect.Disconnect(si)
 
         if not backup_disks:
             self._event_manager.progress_update(
@@ -486,3 +709,69 @@ class ExportProvider(base.BaseExportProvider):
             disk_info["path"] = os.path.abspath(disk_path["path"])
 
         return vm_info
+
+    def get_replica_instance_info(self, ctxt, connection_info, instance_name):
+        self._event_manager.progress_update("Connecting to vSphere host")
+        with self._connect(connection_info) as (context, si):
+            self._event_manager.progress_update(
+                "Retrieving virtual machine data")
+            vm_info, vm = self._get_vm_info(si, instance_name)
+
+            if not vm.config.changeTrackingEnabled:
+                raise exception.CoriolisException(
+                    "Changed Block Tracking must be enabled in order to "
+                    "replicate a VM")
+
+        return vm_info
+
+    def shutdown_instance(self, ctxt, connection_info, instance_name):
+        self._event_manager.progress_update("Connecting to vSphere host")
+        with self._connect(connection_info) as (context, si):
+            vm = self._get_vm(si, instance_name)
+            self._shutdown_vm(vm)
+
+    def replicate_disks(self, ctxt, connection_info, instance_name,
+                        source_conn_info, target_conn_info, volumes_info,
+                        incremental):
+        ip = target_conn_info["ip"]
+        port = target_conn_info.get("port", 22)
+        username = target_conn_info["username"]
+        pkey = target_conn_info.get("pkey")
+        password = target_conn_info.get("password")
+
+        LOG.info("Waiting for connectivity on host: %(ip)s:%(port)s",
+                 {"ip": ip, "port": port})
+        utils.wait_for_port_connectivity(ip, port)
+
+        with self._connect(connection_info) as (context, si):
+            vm = self._get_vm(si, instance_name)
+
+            backup_writer = _SSHBackupWriter(
+                ip, port, username, pkey, password, volumes_info)
+
+            snapshot_name = str(uuid.uuid4())
+            disk_paths = []
+            for volume_info in volumes_info:
+                disk_paths.append(
+                    {"id": volume_info["disk_id"],
+                     "change_id": volume_info.get("change_id", "*"),
+                     "path": ""})
+
+            with self._take_temp_vm_snapshot(vm, snapshot_name) as snapshot:
+                self._backup_snapshot_disks(
+                    snapshot, "", connection_info, context,
+                    disk_paths, backup_writer, incremental)
+
+            for volume_info in volumes_info:
+                change_id = [d["change_id"] for d in disk_paths
+                             if d["id"] == volume_info["disk_id"]][0]
+                volume_info["change_id"] = change_id
+
+        return volumes_info
+
+    def deploy_replica_source_resources(self, ctxt, connection_info):
+        return {"migr_resources": None, "connection_info": None}
+
+    def delete_replica_source_resources(self, ctxt, connection_info,
+                                        migr_resources_dict):
+        pass

coriolis/providers/vmware_vsphere/schemas/connection_info_schema.json

@@ -1,38 +1,42 @@
 {
   "$schema": "http://cloudbase.it/coriolis/schemas/vmware_vsphere_connection#",
-  "type": "object",
-  "properties": {
-    "secret_ref": {
-      "type": "string"
-    },
-    "host": {
-      "type": "string"
-    },
-    "port": {
-      "type": "integer"
-    },
-    "username": {
-      "type": "string"
-    },
-    "password": {
-      "type": "string"
-    },
-    "allow_untrusted": {
-      "type": "boolean",
-      "default": false
-    }
-  },
   "oneOf": [
     {
-      "required": ["secret_ref"]
-    },
-    {
+      "type": "object",
+      "properties": {
+        "host": {
+          "type": "string"
+        },
+        "port": {
+          "type": "integer"
+        },
+        "username": {
+          "type": "string"
+        },
+        "password": {
+          "type": "string"
+        },
+        "allow_untrusted": {
+          "type": "boolean"
+        }
+      },
       "required": [
         "host",
         "port",
         "username",
         "password"
-      ]
+      ],
+      "additionalProperties": false
+    },
+    {
+      "type": "object",
+      "properties": {
+        "secret_ref": {
+          "type": "string"
+        }
+      },
+      "required": ["secret_ref"],
+      "additionalProperties": false
     }
   ]
 }

coriolis/providers/vmware_vsphere/vixdisklib.py

@@ -5,6 +5,10 @@ import contextlib
 import ctypes
 import os
 
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__)
+
 if os.name == 'nt':
     vixDiskLibName = 'vixDiskLib.dll'
 else:
@@ -144,6 +148,8 @@ def get_transport_modes():
 @contextlib.contextmanager
 def connect(server_name, thumbprint, username, password, vmx_spec=None,
             snapshot_ref=None, read_only=True, transport_modes=None, port=443):
+    LOG.debug("Connecting VixDiskLib: %s", server_name)
+
     connectParams = VixDiskLibConnectParams()
 
     connectParams.serverName = server_name.encode()
@@ -175,6 +181,8 @@ def connect(server_name, thumbprint, username, password, vmx_spec=None,
 
 @contextlib.contextmanager
 def open(conn, disk_path, flags=VIXDISKLIB_FLAG_OPEN_READ_ONLY):
+    LOG.debug("Openning VixDiskLib disk: %s", disk_path)
+
     disk_handle = ctypes.c_void_p()
     _check_err(vixDiskLib.VixDiskLib_Open(
         conn, disk_path.encode(), flags, ctypes.byref(disk_handle)))
@@ -223,10 +231,12 @@ def read(disk_handle, start_sector, num_sectors, buf):
 
 
 def close(disk_handle):
+    LOG.debug("Closing VixDiskLib disk handle: %s", disk_handle)
     _check_err(vixDiskLib.VixDiskLib_Close(disk_handle))
 
 
 def disconnect(conn):
+    LOG.debug("Disconnecting VixDiskLib")
     _check_err(vixDiskLib.VixDiskLib_Disconnect(conn))
 
 

coriolis/replica_tasks_executions/api.py

@@ -0,0 +1,29 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from coriolis.conductor.rpc import client as rpc_client
+
+
+class API(object):
+    def __init__(self):
+        self._rpc_client = rpc_client.ConductorClient()
+
+    def create(self, ctxt, replica_id, shutdown_instances):
+        return self._rpc_client.execute_replica_tasks(
+            ctxt, replica_id, shutdown_instances)
+
+    def delete(self, ctxt, execution_id):
+        self._rpc_client.delete_replica_tasks_execution(
+            ctxt, execution_id)
+
+    def cancel(self, ctxt, execution_id, force):
+        self._rpc_client.cancel_replica_tasks_execution(
+            ctxt, execution_id, force)
+
+    def get_executions(self, ctxt, replica_id, include_tasks=False):
+        return self._rpc_client.get_replica_tasks_executions(
+            ctxt, replica_id, include_tasks)
+
+    def get_execution(self, ctxt, execution_id):
+        return self._rpc_client.get_replica_tasks_execution(
+            ctxt, execution_id)

coriolis/replicas/api.py

@@ -0,0 +1,25 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from coriolis.conductor.rpc import client as rpc_client
+
+
+class API(object):
+    def __init__(self):
+        self._rpc_client = rpc_client.ConductorClient()
+
+    def create(self, ctxt, origin, destination, instances):
+        return self._rpc_client.create_instances_replica(
+            ctxt, origin, destination, instances)
+
+    def delete(self, ctxt, replica_id):
+        self._rpc_client.delete_replica(ctxt, replica_id)
+
+    def get_replicas(self, ctxt, include_tasks_executions=False):
+        return self._rpc_client.get_replicas(ctxt, include_tasks_executions)
+
+    def get_replica(self, ctxt, replica_id):
+        return self._rpc_client.get_replica(ctxt, replica_id)
+
+    def delete_disks(self, ctxt, replica_id):
+        return self._rpc_client.delete_replica_disks(ctxt, replica_id)

coriolis/schemas.py

@@ -5,10 +5,10 @@
 
 import json
 
-import logging
 import jinja2
 import jsonschema
 
+from oslo_log import log as logging
 
 LOG = logging.getLogger(__name__)
 
@@ -32,7 +32,7 @@ def get_schema(package_name, schema_name,
     schema = json.loads(template_env.get_template(schema_name).render())
 
     LOG.debug("Succesfully loaded and parsed schema '%s' from '%s'.",
-             schema_name, package_name)
+              schema_name, package_name)
     return schema
 
 
@@ -44,14 +44,14 @@ def validate_value(val, schema):
     jsonschema.validate(val, schema)
 
 
-def validate_string(string, schema):
+def validate_string(json_string, schema):
     """ Attempts to validate the json value provided as a string against the
     given JSON schema.
 
     Runs silently on success or raises an exception otherwise.
     Silently passes empty schemas.
     """
-    jsonschema.validate(json.loads(string), schema)
+    validate_value(json.loads(json_string), schema)
 
 
 # Global schemas:

coriolis/tasks/base.py

@@ -0,0 +1,26 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import abc
+
+from coriolis import secrets
+
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+class TaskRunner(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        pass
+
+
+def get_connection_info(ctxt, data):
+    connection_info = data.get("connection_info") or {}
+    secret_ref = connection_info.get("secret_ref")
+    if secret_ref:
+        LOG.info("Retrieving connection info from secret: %s", secret_ref)
+        connection_info = secrets.get_secret(ctxt, secret_ref)
+    return connection_info

coriolis/tasks/factory.py

@@ -0,0 +1,48 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from coriolis import constants
+from coriolis import exception
+from coriolis.tasks import migration_tasks
+from coriolis.tasks import replica_tasks
+
+_TASKS_MAP = {
+    constants.TASK_TYPE_EXPORT_INSTANCE:
+        migration_tasks.ExportInstanceTask,
+    constants.TASK_TYPE_IMPORT_INSTANCE:
+        migration_tasks.ImportInstanceTask,
+    constants.TASK_TYPE_GET_INSTANCE_INFO:
+        replica_tasks.GetInstanceInfoTask,
+    constants.TASK_TYPE_REPLICATE_DISKS:
+        replica_tasks.ReplicateDisksTask,
+    constants.TASK_TYPE_SHUTDOWN_INSTANCE:
+        replica_tasks.ShutdownInstanceTask,
+    constants.TASK_TYPE_DEPLOY_REPLICA_DISKS:
+        replica_tasks.DeployReplicaDisksTask,
+    constants.TASK_TYPE_DELETE_REPLICA_DISKS:
+        replica_tasks.DeleteReplicaDisksTask,
+    constants.TASK_TYPE_DEPLOY_REPLICA_TARGET_RESOURCES:
+        replica_tasks.DeployReplicaTargetResourcesTask,
+    constants.TASK_TYPE_DELETE_REPLICA_TARGET_RESOURCES:
+        replica_tasks.DeleteReplicaTargetResourcesTask,
+    constants.TASK_TYPE_DEPLOY_REPLICA_SOURCE_RESOURCES:
+        replica_tasks.DeployReplicaSourceResourcesTask,
+    constants.TASK_TYPE_DELETE_REPLICA_SOURCE_RESOURCES:
+        replica_tasks.DeleteReplicaSourceResourcesTask,
+    constants.TASK_TYPE_DEPLOY_REPLICA_INSTANCE:
+        replica_tasks.DeployReplicaInstanceTask,
+    constants.TASK_TYPE_CREATE_REPLICA_DISK_SNAPSHOTS:
+        replica_tasks.CreateReplicaDiskSnapshotsTask,
+    constants.TASK_TYPE_DELETE_REPLICA_DISK_SNAPSHOTS:
+        replica_tasks.DeleteReplicaDiskSnapshotsTask,
+    constants.TASK_TYPE_RESTORE_REPLICA_DISK_SNAPSHOTS:
+        replica_tasks.RestoreReplicaDiskSnapshotsTask,
+}
+
+
+def get_task_runner(task_type):
+    cls = _TASKS_MAP.get(task_type)
+    if not cls:
+        raise exception.NotFound(
+            "TaskRunner not found for task type: %s" % task_type)
+    return cls()

coriolis/tasks/migration_tasks.py

@@ -0,0 +1,47 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from coriolis import constants
+from coriolis.providers import factory as providers_factory
+from coriolis import schemas
+from coriolis.tasks import base
+
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+class ExportInstanceTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            origin["type"], constants.PROVIDER_TYPE_EXPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, origin)
+        export_path = task_info["export_path"]
+
+        export_info = provider.export_instance(
+            ctxt, connection_info, instance, export_path)
+
+        # Validate the output
+        schemas.validate_value(
+            export_info, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA)
+        task_info["export_info"] = export_info
+        task_info["retain_export_path"] = True
+
+        return task_info
+
+
+class ImportInstanceTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        target_environment = destination.get("target_environment") or {}
+        export_info = task_info["export_info"]
+
+        provider = providers_factory.get_provider(
+            destination["type"], constants.PROVIDER_TYPE_IMPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, destination)
+
+        provider.import_instance(
+            ctxt, connection_info, target_environment, instance, export_info)
+
+        return task_info

coriolis/tasks/replica_tasks.py

@@ -0,0 +1,297 @@
+# Copyright 2016 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+from coriolis import constants
+from coriolis import exception
+from coriolis.providers import factory as providers_factory
+from coriolis import schemas
+from coriolis.tasks import base
+from coriolis import utils
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+serialization_opts = [
+    cfg.StrOpt('temp_keypair_password',
+               default=None,
+               help='Password to be used when serializing temporary keys'),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(serialization_opts, 'serialization')
+
+LOG = logging.getLogger(__name__)
+
+
+def _marshal_migr_conn_info(migr_connection_info):
+    if migr_connection_info and "pkey" in migr_connection_info:
+        migr_connection_info = migr_connection_info.copy()
+        migr_connection_info["pkey"] = utils.serialize_key(
+            migr_connection_info["pkey"],
+            CONF.serialization.temp_keypair_password)
+    return migr_connection_info
+
+
+def _unmarshal_migr_conn_info(migr_connection_info):
+    if migr_connection_info and "pkey" in migr_connection_info:
+        migr_connection_info = migr_connection_info.copy()
+        pkey_str = migr_connection_info["pkey"]
+        migr_connection_info["pkey"] = utils.deserialize_key(
+            pkey_str, CONF.serialization.temp_keypair_password)
+    return migr_connection_info
+
+
+def _get_volumes_info(task_info):
+    volumes_info = task_info.get("volumes_info")
+    if not volumes_info:
+        raise exception.InvalidActionTasksExecutionState(
+            "No volumes information present")
+    return volumes_info
+
+
+class GetInstanceInfoTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            origin["type"], constants.PROVIDER_TYPE_EXPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, origin)
+
+        export_info = provider.get_replica_instance_info(
+            ctxt, connection_info, instance)
+
+        # Validate the output
+        schemas.validate_value(
+            export_info, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA)
+        task_info["export_info"] = export_info
+
+        return task_info
+
+
+class ShutdownInstanceTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            origin["type"], constants.PROVIDER_TYPE_EXPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, origin)
+
+        provider.shutdown_instance(ctxt, connection_info, instance)
+
+        return task_info
+
+
+class ReplicateDisksTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            origin["type"], constants.PROVIDER_TYPE_EXPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, origin)
+
+        volumes_info = _get_volumes_info(task_info)
+
+        migr_source_conn_info = _unmarshal_migr_conn_info(
+            task_info["migr_source_connection_info"])
+
+        migr_target_conn_info = _unmarshal_migr_conn_info(
+            task_info["migr_target_connection_info"])
+
+        incremental = task_info.get("incremental", True)
+
+        volumes_info = provider.replicate_disks(
+            ctxt, connection_info, instance, migr_source_conn_info,
+            migr_target_conn_info, volumes_info, incremental)
+
+        task_info["volumes_info"] = volumes_info
+
+        return task_info
+
+
+class DeployReplicaDisksTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        target_environment = destination.get("target_environment") or {}
+        export_info = task_info["export_info"]
+
+        provider = providers_factory.get_provider(
+            destination["type"], constants.PROVIDER_TYPE_IMPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, destination)
+
+        volumes_info = task_info.get("volumes_info") or []
+
+        volumes_info = provider.deploy_replica_disks(
+            ctxt, connection_info, target_environment, instance, export_info,
+            volumes_info)
+
+        task_info["volumes_info"] = volumes_info
+
+        return task_info
+
+
+class DeleteReplicaDisksTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            destination["type"], constants.PROVIDER_TYPE_IMPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, destination)
+
+        volumes_info = _get_volumes_info(task_info)
+
+        provider.delete_replica_disks(
+            ctxt, connection_info, volumes_info)
+
+        task_info["volumes_info"] = None
+
+        return task_info
+
+
+class DeployReplicaSourceResourcesTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            origin["type"], constants.PROVIDER_TYPE_EXPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, origin)
+
+        replica_resources_info = provider.deploy_replica_source_resources(
+            ctxt, connection_info)
+
+        task_info["migr_source_resources"] = replica_resources_info[
+            "migr_resources"]
+        migr_connection_info = _marshal_migr_conn_info(
+            replica_resources_info["connection_info"])
+        task_info["migr_source_connection_info"] = migr_connection_info
+
+        return task_info
+
+
+class DeleteReplicaSourceResourcesTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            origin["type"], constants.PROVIDER_TYPE_EXPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, origin)
+
+        migr_resources = task_info.get("migr_source_resources")
+
+        if migr_resources:
+            provider.delete_replica_source_resources(
+                ctxt, connection_info, migr_resources)
+
+        task_info["migr_source_resources"] = None
+        task_info["migr_source_connection_info"] = None
+
+        return task_info
+
+
+class DeployReplicaTargetResourcesTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        target_environment = destination.get("target_environment") or {}
+
+        provider = providers_factory.get_provider(
+            destination["type"], constants.PROVIDER_TYPE_IMPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, destination)
+
+        volumes_info = _get_volumes_info(task_info)
+
+        replica_resources_info = provider.deploy_replica_target_resources(
+            ctxt, connection_info, target_environment, volumes_info)
+
+        task_info["volumes_info"] = replica_resources_info["volumes_info"]
+        task_info["migr_target_resources"] = replica_resources_info[
+            "migr_resources"]
+
+        migr_connection_info = _marshal_migr_conn_info(
+            replica_resources_info["connection_info"])
+        task_info["migr_target_connection_info"] = migr_connection_info
+
+        return task_info
+
+
+class DeleteReplicaTargetResourcesTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            destination["type"], constants.PROVIDER_TYPE_IMPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, destination)
+
+        migr_resources = task_info.get("migr_target_resources")
+
+        if migr_resources:
+            provider.delete_replica_target_resources(
+                ctxt, connection_info, migr_resources)
+
+        task_info["migr_target_resources"] = None
+        task_info["migr_target_connection_info"] = None
+
+        return task_info
+
+
+class DeployReplicaInstanceTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        target_environment = destination.get("target_environment") or {}
+        export_info = task_info["export_info"]
+
+        provider = providers_factory.get_provider(
+            destination["type"], constants.PROVIDER_TYPE_IMPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, destination)
+
+        volumes_info = _get_volumes_info(task_info)
+        clone_disks = task_info.get("clone_disks", True)
+        LOG.debug("Clone disks: %s", clone_disks)
+
+        provider.deploy_replica_instance(
+            ctxt, connection_info, target_environment, instance,
+            export_info, volumes_info, clone_disks)
+
+        return task_info
+
+
+class CreateReplicaDiskSnapshotsTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            destination["type"], constants.PROVIDER_TYPE_IMPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, destination)
+
+        volumes_info = _get_volumes_info(task_info)
+
+        volumes_info = provider.create_replica_disk_snapshots(
+            ctxt, connection_info, volumes_info)
+
+        task_info["volumes_info"] = volumes_info
+
+        return task_info
+
+
+class DeleteReplicaDiskSnapshotsTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            destination["type"], constants.PROVIDER_TYPE_IMPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, destination)
+
+        volumes_info = _get_volumes_info(task_info)
+
+        volumes_info = provider.delete_replica_disk_snapshots(
+            ctxt, connection_info, volumes_info)
+
+        task_info["volumes_info"] = volumes_info
+
+        return task_info
+
+
+class RestoreReplicaDiskSnapshotsTask(base.TaskRunner):
+    def run(self, ctxt, instance, origin, destination, task_info,
+            event_handler):
+        provider = providers_factory.get_provider(
+            destination["type"], constants.PROVIDER_TYPE_IMPORT, event_handler)
+        connection_info = base.get_connection_info(ctxt, destination)
+
+        volumes_info = _get_volumes_info(task_info)
+
+        volumes_info = provider.restore_replica_disk_snapshots(
+            ctxt, connection_info, volumes_info)
+
+        task_info["volumes_info"] = volumes_info
+
+        return task_info

coriolis/utils.py

@@ -2,7 +2,10 @@
 # All Rights Reserved.
 
 import functools
+import io
 import json
+import os
+import pickle
 import re
 import socket
 import subprocess
@@ -12,6 +15,8 @@ import traceback
 import OpenSSL
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_serialization import jsonutils
+import paramiko
 
 from coriolis import constants
 from coriolis import exception
@@ -52,6 +57,10 @@ def retry_on_error(max_attempts=5, sleep_seconds=0,
             while True:
                 try:
                     return func(*args, **kwargs)
+                except KeyboardInterrupt as ex:
+                    LOG.debug("Got a KeyboardInterrupt, skip retrying")
+                    LOG.exception(ex)
+                    raise
                 except Exception as ex:
                     if any([isinstance(ex, tex)
                             for tex in terminal_exceptions]):
@@ -236,3 +245,72 @@ def get_ssl_cert_thumbprint(context, host, port=443, digest_algorithm="sha1"):
 
     x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert)
     return x509.digest('sha1').decode()
+
+
+def _get_base_dir():
+    return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def get_resources_dir():
+    return os.path.join(_get_base_dir(), "resources")
+
+
+def serialize_key(key, password=None):
+    key_io = io.StringIO()
+    key.write_private_key(key_io, password)
+    return key_io.getvalue()
+
+
+def deserialize_key(key_bytes, password=None):
+    key_io = io.StringIO(key_bytes)
+    return paramiko.RSAKey.from_private_key(key_io, password)
+
+
+def is_serializable(obj):
+    pickle.dumps(obj)
+
+
+def to_dict(obj, max_depth=10):
+    # jsonutils.dumps() has a max_depth of 3 by default
+    def _to_primitive(value, convert_instances=False,
+                      convert_datetime=True, level=0,
+                      max_depth=max_depth):
+        return jsonutils.to_primitive(
+            value, convert_instances, convert_datetime, level, max_depth)
+    return jsonutils.loads(jsonutils.dumps(obj, default=_to_primitive))
+
+
+def topological_graph_sorting(items, id="id", depends_on="depends_on",
+                              sort_key=None):
+    """
+    Kahn's algorithm
+    """
+    if sort_key:
+        # Sort siblings
+        items = sorted(items, key=lambda t: t[sort_key], reverse=True)
+
+    a = []
+    for i in items:
+        a.append({"id": i[id],
+                  "depends_on": list(i[depends_on] or []),
+                  "item": i})
+
+    s = []
+    l = []
+    for n in a:
+        if not n["depends_on"]:
+            s.append(n)
+    while s:
+        n = s.pop()
+        l.append(n["item"])
+
+        for m in a:
+            if n["id"] in m["depends_on"]:
+                m["depends_on"].remove(n["id"])
+                if not m["depends_on"]:
+                    s.append(m)
+
+    if len(l) != len(a):
+        raise ValueError("The graph contains cycles")
+
+    return l

coriolis/worker/rpc/client.py

@@ -21,10 +21,10 @@ class WorkerClient(object):
             origin=origin, destination=destination, instance=instance,
             task_info=task_info)
 
-    def cancel_task(self, ctxt, server, process_id):
+    def cancel_task(self, ctxt, server, process_id, force):
         # Needs to be executed on the same server
         cctxt = self._client.prepare(server=server)
-        cctxt.call(ctxt, 'cancel_task', process_id=process_id)
+        cctxt.call(ctxt, 'cancel_task', process_id=process_id, force=force)
 
     def update_migration_status(self, ctxt, task_id, status):
         self._client.call(ctxt, "update_migration_status", status=status)

coriolis/worker/rpc/server.py

@@ -6,6 +6,7 @@ import multiprocessing
 import os
 import queue
 import shutil
+import signal
 import sys
 
 import psutil
@@ -16,11 +17,10 @@ from coriolis.conductor.rpc import client as rpc_conductor_client
 from coriolis import constants
 from coriolis import events
 from coriolis import exception
-from coriolis.providers import factory
-from coriolis import schemas
-from coriolis import secrets
+from coriolis.tasks import factory as task_runners_factory
 from coriolis import utils
 
+
 worker_opts = [
     cfg.StrOpt('export_base_path',
                default='/tmp',
@@ -32,8 +32,6 @@ CONF.register_opts(worker_opts, 'worker')
 
 LOG = logging.getLogger(__name__)
 
-TMP_DIRS_KEY = "__tmp_dirs"
-
 VERSION = "1.0"
 
 
@@ -69,32 +67,29 @@ class WorkerServerEndpoint(object):
         self._server = utils.get_hostname()
         self._rpc_conductor_client = rpc_conductor_client.ConductorClient()
 
-    def _cleanup_task_resources(self, task_id, task_info=None):
+    def _check_remove_dir(self, path):
         try:
-            export_path = _get_task_export_path(task_id)
-            if (not task_info or export_path not in
-                    task_info.get(TMP_DIRS_KEY, [])):
-                # Don't remove folder if it's needed by the dependent tasks
-                if os.path.exists(export_path):
-                    shutil.rmtree(export_path)
+            if os.path.exists(path):
+                shutil.rmtree(path)
         except Exception as ex:
             # Ignore the exception
             LOG.exception(ex)
 
-    def _remove_tmp_dirs(self, task_info):
-        if task_info:
-            for tmp_dir in task_info.get(TMP_DIRS_KEY, []):
-                if os.path.exists(tmp_dir):
-                    try:
-                        shutil.rmtree(tmp_dir)
-                    except Exception as ex:
-                        # Ignore exception
-                        LOG.exception(ex)
-
-    def cancel_task(self, ctxt, process_id):
+    def cancel_task(self, ctxt, process_id, force):
+        if not force and os.name == "nt":
+            LOG.warn("Windows does not support SIGINT, performing a "
+                     "forced task termination")
+            force = True
+
         try:
             p = psutil.Process(process_id)
-            p.kill()
+
+            if force:
+                LOG.warn("Killing process: %s", process_id)
+                p.kill()
+            else:
+                LOG.info("Sending SIGINT to process: %s", process_id)
+                p.send_signal(signal.SIGINT)
         except psutil.NoSuchProcess:
             LOG.info("Task process not found: %s", process_id)
 
@@ -138,6 +133,13 @@ class WorkerServerEndpoint(object):
 
     def exec_task(self, ctxt, task_id, task_type, origin, destination,
                   instance, task_info):
+        export_path = task_info.get("export_path")
+        if not export_path:
+            export_path = _get_task_export_path(task_id, create=True)
+            task_info["export_path"] = export_path
+        retain_export_path = False
+        task_info["retain_export_path"] = retain_export_path
+
         try:
             new_task_info = self._exec_task_process(
                 ctxt, task_id, task_type, origin, destination,
@@ -146,18 +148,20 @@ class WorkerServerEndpoint(object):
             if new_task_info:
                 LOG.info("Task info: %s", new_task_info)
 
+            # TODO: replace the temp storage with a host independent option
+            retain_export_path = new_task_info.get("retain_export_path", False)
+            if not retain_export_path:
+                del new_task_info["export_path"]
+
             LOG.info("Task completed: %s", task_id)
             self._rpc_conductor_client.task_completed(ctxt, task_id,
                                                       new_task_info)
-
-            self._cleanup_task_resources(task_id, new_task_info)
         except Exception as ex:
             LOG.exception(ex)
             self._rpc_conductor_client.set_task_error(ctxt, task_id, str(ex))
-
-            self._cleanup_task_resources(task_id)
         finally:
-            self._remove_tmp_dirs(task_info)
+            if not retain_export_path:
+                self._check_remove_dir(export_path)
 
 
 def _get_task_export_path(task_id, create=False):
@@ -184,43 +188,23 @@ def _task_process(ctxt, task_id, task_type, origin, destination, instance,
     try:
         _setup_task_process(mp_log_q)
 
-        if task_type == constants.TASK_TYPE_EXPORT_INSTANCE:
-            provider_type = constants.PROVIDER_TYPE_EXPORT
-            data = origin
-        elif task_type == constants.TASK_TYPE_IMPORT_INSTANCE:
-            provider_type = constants.PROVIDER_TYPE_IMPORT
-            data = destination
-        else:
-            raise exception.NotFound(
-                "Unknown task type: %s" % task_type)
-
+        task_runner = task_runners_factory.get_task_runner(task_type)
         event_handler = _ConductorProviderEventHandler(ctxt, task_id)
-        provider = factory.get_provider(data["type"], provider_type,
-                                        event_handler)
 
-        connection_info = data.get("connection_info") or {}
-        target_environment = data.get("target_environment") or {}
+        LOG.debug("Executing task: %(task_id)s, type: %(task_type)s, "
+                  "origin: %(origin)s, destination: %(destination)s, "
+                  "instance: %(instance)s, task_info: %(task_info)s",
+                  {"task_id": task_id, "task_type": task_type,
+                   "origin": origin, "destination": destination,
+                   "instance": instance, "task_info": task_info})
 
-        secret_ref = connection_info.get("secret_ref")
-        if secret_ref:
-            LOG.info("Retrieving connection info from secret: %s", secret_ref)
-            connection_info = secrets.get_secret(ctxt, secret_ref)
+        new_task_info = task_runner.run(
+            ctxt, instance, origin, destination, task_info, event_handler)
 
-        if provider_type == constants.PROVIDER_TYPE_EXPORT:
-            export_path = _get_task_export_path(task_id, create=True)
+        # mq_p.put() doesn't raise if new_task_info is not serializable
+        utils.is_serializable(new_task_info)
 
-            result = provider.export_instance(ctxt, connection_info, instance,
-                                              export_path)
-            result[TMP_DIRS_KEY] = [export_path]
-
-            # validate the outputted VM info:
-            schemas.validate_value(
-                result, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA)
-        else:
-            result = provider.import_instance(ctxt, connection_info,
-                                              target_environment, instance,
-                                              task_info)
-        mp_q.put(result)
+        mp_q.put(new_task_info)
     except Exception as ex:
         mp_q.put(str(ex))
         LOG.exception(ex)

resources/makefile

@@ -0,0 +1,3 @@
+write_data: write_data.c
+	gcc -o write_data write_data.c -lz
+

resources/write_data.c

@@ -0,0 +1,178 @@
+// Copyright 2016 Cloudbase Solutions Srl
+// All Rights Reserved.
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <zlib.h>
+
+#define MIN_MSG_SIZE (sizeof(uint64_t) + 1)
+#define MAX_MSG_SIZE (100 * 1024 * 1024)
+
+#define ERR_MORE_MSG            -1
+#define ERR_DONE                0
+#define ERR_READ_MSG_SIZE       1
+#define ERR_MSG_SIZE            2
+#define ERR_OPEN_FILE           3
+#define ERR_DATA                4
+#define ERR_IO_OPEN             5
+#define ERR_IO_SEEK             6
+#define ERR_IO_WRITE            7
+#define ERR_IO_CLOSE            8
+#define ERR_NO_MEM              9
+#define ERR_INVALID_ARGS        10
+#define ERR_READ_MSG_ID         11
+#define ERR_MSG_SIZE_INFLATED   12
+#define ERR_ZLIB                13
+
+int write_msg_id(uint32_t msg_id)
+{
+    size_t c = fwrite(&msg_id, 1, sizeof(msg_id), stdout);
+    if (c != sizeof(msg_id))
+       return ERR_IO_WRITE;
+    if(fflush(stdout))
+        return ERR_IO_WRITE;
+    return ERR_DONE;
+}
+
+int inflate_buf(uint32_t msg_size, void* buf, uint32_t msg_size_inflated,
+                void* inflated_buf)
+{
+    z_stream strm;
+    memset(&strm, 0, sizeof(z_stream));
+    int ret = inflateInit(&strm);
+    if (ret != Z_OK)
+        return ERR_ZLIB;
+
+    strm.avail_in = msg_size;
+    strm.next_in = buf;
+    strm.avail_out = msg_size_inflated;
+    strm.next_out = inflated_buf;
+
+    ret = inflate(&strm, Z_FINISH);
+    if(ret != Z_STREAM_END)
+        return ERR_ZLIB;
+
+    inflateEnd(&strm);
+    return ERR_DONE;
+}
+
+int handle_msg(FILE* input_stream)
+{
+    uint32_t msg_id = 0;
+    size_t c = fread(&msg_id, 1, sizeof(uint32_t), input_stream);
+    if (c != sizeof(uint32_t))
+        return ERR_READ_MSG_ID;
+
+    uint32_t msg_size = 0;
+    c = fread(&msg_size, 1, sizeof(uint32_t), input_stream);
+    if (c != sizeof(uint32_t))
+        return ERR_READ_MSG_SIZE;
+    if (!msg_size)
+    {
+        int err = write_msg_id(msg_id);
+        if(err)
+            return err;
+        return ERR_DONE;
+    }
+    if (msg_size < MIN_MSG_SIZE || msg_size > MAX_MSG_SIZE)
+        return ERR_MSG_SIZE;
+
+    uint32_t msg_size_inflated = 0;
+    c = fread(&msg_size_inflated, 1, sizeof(uint32_t), input_stream);
+    if (c != sizeof(uint32_t))
+        return ERR_MSG_SIZE_INFLATED;
+    if (msg_size_inflated != 0 && (msg_size_inflated < MIN_MSG_SIZE ||
+            msg_size_inflated > MAX_MSG_SIZE))
+        return ERR_MSG_SIZE_INFLATED;
+
+    unsigned char* buf = (unsigned char*)malloc(msg_size);
+    if (!buf)
+        return ERR_NO_MEM;
+
+    c = fread(buf, 1, msg_size, input_stream);
+    if (c != msg_size)
+        return ERR_IO_OPEN;
+
+    if(msg_size_inflated)
+    {
+        unsigned char* inflated_buf = (unsigned char*)malloc(msg_size_inflated);
+        if (!inflated_buf)
+            return ERR_NO_MEM;
+
+        int err = inflate_buf(msg_size, buf, msg_size_inflated, inflated_buf);
+        if(err != ERR_DONE)
+        {
+            free(inflated_buf);
+            return err;
+        }
+
+        free(buf);
+        buf = inflated_buf;
+        msg_size = msg_size_inflated;
+    }
+
+    char* path = (char*)buf;
+    // strlen is unsafe
+    unsigned char* data = (unsigned char*)memchr(path, '\0', msg_size);
+    if (!data)
+        return ERR_DATA;
+    data++;
+
+    uint64_t offset = *((uint64_t*)data);
+    data += sizeof(uint64_t);
+
+    // Create an empty file in case it does not exist
+    FILE* f = fopen(path, "ab+");
+    if (!f)
+        return ERR_OPEN_FILE;
+    if (fclose(f))
+        return ERR_IO_CLOSE;
+
+    // Use rb+ to allow fseek when writing
+    f = fopen(path, "rb+");
+    if (!f)
+        return ERR_OPEN_FILE;
+    if (fseek(f, (long)offset, SEEK_SET))
+        return ERR_IO_SEEK;
+
+    size_t data_size = msg_size - (data - buf);
+    c = fwrite(data, 1, data_size, f);
+    if (c != data_size)
+        return ERR_IO_WRITE;
+    if (fclose(f))
+        return ERR_IO_CLOSE;
+
+    // TODO: free also in case of errors
+    free(buf);
+
+    int err = write_msg_id(msg_id);
+    if(err)
+        return err;
+
+    return ERR_MORE_MSG;
+}
+
+int main(int argc, char **argv)
+{
+    FILE* input_stream = NULL;
+    if (argc == 2)
+    {
+        char* input_path = argv[1];
+        if (!(input_stream = fopen(input_path, "rb")))
+            return ERR_IO_OPEN;
+    }
+    else if (argc == 1)
+        input_stream = stdin;
+    else
+        return ERR_INVALID_ARGS;
+
+    int err;
+    while ((err = handle_msg(input_stream)) == ERR_MORE_MSG);
+
+    if (input_stream != stdin)
+        fclose(input_stream);
+
+    return err;
+}