Exclude task info from result

task info currently holds replica state for cases where the
coriolis replicator is used. We need to exclude this field when
returning a large result set, as the replica state can be quite
large, and the serialization of that field can take up considerable
resources. Moreover, the information in that field is only
relevant to the worker, not to the user.

coriolis/api/v1/migrations.py

@@ -1,5 +1,6 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
+import json
 
 from oslo_log import log as logging
 from webob import exc
@@ -30,15 +31,34 @@ class MigrationController(api_wsgi.Controller):
 
         return migration_view.single(req, migration)
 
+    def _get_show_deleted(self, val):
+        if val is None:
+            return val
+        try:
+            show_deleted = json.loads(val)
+            if type(show_deleted) is bool:
+                return show_deleted
+        except Exception as err:
+            LOG.warn(
+                "failed to parse show_deleted: %s" % err)
+            pass
+        return None
+
     def index(self, req):
+        show_deleted = self._get_show_deleted(
+            req.GET.get("show_deleted", None))
         context = req.environ["coriolis.context"]
+        context.show_deleted = show_deleted
         context.can(migration_policies.get_migrations_policy_label("show"))
         return migration_view.collection(
             req, self._migration_api.get_migrations(
                 context, include_tasks=False))
 
     def detail(self, req):
+        show_deleted = self._get_show_deleted(
+            req.GET.get("show_deleted", None))
         context = req.environ["coriolis.context"]
+        context.show_deleted = show_deleted
         context.can(
             migration_policies.get_migrations_policy_label("show_execution"))
         return migration_view.collection(

coriolis/api/v1/replicas.py

@@ -31,15 +31,17 @@ class ReplicaController(api_wsgi.Controller):
 
         return replica_view.single(req, replica)
 
-    def index(self, req):
+    def index(self, req, show_deleted=False):
         context = req.environ["coriolis.context"]
+        context["show_deleted"] = show_deleted
         context.can(replica_policies.get_replicas_policy_label("list"))
         return replica_view.collection(
             req, self._replica_api.get_replicas(
                 context, include_tasks_executions=False))
 
-    def detail(self, req):
+    def detail(self, req, show_deleted=False):
         context = req.environ["coriolis.context"]
+        context["show_deleted"] = show_deleted
         context.can(
             replica_policies.get_replicas_policy_label("show_executions"))
         return replica_view.collection(

coriolis/api/v1/views/migration_view.py

@@ -5,7 +5,7 @@ import itertools
 
 from oslo_config import cfg as conf
 
-from coriolis.api.v1.views import replica_tasks_execution_view
+from coriolis.api.v1.views import replica_tasks_execution_view as view
 
 
 MIGRATIONS_API_OPTS = [
@@ -27,10 +27,14 @@ def _format_migration(req, migration, keys=None):
     migration_dict = dict(itertools.chain.from_iterable(
         transform(k, v) for k, v in migration.items()))
 
-    execution = replica_tasks_execution_view.format_replica_tasks_execution(
-        req, migration_dict["executions"][0])
+    if len(migration_dict["executions"]):
 
-    migration_dict["status"] = execution["status"]
+        execution = view.format_replica_tasks_execution(
+            req, migration_dict["executions"][0])
+    else:
+        execution = {}
+
+    migration_dict["status"] = execution.get("status")
     tasks = execution.get("tasks")
     if tasks:
         migration_dict["tasks"] = tasks

coriolis/conductor/rpc/client.py

@@ -183,9 +183,11 @@ class ConductorClient(object):
         return self._client.call(
             ctxt, 'delete_replica_disks', replica_id=replica_id)
 
-    def get_migrations(self, ctxt, include_tasks=False):
+    def get_migrations(self, ctxt, include_tasks=False,
+                       include_info=False):
         return self._client.call(ctxt, 'get_migrations',
-                                 include_tasks=include_tasks)
+                                 include_tasks=include_tasks,
+                                 include_info=include_info)
 
     def get_migration(self, ctxt, migration_id):
         return self._client.call(

coriolis/conductor/rpc/server.py

@@ -536,8 +536,11 @@ class ConductorServerEndpoint(object):
             raise exception.NotFound("Replica not found")
         return replica
 
-    def get_migrations(self, ctxt, include_tasks):
-        return db_api.get_migrations(ctxt, include_tasks)
+    def get_migrations(self, ctxt, include_tasks,
+                       include_info=False):
+        return db_api.get_migrations(
+            ctxt, include_tasks,
+            include_info=include_info)
 
     @migration_synchronized
     def get_migration(self, ctxt, migration_id):

coriolis/db/api.py

@@ -350,16 +350,22 @@ def get_replica_migrations(context, replica_id):
 
 
 @enginefacade.reader
-def get_migrations(context, include_tasks=False):
+def get_migrations(context, include_tasks=False,
+                   include_info=False):
     q = _soft_delete_aware_query(context, models.Migration)
     if include_tasks:
         q = _get_migration_task_query_options(q)
     else:
         q = q.options(orm.joinedload("executions"))
+    if include_info is False:
+        q = q.options(orm.defer('info'))
+
     args = {}
     if is_user_context(context):
         args["project_id"] = context.tenant
-    return q.filter_by(**args).all()
+    result = q.filter_by(**args).all()
+    to_dict = [i.to_dict(include_info=include_info) for i in result]
+    return to_dict
 
 
 def _get_tasks_with_details_options(query):

coriolis/db/sqlalchemy/models.py

@@ -26,6 +26,19 @@ class TaskEvent(BASE, models.TimestampMixin, models.SoftDeleteMixin,
     level = sqlalchemy.Column(sqlalchemy.String(20), nullable=False)
     message = sqlalchemy.Column(sqlalchemy.String(1024), nullable=False)
 
+    def to_dict(self):
+        result = {
+            "id": self.id,
+            "task_id": self.task_id,
+            "level": self.level,
+            "message": self.message,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "deleted_at": self.deleted_at,
+            "deleted": self.deleted,
+        }
+        return result
+
 
 class TaskProgressUpdate(BASE, models.TimestampMixin, models.SoftDeleteMixin,
                          models.ModelBase):
@@ -41,6 +54,20 @@ class TaskProgressUpdate(BASE, models.TimestampMixin, models.SoftDeleteMixin,
     total_steps = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
     message = sqlalchemy.Column(sqlalchemy.String(1024), nullable=True)
 
+    def to_dict(self):
+        result = {
+            "id": self.id,
+            "task_id": self.task_id,
+            "current_step": self.current_step,
+            "total_steps": self.total_steps,
+            "message": self.message,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "deleted_at": self.deleted_at,
+            "deleted": self.deleted,
+        }
+        return result
+
 
 class Task(BASE, models.TimestampMixin, models.SoftDeleteMixin,
            models.ModelBase):
@@ -69,6 +96,35 @@ class Task(BASE, models.TimestampMixin, models.SoftDeleteMixin,
                                         cascade="all,delete",
                                         backref=orm.backref('task'))
 
+    def to_dict(self):
+        result = {
+            "id": self.id,
+            "execution_id": self.execution_id,
+            "instance": self.instance,
+            "host": self.host,
+            "process_id": self.process_id,
+            "status": self.status,
+            "task_type": self.task_type,
+            "exception_details": self.exception_details,
+            "depends_on": self.depends_on,
+            "index": self.index,
+            "on_error": self.on_error,
+            "events": [],
+            "progress_updates": [],
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "deleted_at": self.deleted_at,
+            "deleted": self.deleted,
+        }
+
+        for evt in self.events:
+            result["events"].append(evt.to_dict())
+
+        for pgu in self.progress_updates:
+            result["progress_updates"].append(
+                pgu.to_dict())
+        return result
+
 
 class TasksExecution(BASE, models.TimestampMixin, models.ModelBase,
                      models.SoftDeleteMixin):
@@ -87,6 +143,23 @@ class TasksExecution(BASE, models.TimestampMixin, models.ModelBase,
     number = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
     type = sqlalchemy.Column(sqlalchemy.String(20))
 
+    def to_dict(self):
+        result = {
+            "id": self.id,
+            "action_id": self.action_id,
+            "tasks": [],
+            "status": self.status,
+            "number": self.number,
+            "type": self.type,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "deleted_at": self.deleted_at,
+            "deleted": self.deleted,
+        }
+        for tsk in self.tasks:
+            result["tasks"].append(tsk.to_dict())
+        return result
+
 
 class BaseTransferAction(BASE, models.TimestampMixin, models.ModelBase,
                          models.SoftDeleteMixin):
@@ -124,6 +197,34 @@ class BaseTransferAction(BASE, models.TimestampMixin, models.ModelBase,
         'polymorphic_on': type,
     }
 
+    def to_dict(self, include_info=True):
+        result = {
+            "base_id": self.base_id,
+            "user_id": self.user_id,
+            "project_id": self.project_id,
+            "destination_environment": self.destination_environment,
+            "type": self.type,
+            "executions": [],
+            "instances": self.instances,
+            "reservation_id": self.reservation_id,
+            "notes": self.notes,
+            "origin_endpoint_id": self.origin_endpoint_id,
+            "destination_endpoint_id": self.destination_endpoint_id,
+            "transfer_result": self.transfer_result,
+            "network_map": self.network_map,
+            "storage_mappings": self.storage_mappings,
+            "source_environment": self.source_environment,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "deleted_at": self.deleted_at,
+            "deleted": self.deleted,
+        }
+        for ex in self.executions:
+            result["executions"].append(ex.to_dict())
+        if include_info:
+            result["info"] = self.info
+        return result
+
 
 class Replica(BaseTransferAction):
     __tablename__ = 'replica'
@@ -137,6 +238,12 @@ class Replica(BaseTransferAction):
         'polymorphic_identity': 'replica',
     }
 
+    def to_dict(self, include_info=True):
+        base = super(Replica, self).to_dict(
+            include_info=include_info)
+        base.update({"id": self.id})
+        return base
+
 
 class Migration(BaseTransferAction):
     __tablename__ = 'migration'
@@ -159,6 +266,17 @@ class Migration(BaseTransferAction):
         'polymorphic_identity': 'migration',
     }
 
+    def to_dict(self, include_info=True):
+        base = super(Migration, self).to_dict(
+            include_info=include_info)
+        base.update({
+            "id": self.id,
+            "replica_id": self.replica_id,
+            "shutdown_instances": self.shutdown_instances,
+            "replication_count": self.replication_count,
+        })
+        return base
+
 
 class Endpoint(BASE, models.TimestampMixin, models.ModelBase,
                models.SoftDeleteMixin):