Merge pull request #164 from Dany9966/user_scripts_column

Add 'user_scripts' column to migration table

coriolis/api/v1/migrations.py

@@ -104,11 +104,13 @@ class MigrationController(api_wsgi.Controller):
                 shutdown_instances, network_map, storage_mappings)
 
     def create(self, req, body):
-        # TODO(alexpilotti): validate body
         migration_body = body.get("migration", {})
         context = req.environ['coriolis.context']
         context.can(migration_policies.get_migrations_policy_label("create"))
-        user_scripts = migration_body.get("user_scripts", {})
+        user_scripts = migration_body.get('user_scripts', {})
+        api_utils.validate_user_scripts(user_scripts)
+        user_scripts = api_utils.normalize_user_scripts(
+            user_scripts, migration_body.get("instances", []))
         replica_id = migration_body.get("replica_id")
         if replica_id:
             clone_disks = migration_body.get("clone_disks", True)

coriolis/api/v1/replicas.py

@@ -78,6 +78,11 @@ class ReplicaController(api_wsgi.Controller):
         instance_osmorphing_minion_pool_mappings = replica.get(
             'instance_osmorphing_minion_pool_mappings', {})
 
+        user_scripts = replica.get('user_scripts', {})
+        api_utils.validate_user_scripts(user_scripts)
+        user_scripts = api_utils.normalize_user_scripts(
+            user_scripts, instances)
+
         # NOTE(aznashwan): we validate the destination environment for the
         # import provider before appending the 'storage_mappings' parameter
         # for plugins with strict property name checks which do not yet
@@ -97,7 +102,7 @@ class ReplicaController(api_wsgi.Controller):
                 source_environment, destination_environment, instances,
                 network_map, storage_mappings, notes,
                 origin_minion_pool_id, destination_minion_pool_id,
-                instance_osmorphing_minion_pool_mappings)
+                instance_osmorphing_minion_pool_mappings, user_scripts)
 
     def create(self, req, body):
         context = req.environ["coriolis.context"]
@@ -107,7 +112,7 @@ class ReplicaController(api_wsgi.Controller):
          source_environment, destination_environment, instances, network_map,
          storage_mappings, notes, origin_minion_pool_id,
          destination_minion_pool_id,
-         instance_osmorphing_minion_pool_mappings) = (
+         instance_osmorphing_minion_pool_mappings, user_scripts) = (
             self._validate_create_body(context, body))
 
         return replica_view.single(req, self._replica_api.create(
@@ -115,7 +120,7 @@ class ReplicaController(api_wsgi.Controller):
             origin_minion_pool_id, destination_minion_pool_id,
             instance_osmorphing_minion_pool_mappings, source_environment,
             destination_environment, instances, network_map,
-            storage_mappings, notes))
+            storage_mappings, notes, user_scripts))
 
     def delete(self, req, id):
         context = req.environ["coriolis.context"]
@@ -171,12 +176,31 @@ class ReplicaController(api_wsgi.Controller):
 
         return storage_mappings
 
+    @staticmethod
+    def _get_updated_user_scripts(original_user_scripts, new_user_scripts):
+        global_scripts = original_user_scripts.get('global', {})
+        new_global_scripts = new_user_scripts.get('global', {})
+        if new_global_scripts:
+            global_scripts.update(new_global_scripts)
+
+        instance_scripts = original_user_scripts.get('instances', {})
+        new_instance_scripts = new_user_scripts.get('instances', {})
+        if new_instance_scripts:
+            instance_scripts.update(new_instance_scripts)
+
+        user_scripts = {
+            "global": global_scripts,
+            "instances": instance_scripts,
+        }
+
+        return user_scripts
+
     def _get_merged_replica_values(self, replica, updated_values):
         """ Looks for the following keys in the original replica body and
         updated values (preferring the updated values where needed, but using
         `.update()` on dicts):
         "source_environment", "destination_environment", "network_map", "notes"
-        Does special merging for the "storage_mappings"
+        Does special merging for the "storage_mappings" and "user_scripts"
         Returns a dict with the merged values (or at least all if the keys
         having a default value of {})
         """
@@ -207,6 +231,10 @@ class ReplicaController(api_wsgi.Controller):
         final_values['storage_mappings'] = self._update_storage_mappings(
             original_storage_mappings, new_storage_mappings)
 
+        final_values['user_scripts'] = self._get_updated_user_scripts(
+            replica.get('user_scripts', {}),
+            updated_values.get('user_scripts', {}))
+
         if 'notes' in updated_values:
             final_values['notes'] = updated_values.get('notes', '')
         else:
@@ -276,6 +304,11 @@ class ReplicaController(api_wsgi.Controller):
         api_utils.validate_storage_mappings(
             merged_body["storage_mappings"])
 
+        user_scripts = merged_body['user_scripts']
+        api_utils.validate_user_scripts(user_scripts)
+        merged_body['user_scripts'] = api_utils.normalize_user_scripts(
+            user_scripts, replica.get('instances', []))
+
         return merged_body
 
     def update(self, req, id, body):

coriolis/api/v1/utils.py

@@ -7,6 +7,7 @@ import json
 from oslo_log import log as logging
 from webob import exc
 
+from coriolis import constants
 from coriolis import exception
 from coriolis import schemas
 
@@ -65,8 +66,8 @@ def format_keyerror_message(resource='', method=''):
                 raise exception.InvalidInput(exc_message)
             except Exception as err:
                 LOG.exception(err)
-                msg = getattr(err, "message", str(err))
-                raise exception.InvalidInput(msg)
+                msg = getattr(err, "msg", str(err))
+                raise exception.InvalidInput(reason=msg)
         return _wrapper
     return _format_keyerror_message
 
@@ -85,3 +86,42 @@ def _build_keyerror_message(resource, method, key):
             resource, method_mapping[method], key)
 
     return msg
+
+
+def validate_user_scripts(user_scripts):
+    if not isinstance(user_scripts, dict):
+        raise exception.InvalidInput(
+            reason='"user_scripts" must be of JSON object format')
+
+    global_scripts = user_scripts.get('global', {})
+    if not isinstance(global_scripts, dict):
+        raise exception.InvalidInput(
+            reason='"global" must be a mapping between the identifiers of the '
+                   'supported OS types and their respective scripts.')
+    for os_type in global_scripts.keys():
+        if os_type not in constants.VALID_OS_TYPES:
+            raise exception.InvalidInput(
+                reason='The provided global user script os_type "%s" is '
+                       'invalid. Must be one of the '
+                       'following: %s' % (os_type, constants.VALID_OS_TYPES))
+
+    instance_scripts = user_scripts.get('instances', {})
+    if not isinstance(instance_scripts, dict):
+        raise exception.InvalidInput(
+            reason='"instances" must be a mapping between the identifiers of '
+                   'the instances in the Replica/Migration and their '
+                   'respective scripts.')
+
+
+def normalize_user_scripts(user_scripts, instances):
+    """ Removes instance user_scripts if said instance is not one of the
+        selected instances for the replica/migration """
+    for instance in user_scripts.get('instances', {}).keys():
+        if instance not in instances:
+            LOG.warn("Removing provided instance '%s' from user_scripts body "
+                     "because it's not included in one of the selected "
+                     "instances for this replica/migration: %s",
+                     instance, instances)
+            user_scripts.pop(instance, None)
+
+    return user_scripts

coriolis/conductor/rpc/client.py

@@ -161,7 +161,7 @@ class ConductorClient(rpc.BaseRPCClient):
                                  instance_osmorphing_minion_pool_mappings,
                                  source_environment, destination_environment,
                                  instances, network_map, storage_mappings,
-                                 notes=None):
+                                 notes=None, user_scripts=None):
         return self._call(
             ctxt, 'create_instances_replica',
             origin_endpoint_id=origin_endpoint_id,
@@ -175,7 +175,8 @@ class ConductorClient(rpc.BaseRPCClient):
             notes=notes,
             network_map=network_map,
             storage_mappings=storage_mappings,
-            source_environment=source_environment)
+            source_environment=source_environment,
+            user_scripts=user_scripts)
 
     def get_replicas(self, ctxt, include_tasks_executions=False):
         return self._call(

coriolis/conductor/rpc/server.py

@@ -1397,7 +1397,8 @@ class ConductorServerEndpoint(object):
                                  instance_osmorphing_minion_pool_mappings,
                                  source_environment,
                                  destination_environment, instances,
-                                 network_map, storage_mappings, notes=None):
+                                 network_map, storage_mappings, notes=None,
+                                 user_scripts=None):
         origin_endpoint = self.get_endpoint(ctxt, origin_endpoint_id)
         destination_endpoint = self.get_endpoint(ctxt, destination_endpoint_id)
         self._check_endpoints(ctxt, origin_endpoint, destination_endpoint)
@@ -1421,6 +1422,7 @@ class ConductorServerEndpoint(object):
         replica.storage_mappings = storage_mappings
         replica.instance_osmorphing_minion_pool_mappings = (
             instance_osmorphing_minion_pool_mappings)
+        replica.user_scripts = user_scripts
 
         self._check_minion_pools_for_action(ctxt, replica)
 
@@ -1505,6 +1507,7 @@ class ConductorServerEndpoint(object):
             replica, licensing_client.RESERVATION_TYPE_REPLICA)
         self._check_replica_running_executions(ctxt, replica)
         self._check_valid_replica_tasks_execution(replica, force)
+        user_scripts = user_scripts or replica.user_scripts
 
         destination_endpoint = self.get_endpoint(
             ctxt, replica.destination_endpoint_id)
@@ -1538,6 +1541,7 @@ class ConductorServerEndpoint(object):
         migration.instances = instances
         migration.replica = replica
         migration.info = replica.info
+        migration.user_scripts = user_scripts
         migration.origin_minion_pool_id = replica.origin_minion_pool_id
         migration.destination_minion_pool_id = (
             replica.destination_minion_pool_id)
@@ -2035,6 +2039,7 @@ class ConductorServerEndpoint(object):
         migration.executions = [execution]
         migration.instances = instances
         migration.info = {}
+        migration.user_scripts = user_scripts
         migration.notes = notes
         migration.shutdown_instances = shutdown_instances
         migration.replication_count = replication_count
@@ -3845,6 +3850,8 @@ class ConductorServerEndpoint(object):
 
         self._check_replica_running_executions(ctxt, replica)
         self._check_valid_replica_tasks_execution(replica, force=True)
+        if updated_properties.get('user_scripts'):
+            replica.user_scripts = updated_properties['user_scripts']
         execution = models.TasksExecution()
         execution.id = str(uuid.uuid4())
         execution.status = constants.EXECUTION_STATUS_UNEXECUTED

coriolis/db/sqlalchemy/migrate_repo/versions/017_adds_user_scripts_column.py

@@ -0,0 +1,17 @@
+# Copyright 2020 Cloudbase Solutions Srl
+# All Rights Reserved.
+
+import sqlalchemy
+
+
+def upgrade(migrate_engine):
+    meta = sqlalchemy.MetaData()
+    meta.bind = migrate_engine
+
+    # add 'user_scripts' column to 'base_transfer_action':
+    base_transfer = sqlalchemy.Table(
+        'base_transfer_action', meta, autoload=True)
+
+    user_scripts = sqlalchemy.Column(
+        "user_scripts", sqlalchemy.Text, nullable=True)
+    base_transfer.create_column(user_scripts)

coriolis/db/sqlalchemy/models.py

@@ -206,6 +206,7 @@ class BaseTransferAction(BASE, models.TimestampMixin, models.ModelBase,
         sqlalchemy.String(36), nullable=True)
     instance_osmorphing_minion_pool_mappings = sqlalchemy.Column(
         types.Json, nullable=False, default=lambda: {})
+    user_scripts = sqlalchemy.Column(types.Json, nullable=True)
 
     __mapper_args__ = {
         'polymorphic_identity': 'base_transfer_action',
@@ -237,7 +238,8 @@ class BaseTransferAction(BASE, models.TimestampMixin, models.ModelBase,
             "origin_minion_pool_id": self.origin_minion_pool_id,
             "destination_minion_pool_id": self.destination_minion_pool_id,
             "instance_osmorphing_minion_pool_mappings":
-                self.instance_osmorphing_minion_pool_mappings
+                self.instance_osmorphing_minion_pool_mappings,
+            "user_scripts": self.user_scripts,
         }
         if include_executions:
             for ex in self.executions:

coriolis/replicas/api.py

@@ -12,13 +12,13 @@ class API(object):
                origin_minion_pool_id, destination_minion_pool_id,
                instance_osmorphing_minion_pool_mappings,
                source_environment, destination_environment, instances,
-               network_map, storage_mappings, notes=None):
+               network_map, storage_mappings, notes=None, user_scripts=None):
         return self._rpc_client.create_instances_replica(
             ctxt, origin_endpoint_id, destination_endpoint_id,
             origin_minion_pool_id, destination_minion_pool_id,
             instance_osmorphing_minion_pool_mappings,
             source_environment, destination_environment, instances,
-            network_map, storage_mappings, notes)
+            network_map, storage_mappings, notes, user_scripts)
 
     def update(self, ctxt, replica_id, updated_properties):
         return self._rpc_client.update_replica(