db: Port database migration scripts from sqlalchemy-migrate to alembic

Updates database migrations to alembic. Most updates are
straight-forward and repetitive, which some exceptions: 001, 016, 018, 020.

In some scripts (e.g.: 001), the code was shifted around to account for the
proper table creation order (e.g.: table Y might have a foreign key from table X).

Scripts 014 and 016 deviate from the alembic db migration script pattern in
order to keep diffs to a minimum.

Downgrades are left unimplemented (raise NotImplementedError), matching the
original scripts which never defined a downgrade() either.

coriolis/db/sqlalchemy/alembic/versions/001_initial.py

@@ -1,17 +1,28 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""initial
+
+Revision ID: 001
+Revises:
+Create Date: 2016-01-15 22:28:25.000000
+"""
+
 import uuid
 
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "001"
+down_revision = None
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta,
+def upgrade():
+    op.create_table(
+        'base_transfer_action',
         sqlalchemy.Column("base_id", sqlalchemy.String(36), primary_key=True,
                           default=lambda: str(uuid.uuid4())),
         sqlalchemy.Column('created_at', sqlalchemy.DateTime),
@@ -31,8 +42,18 @@ def upgrade(migrate_engine):
         mysql_charset='utf8'
     )
 
-    migration = sqlalchemy.Table(
-        'migration', meta,
+    op.create_table(
+        'replica',
+        sqlalchemy.Column("id", sqlalchemy.String(36),
+                          sqlalchemy.ForeignKey(
+                              'base_transfer_action.base_id'),
+                          primary_key=True),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'
+    )
+
+    op.create_table(
+        'migration',
         sqlalchemy.Column("id", sqlalchemy.String(36),
                           sqlalchemy.ForeignKey(
                               'base_transfer_action.base_id'),
@@ -44,8 +65,26 @@ def upgrade(migrate_engine):
         mysql_charset='utf8'
     )
 
-    task = sqlalchemy.Table(
-        'task', meta, sqlalchemy.Column(
+    op.create_table(
+        'tasks_execution',
+        sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True,
+                          default=lambda: str(uuid.uuid4())),
+        sqlalchemy.Column('created_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('deleted_at', sqlalchemy.DateTime),
+        sqlalchemy.Column('deleted', sqlalchemy.String(36)),
+        sqlalchemy.Column("action_id", sqlalchemy.String(36),
+                          sqlalchemy.ForeignKey(
+                              'base_transfer_action.base_id'),
+                          nullable=False),
+        sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False),
+        sqlalchemy.Column("number", sqlalchemy.Integer, nullable=False),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'
+    )
+
+    op.create_table(
+        'task', sqlalchemy.Column(
             'id', sqlalchemy.String(36),
             primary_key=True, default=lambda: str(uuid.uuid4())),
         sqlalchemy.Column('created_at', sqlalchemy.DateTime),
@@ -76,26 +115,8 @@ def upgrade(migrate_engine):
         sqlalchemy.Column("on_error", sqlalchemy.Boolean, nullable=True),
         mysql_engine='InnoDB', mysql_charset='utf8')
 
-    tasks_execution = sqlalchemy.Table(
-        'tasks_execution', meta,
-        sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True,
-                          default=lambda: str(uuid.uuid4())),
-        sqlalchemy.Column('created_at', sqlalchemy.DateTime),
-        sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
-        sqlalchemy.Column('deleted_at', sqlalchemy.DateTime),
-        sqlalchemy.Column('deleted', sqlalchemy.String(36)),
-        sqlalchemy.Column("action_id", sqlalchemy.String(36),
-                          sqlalchemy.ForeignKey(
-                              'base_transfer_action.base_id'),
-                          nullable=False),
-        sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False),
-        sqlalchemy.Column("number", sqlalchemy.Integer, nullable=False),
-        mysql_engine='InnoDB',
-        mysql_charset='utf8'
-    )
-
-    task_progress_update = sqlalchemy.Table(
-        'task_progress_update', meta,
+    op.create_table(
+        'task_progress_update',
         sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True,
                           default=lambda: str(uuid.uuid4())),
         sqlalchemy.Column('created_at', sqlalchemy.DateTime),
@@ -112,8 +133,8 @@ def upgrade(migrate_engine):
         mysql_charset='utf8'
     )
 
-    task_events = sqlalchemy.Table(
-        'task_event', meta,
+    op.create_table(
+        'task_event',
         sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True,
                           default=lambda: str(uuid.uuid4())),
         sqlalchemy.Column('created_at', sqlalchemy.DateTime),
@@ -129,31 +150,8 @@ def upgrade(migrate_engine):
         mysql_charset='utf8'
     )
 
-    replica = sqlalchemy.Table(
-        'replica', meta,
-        sqlalchemy.Column("id", sqlalchemy.String(36),
-                          sqlalchemy.ForeignKey(
-                              'base_transfer_action.base_id'),
-                          primary_key=True),
-        mysql_engine='InnoDB',
-        mysql_charset='utf8'
-    )
-
-    tables = (
-        base_transfer_action,
-        replica,
-        migration,
-        tasks_execution,
-        task,
-        task_progress_update,
-        task_events,
-    )
 
-    for index, table in enumerate(tables):
-        try:
-            table.create()
-        except Exception:
-            # If an error occurs, drop all tables created so far to return
-            # to the previously existing state.
-            meta.drop_all(tables=tables[:index])
-            raise
+def downgrade():
+    # Downgrades were never supported by the original sqlalchemy-migrate
+    # migrations this revision chain was ported from.
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/002_adds_endpoints.py

@@ -1,17 +1,28 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds endpoints
+
+Revision ID: 002
+Revises: 001
+Create Date: 2017-03-22 22:17:09.000000
+"""
+
 import uuid
 
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "002"
+down_revision = "001"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    endpoint = sqlalchemy.Table(
-        'endpoint', meta,
+def upgrade():
+    op.create_table(
+        'endpoint',
         sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True,
                           default=lambda: str(uuid.uuid4())),
         sqlalchemy.Column('created_at', sqlalchemy.DateTime),
@@ -29,37 +40,27 @@ def upgrade(migrate_engine):
         mysql_charset='utf8'
     )
 
-    tables = (
-        endpoint,
-    )
-
-    for index, table in enumerate(tables):
-        try:
-            table.create()
-        except Exception:
-            # If an error occurs, drop all tables created so far to return
-            # to the previously existing state.
-            meta.drop_all(tables=tables[:index])
-            raise
-
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True)
-
     # NOTE(alexpilotti) delete all records in base_transfer_action
     # before performing this migration
     origin_endpoint_id = sqlalchemy.Column(
         "origin_endpoint_id", sqlalchemy.String(36),
         sqlalchemy.ForeignKey('endpoint.id'), nullable=False)
-    base_transfer_action.create_column(origin_endpoint_id)
+    op.add_column("base_transfer_action", origin_endpoint_id)
 
     destination_endpoint_id = sqlalchemy.Column(
         "destination_endpoint_id", sqlalchemy.String(36),
         sqlalchemy.ForeignKey('endpoint.id'), nullable=False)
-    base_transfer_action.create_column(destination_endpoint_id)
+    op.add_column("base_transfer_action", destination_endpoint_id)
 
     destination_environment = sqlalchemy.Column(
         "destination_environment", sqlalchemy.Text, nullable=True)
-    base_transfer_action.create_column(destination_environment)
+    op.add_column("base_transfer_action", destination_environment)
+
+    op.drop_column("base_transfer_action", "origin")
+    op.drop_column("base_transfer_action", "destination")
+
 
-    base_transfer_action.drop_column("origin")
-    base_transfer_action.drop_column("destination")
+def downgrade():
+    # Downgrades were never supported by the original sqlalchemy-migrate
+    # migrations this revision chain was ported from.
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/003_adds_notes.py

@@ -1,17 +1,29 @@
 # Copyright 2017 Cloudbase Solutions Srl
 # All Rights Reserved.
 
-import sqlalchemy
+"""adds notes
+
+Revision ID: 003
+Revises: 002
+Create Date: 2017-05-04 13:47:52.000000
+"""
 
+from alembic import op
+import sqlalchemy
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
+# revision identifiers, used by Alembic.
+revision = "003"
+down_revision = "002"
+branch_labels = None
+depends_on = None
 
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True)
 
+def upgrade():
     notes = sqlalchemy.Column(
         "notes", sqlalchemy.Text, nullable=True)
 
-    base_transfer_action.create_column(notes)
+    op.add_column("base_transfer_action", notes)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/004_adds_replica_schedules.py

@@ -1,17 +1,25 @@
+"""adds replica schedules
+
+Revision ID: 004
+Revises: 003
+Create Date: 2017-11-04 18:17:31.000000
+"""
+
 import uuid
 
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "004"
+down_revision = "003"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    sqlalchemy.Table(
-        'replica', meta, autoload=True)
-
-    replica_schedules = sqlalchemy.Table(
-        'replica_schedules', meta,
+def upgrade():
+    op.create_table(
+        'replica_schedules',
         sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True,
                           default=lambda: str(uuid.uuid4())),
         sqlalchemy.Column('created_at', sqlalchemy.DateTime),
@@ -32,15 +40,6 @@ def upgrade(migrate_engine):
         mysql_charset='utf8'
     )
 
-    tables = (
-        replica_schedules,
-    )
 
-    for index, table in enumerate(tables):
-        try:
-            table.create()
-        except Exception:
-            # If an error occurs, drop all tables created so far to return
-            # to the previously existing state.
-            meta.drop_all(tables=tables[:index])
-            raise
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/005_adds_transfer_result.py

@@ -1,17 +1,29 @@
 # Copyright 2018 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds transfer result
+
+Revision ID: 005
+Revises: 004
+Create Date: 2018-09-18 18:33:14.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "005"
+down_revision = "004"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
+def upgrade():
     # add 'transfer_result' column to 'base_transfer_action':
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True)
-
     transfer_result = sqlalchemy.Column(
         "transfer_result", sqlalchemy.Text, nullable=True)
-    base_transfer_action.create_column(transfer_result)
+    op.add_column("base_transfer_action", transfer_result)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/006_adds_network_map.py

@@ -1,17 +1,29 @@
 # Copyright 2018 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds network map
+
+Revision ID: 006
+Revises: 005
+Create Date: 2018-11-01 16:43:05.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "006"
+down_revision = "005"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
+def upgrade():
     # add 'network_map' column to 'base_transfer_action':
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True)
-
     network_map = sqlalchemy.Column(
         "network_map", sqlalchemy.Text, nullable=True)
-    base_transfer_action.create_column(network_map)
+    op.add_column("base_transfer_action", network_map)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/007_adds_storage_mappings.py

@@ -1,17 +1,29 @@
 # Copyright 2018 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds storage mappings
+
+Revision ID: 007
+Revises: 006
+Create Date: 2018-11-13 13:37:09.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "007"
+down_revision = "006"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
+def upgrade():
     # add 'storage_mappings' column to 'base_transfer_action':
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True)
-
     storage_mappings = sqlalchemy.Column(
         "storage_mappings", sqlalchemy.Text, nullable=True)
-    base_transfer_action.create_column(storage_mappings)
+    op.add_column("base_transfer_action", storage_mappings)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/008_adds_source_environment.py

@@ -1,17 +1,29 @@
 # Copyright 2018 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds source environment
+
+Revision ID: 008
+Revises: 007
+Create Date: 2018-11-27 18:48:35.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "008"
+down_revision = "007"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
+def upgrade():
     # add 'source_environment' column to 'base_transfer_action':
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True)
-
     source_environment = sqlalchemy.Column(
         "source_environment", sqlalchemy.Text, nullable=True)
-    base_transfer_action.create_column(source_environment)
+    op.add_column("base_transfer_action", source_environment)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/009_migrate_info_to_blob.py

@@ -1,12 +1,24 @@
-import sqlalchemy
+"""migrate info to blob
+
+Revision ID: 009
+Revises: 008
+Create Date: 2019-03-23 00:15:49.000000
+"""
+
+from alembic import op
 from sqlalchemy import types
 
+# revision identifiers, used by Alembic.
+revision = "009"
+down_revision = "008"
+branch_labels = None
+depends_on = None
+
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
+def upgrade():
+    op.alter_column(
+        'base_transfer_action', 'info', type_=types.LargeBinary(4294967295))
 
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True)
 
-    base_transfer_action.c.info.alter(type=types.LargeBinary(4294967295))
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/010_adds_reservation_id.py

@@ -1,17 +1,29 @@
 # Copyright 2018 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds reservation id
+
+Revision ID: 010
+Revises: 009
+Create Date: 2019-04-26 22:47:10.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "010"
+down_revision = "009"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
+def upgrade():
     # add 'reservation_id' column to 'base_transfer_action':
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True)
-
     reservation_id = sqlalchemy.Column(
         "reservation_id", sqlalchemy.String(36), nullable=True)
-    base_transfer_action.create_column(reservation_id)
+    op.add_column("base_transfer_action", reservation_id)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/011_adds_execution_type.py

@@ -1,17 +1,29 @@
 # Copyright 2019 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds execution type
+
+Revision ID: 011
+Revises: 010
+Create Date: 2019-08-15 07:30:35.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "011"
+down_revision = "010"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
+def upgrade():
     # add 'type' column to 'tasks_execution':
-    tasks_execution = sqlalchemy.Table(
-        'tasks_execution', meta, autoload=True)
-
     execution_type = sqlalchemy.Column(
         "type", sqlalchemy.String(20))
-    tasks_execution.create_column(execution_type)
+    op.add_column("tasks_execution", execution_type)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/012_adds_migration_sync_fields.py

@@ -1,19 +1,31 @@
-import sqlalchemy
+"""adds migration sync fields
+
+Revision ID: 012
+Revises: 011
+Create Date: 2019-10-16 15:40:42.000000
+"""
 
+from alembic import op
+import sqlalchemy
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
+# revision identifiers, used by Alembic.
+revision = "012"
+down_revision = "011"
+branch_labels = None
+depends_on = None
 
-    migration = sqlalchemy.Table(
-        'migration', meta, autoload=True)
 
+def upgrade():
     shutdown_instances = sqlalchemy.Column(
         "shutdown_instances", sqlalchemy.Boolean,
-        nullable=False, default=False)
-    migration.create_column(shutdown_instances)
+        nullable=False, server_default=sqlalchemy.false())
+    op.add_column("migration", shutdown_instances)
 
     replication_count = sqlalchemy.Column(
-        "replication_count", sqlalchemy.Integer, default=0,
-        nullable=False)
-    migration.create_column(replication_count)
+        "replication_count", sqlalchemy.Integer,
+        nullable=False, server_default="0")
+    op.add_column("migration", replication_count)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/013_adds_task_index.py

@@ -1,13 +1,25 @@
-import sqlalchemy
+"""adds task index
+
+Revision ID: 013
+Revises: 012
+Create Date: 2019-10-18 19:35:20.000000
+"""
 
+from alembic import op
+import sqlalchemy
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
+# revision identifiers, used by Alembic.
+revision = "013"
+down_revision = "012"
+branch_labels = None
+depends_on = None
 
-    task = sqlalchemy.Table('task', meta, autoload=True)
 
+def upgrade():
     index = sqlalchemy.Column(
-        "index", sqlalchemy.Integer, default=0, nullable=False)
+        "index", sqlalchemy.Integer, nullable=False, server_default="0")
+    op.add_column("task", index)
+
 
-    task.create_column(index)
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/014_adds_worker_service_regions.py

@@ -1,17 +1,31 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds worker service regions
+
+Revision ID: 014
+Revises: 013
+Create Date: 2020-07-28 18:21:57.000000
+"""
+
 import uuid
 
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "014"
+down_revision = "013"
+branch_labels = None
+depends_on = None
+
 
-def upgrade(migrate_engine):
+def upgrade():
     meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    sqlalchemy.Table(
-        'endpoint', meta, autoload=True)
+    # load 'endpoint' into meta so the 'endpoint_id' foreign key below can
+    # be resolved against it.
+    sqlalchemy.Table('endpoint', meta, autoload_with=op.get_bind())
 
     tables = []
 
@@ -122,11 +136,9 @@ def upgrade(migrate_engine):
             mysql_engine='InnoDB',
             mysql_charset='utf8'))
 
-    for index, table in enumerate(tables):
-        try:
-            table.create()
-        except Exception:
-            # If an error occurs, drop all tables created so far to return
-            # to the previously existing state.
-            meta.drop_all(tables=tables[:index])
-            raise
+    for table in tables:
+        table.create(bind=op.get_bind())
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/015_adds_action_last_execution_status.py

@@ -1,20 +1,30 @@
 # Copyright 2020 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds action last execution status
+
+Revision ID: 015
+Revises: 014
+Create Date: 2020-08-27 20:38:25.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "015"
+down_revision = "014"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
+def upgrade():
     # add 'last_execution_status' column to 'base_transfer_action':
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True,
-        mysql_engine="InnoDB",
-        mysql_charset="utf8")
-
     last_execution_status = sqlalchemy.Column(
         "last_execution_status", sqlalchemy.String(255),
-        default=lambda: "UNEXECUTED", nullable=False)
-    base_transfer_action.create_column(last_execution_status)
+        nullable=False, server_default="UNEXECUTED")
+    op.add_column("base_transfer_action", last_execution_status)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/016_adds_minion_vm_pools.py

@@ -1,24 +1,39 @@
 # Copyright 2020 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds minion vm pools
+
+Revision ID: 016
+Revises: 015
+Create Date: 2020-08-26 07:10:03.000000
+"""
+
 import uuid
 
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "016"
+down_revision = "015"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    base_transfer_action = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True,
-        mysql_engine="InnoDB",
-        mysql_charset="utf8")
+def upgrade():
+    meta = sqlalchemy.MetaData()
 
     # extend tasks execution 'type' column:
-    tasks_execution = sqlalchemy.Table(
-        'tasks_execution', meta, autoload=True)
-    tasks_execution.c.type.alter(type=sqlalchemy.String(255))
+    op.alter_column(
+        'tasks_execution', 'type',
+        existing_type=sqlalchemy.String(20),
+        existing_nullable=True,
+        type_=sqlalchemy.String(255),
+        nullable=True)
+
+    # load 'endpoint' into meta so the 'endpoint_id' foreign key below can
+    # be resolved against it.
+    sqlalchemy.Table('endpoint', meta, autoload_with=op.get_bind())
 
     tables = []
 
@@ -153,30 +168,26 @@ def upgrade(migrate_engine):
                 "message", sqlalchemy.Text, nullable=True),
             mysql_engine='InnoDB', mysql_charset='utf8'))
 
+    for table in tables:
+        table.create(bind=op.get_bind())
+
     # add the pool option properties for the transfer:
     origin_minion_pool_id = sqlalchemy.Column(
         "origin_minion_pool_id", sqlalchemy.String(36),
         sqlalchemy.ForeignKey('minion_pool.id'), nullable=True)
+    op.add_column("base_transfer_action", origin_minion_pool_id)
+
     destination_minion_pool_id = sqlalchemy.Column(
         "destination_minion_pool_id", sqlalchemy.String(36),
         sqlalchemy.ForeignKey('minion_pool.id'), nullable=True)
+    op.add_column("base_transfer_action", destination_minion_pool_id)
+
     instance_osmorphing_minion_pool_mappings = sqlalchemy.Column(
         "instance_osmorphing_minion_pool_mappings", sqlalchemy.Text,
-        nullable=False, default='{}')
-
-    created_columns = []
-    try:
-        for index, table in enumerate(tables):
-            table.create()
-        for col in [
-                origin_minion_pool_id, destination_minion_pool_id,
-                instance_osmorphing_minion_pool_mappings]:
-            base_transfer_action.create_column(col)
-            created_columns.append(col)
-    except Exception:
-        # If an error occurs, drop all tables created so far to return
-        # to the previously existing state.
-        for col in created_columns:
-            base_transfer_action.drop_column(col)
-        meta.drop_all(tables=tables[:index])
-        raise
+        nullable=False, server_default='{}')
+    op.add_column(
+        "base_transfer_action", instance_osmorphing_minion_pool_mappings)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/017_adds_user_scripts_column.py

@@ -1,19 +1,29 @@
 # Copyright 2020 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""adds user scripts column
+
+Revision ID: 017
+Revises: 016
+Create Date: 2021-01-26 19:16:44.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "017"
+down_revision = "016"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
+def upgrade():
     # add 'user_scripts' column to 'base_transfer_action':
-    base_transfer = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True,
-        mysql_engine="InnoDB",
-        mysql_charset="utf8")
-
     user_scripts = sqlalchemy.Column(
         "user_scripts", sqlalchemy.Text, nullable=True)
-    base_transfer.create_column(user_scripts)
+    op.add_column("base_transfer_action", user_scripts)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/018_adds_task_progress_idices.py

@@ -1,21 +1,41 @@
+"""adds task progress idices
+
+Revision ID: 018
+Revises: 017
+Create Date: 2021-01-07 17:33:05.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "018"
+down_revision = "017"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    task_event = sqlalchemy.Table('task_event', meta, autoload=True)
+def upgrade():
     event_index = sqlalchemy.Column(
-        "index", sqlalchemy.Integer, default=0, nullable=False)
-    task_event.create_column(event_index)
+        "index", sqlalchemy.Integer, nullable=False, server_default="0")
+    op.add_column("task_event", event_index)
 
-    task_progress_update = sqlalchemy.Table(
-        'task_progress_update', meta, autoload=True,
-        mysql_engine="InnoDB",
-        mysql_charset="utf8")
     progress_index = sqlalchemy.Column(
-        "index", sqlalchemy.Integer, default=0, nullable=False)
-    task_progress_update.create_column(progress_index)
-    task_progress_update.c.current_step.alter(type=sqlalchemy.BigInteger)
-    task_progress_update.c.total_steps.alter(type=sqlalchemy.BigInteger)
+        "index", sqlalchemy.Integer, nullable=False, server_default="0")
+    op.add_column("task_progress_update", progress_index)
+    op.alter_column(
+        "task_progress_update", "current_step",
+        existing_type=sqlalchemy.Integer,
+        existing_nullable=False,
+        type_=sqlalchemy.BigInteger,
+        nullable=False)
+    op.alter_column(
+        "task_progress_update", "total_steps",
+        existing_type=sqlalchemy.Integer,
+        existing_nullable=True,
+        type_=sqlalchemy.BigInteger,
+        nullable=True)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/019_add_replica_scenario_field.py

@@ -1,20 +1,29 @@
 # Copyright 2024 Cloudbase Solutions Srl
 # All Rights Reserved.
 
-import sqlalchemy
+"""add replica scenario field
+
+Revision ID: 019
+Revises: 018
+Create Date: 2024-04-09 14:42:14.000000
+"""
 
+from alembic import op
+import sqlalchemy
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
+# revision identifiers, used by Alembic.
+revision = "019"
+down_revision = "018"
+branch_labels = None
+depends_on = None
 
-    replica = sqlalchemy.Table(
-        'replica', meta, autoload=True,
-        mysql_engine="InnoDB",
-        mysql_charset="utf8")
 
+def upgrade():
     replica_scenario = sqlalchemy.Column(
         "scenario", sqlalchemy.String(255), nullable=False,
-        default="replica")
+        server_default="replica")
+    op.add_column("replica", replica_scenario)
+
 
-    replica.create_column(replica_scenario)
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/020_rename_tables.py

@@ -1,12 +1,40 @@
+"""rename tables
+
+Revision ID: 020
+Revises: 019
+Create Date: 2024-10-29 15:26:01.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "020"
+down_revision = "019"
+branch_labels = None
+depends_on = None
+
+# NOTE(dvincze): Update models polymorphic identity. Due to the model code
+# changes, this cannot be done using the ORM; raw SQL is required.
+_TYPE_RENAMES = (
+    ("base_transfer_action", "replica", "transfer"),
+    ("tasks_execution", "replica_execution", "transfer_execution"),
+    ("tasks_execution", "replica_disks_delete", "transfer_disks_delete"),
+    ("tasks_execution", "replica_deploy", "deployment"),
+    ("tasks_execution", "replica_update", "transfer_update"),
+)
 
-def upgrade(migrate_engine):
+
+def upgrade():
     meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    replica = sqlalchemy.Table('replica', meta, autoload=True)
-    replica.rename('transfer')
+    op.rename_table('replica', 'transfer')
+
+    # load 'base_transfer_action' and 'transfer' into meta so the foreign
+    # keys below can be resolved against them.
+    sqlalchemy.Table(
+        'base_transfer_action', meta, autoload_with=op.get_bind())
+    sqlalchemy.Table('transfer', meta, autoload_with=op.get_bind())
 
     deployment = sqlalchemy.Table(
         'deployment', meta,
@@ -19,32 +47,22 @@ def upgrade(migrate_engine):
                           nullable=False),
         mysql_engine="InnoDB",
         mysql_charset="utf8")
-    try:
-        deployment.create()
-    except Exception:
-        deployment.drop()
-        raise
-
-    replica_schedule = sqlalchemy.Table(
-        'replica_schedules', meta, autoload=True)
-    replica_schedule.rename('transfer_schedules')
-    replica_schedule.c.replica_id.alter(name='transfer_id')
-
-    # NOTE(dvincze): Update models polymorphic identity
-    # Due to the model code changes, this cannot be done using the ORM.
-    # Had to resort to using raw SQL statements.
-    with migrate_engine.connect() as conn:
-        conn.execute(
-            'UPDATE base_transfer_action SET type = "transfer" '
-            'WHERE type = "replica";')
-        conn.execute('UPDATE tasks_execution SET type = "transfer_execution"'
-                     'WHERE type = "replica_execution"')
-        conn.execute(
-            'UPDATE tasks_execution SET type = "transfer_disks_delete"'
-            'WHERE type = "replica_disks_delete"')
-        conn.execute(
-            'UPDATE tasks_execution SET type = "deployment"'
-            'WHERE type = "replica_deploy"')
-        conn.execute(
-            'UPDATE tasks_execution SET type = "transfer_update"'
-            'WHERE type = "replica_update"')
+    deployment.create(bind=op.get_bind())
+
+    op.rename_table('replica_schedules', 'transfer_schedules')
+    op.alter_column(
+        'transfer_schedules', 'replica_id',
+        new_column_name='transfer_id',
+        existing_type=sqlalchemy.String(36),
+        existing_nullable=False)
+
+    for table, old_type, new_type in _TYPE_RENAMES:
+        op.execute(
+            sqlalchemy.text(
+                f"UPDATE {table} SET type = :new_type "
+                f"WHERE type = :old_type"
+            ).bindparams(new_type=new_type, old_type=old_type))
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/021_add_deployment_defaults.py

@@ -1,17 +1,30 @@
+"""add deployment defaults
+
+Revision ID: 021
+Revises: 020
+Create Date: 2025-01-29 19:13:39.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "021"
+down_revision = "020"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    base_transfer = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True,
-        mysql_engine="InnoDB",
-        mysql_charset="utf8")
+def upgrade():
     clone_disks = sqlalchemy.Column(
-        "clone_disks", sqlalchemy.Boolean, nullable=False, default=True)
-    base_transfer.create_column(clone_disks)
+        "clone_disks", sqlalchemy.Boolean, nullable=False,
+        server_default=sqlalchemy.true())
+    op.add_column("base_transfer_action", clone_disks)
     skip_os_morphing = sqlalchemy.Column(
-        "skip_os_morphing", sqlalchemy.Boolean, nullable=False, default=False)
-    base_transfer.create_column(skip_os_morphing)
+        "skip_os_morphing", sqlalchemy.Boolean, nullable=False,
+        server_default=sqlalchemy.false())
+    op.add_column("base_transfer_action", skip_os_morphing)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/022_adds_auto_deploy_column.py

@@ -1,14 +1,26 @@
+"""adds auto deploy column
+
+Revision ID: 022
+Revises: 021
+Create Date: 2025-01-30 13:54:39.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "022"
+down_revision = "021"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    transfer_schedule = sqlalchemy.Table(
-        'transfer_schedules', meta, autoload=True,
-        mysql_engine="InnoDB",
-        mysql_charset="utf8")
+def upgrade():
     auto_deploy = sqlalchemy.Column(
-        'auto_deploy', sqlalchemy.Boolean, nullable=False, default=False)
-    transfer_schedule.create_column(auto_deploy)
+        'auto_deploy', sqlalchemy.Boolean, nullable=False,
+        server_default=sqlalchemy.false())
+    op.add_column("transfer_schedules", auto_deploy)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/023_add_deployer_id.py

@@ -1,25 +1,28 @@
+"""add deployer id
+
+Revision ID: 023
+Revises: 022
+Create Date: 2025-02-06 14:42:29.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "023"
+down_revision = "022"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    deployment = sqlalchemy.Table(
-        'deployment', meta, autoload=True,
-        mysql_engine="InnoDB",
-        mysql_charset="utf8")
+def upgrade():
     deployer_id = sqlalchemy.Column(
         'deployer_id', sqlalchemy.String(36), nullable=True)
+    op.add_column("deployment", deployer_id)
     trust_id = sqlalchemy.Column(
         'trust_id', sqlalchemy.String(255), nullable=True)
-    created_columns = []
-    try:
-        deployment.create_column(deployer_id)
-        created_columns.append(deployer_id)
-        deployment.create_column(trust_id)
-        created_columns.append(trust_id)
-    except Exception:
-        for c in created_columns:
-            deployment.drop_column(c)
-        raise
+    op.add_column("deployment", trust_id)
+
+
+def downgrade():
+    raise NotImplementedError()

coriolis/db/sqlalchemy/alembic/versions/024_add_clustered_to_base_transfer_action.py

@@ -1,20 +1,36 @@
 # Copyright 2026 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+"""add clustered to base transfer action
+
+Revision ID: 024
+Revises: 023
+Create Date: 2026-04-09 04:03:23.000000
+"""
+
+from alembic import op
 import sqlalchemy
 
+# revision identifiers, used by Alembic.
+revision = "024"
+down_revision = "023"
+branch_labels = None
+depends_on = None
 
-def upgrade(migrate_engine):
-    meta = sqlalchemy.MetaData()
-    meta.bind = migrate_engine
 
-    base_transfer = sqlalchemy.Table(
-        'base_transfer_action', meta, autoload=True)
-    if 'clustered' in base_transfer.c:
+def upgrade():
+    inspector = sqlalchemy.inspect(op.get_bind())
+    columns = [c['name'] for c in inspector.get_columns(
+        'base_transfer_action')]
+    if 'clustered' in columns:
         return
     # server_default so existing rows get a value when the column is added
     # (MySQL stores booleans as TINYINT).
     clustered = sqlalchemy.Column(
         'clustered', sqlalchemy.Boolean, nullable=False,
         server_default=sqlalchemy.text('0'))
-    base_transfer.create_column(clustered)
+    op.add_column("base_transfer_action", clustered)
+
+
+def downgrade():
+    raise NotImplementedError()