db: Switch coriolis-dbsync from sqlalchemy-migrate to alembic

Rewrites coriolis/db/sqlalchemy/migration.py to drive the alembic
revision chain added instead of oslo_db.sqlalchemy.migration, which
no longer exists in current oslo.db releases.

db_sync auto-detects a pre-existing sqlalchemy-migrate `migrate_version`
table left over from the old migrate_repo. It stamps the database onto the
equivalent alembic revision instead of re-running already-applied migrations,
then proceeds to upgrade normally.

The now-empty migrate_repo/ is removed entirely.

The oslo.db upper constraint is removed.

coriolis/db/sqlalchemy/api.py

@@ -39,9 +39,11 @@ def get_backend():
 
 def db_sync(engine, version=None):
     """Migrate the database to `version` or the most recent version."""
-    if version is not None and int(version) < db_version(engine):
-        raise exception.CoriolisException(
-            _("Cannot migrate to lower schema version."))
+    if version is not None:
+        current_version = db_version(engine)
+        if current_version is not None and int(version) < int(current_version):
+            raise exception.CoriolisException(
+                _("Cannot migrate to lower schema version."))
 
     return migration.db_sync(engine, version=version)
 

coriolis/db/sqlalchemy/migrate_repo/manage.py

@@ -1,9 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2016 Cloudbase Solutions Srl
-# All Rights Reserved.
-
-from migrate.versioning.shell import main
-
-if __name__ == '__main__':
-    main(debug='False')

coriolis/db/sqlalchemy/migrate_repo/migrate.cfg

@@ -1,25 +0,0 @@
-[db_settings]
-# Used to identify which repository this database is versioned under.
-# You can use the name of your project.
-repository_id=coriolis
-
-# The name of the database table used to track the schema version.
-# This name shouldn't already be used by your project.
-# If this is changed once a database is under version control, you'll need to 
-# change the table name in each database too. 
-version_table=migrate_version
-
-# When committing a change script, Migrate will attempt to generate the 
-# sql for all supported databases; normally, if one of them fails - probably
-# because you don't have that database installed - it is ignored and the 
-# commit continues, perhaps ending successfully. 
-# Databases in this list MUST compile successfully during a commit, or the 
-# entire commit will fail. List the databases your application will actually 
-# be using to ensure your updates to that database work properly.
-# This must be a list; example: ['postgres','sqlite']
-required_dbs=[]
-
-# When creating new change scripts, Migrate will stamp the new script with
-# a version number. By default this is latest_version + 1. You can set this
-# to 'true' to tell Migrate to use the UTC timestamp instead.
-use_timestamp_numbering=False

coriolis/db/sqlalchemy/migration.py

@@ -3,25 +3,71 @@
 
 import os
 
-from oslo_db.sqlalchemy import migration as oslo_migration
+from alembic import command
+from alembic import config as alembic_config
+from alembic.runtime import migration as alembic_migration
+import sqlalchemy
 
-INIT_VERSION = 0
+from coriolis import exception
+from coriolis.i18n import _
+
+ALEMBIC_DIR = os.path.join(os.path.dirname(__file__), "alembic")
+ALEMBIC_INI_PATH = os.path.join(ALEMBIC_DIR, "alembic.ini")
+
+# The final version stamped by the old sqlalchemy-migrate migrate_repo
+# (migrate_repo/versions/024_add_clustered_to_base_transfer_action.py).
+LEGACY_VERSION_TABLE = "migrate_version"
+LEGACY_FINAL_VERSION = 24
+
+
+def _get_alembic_config():
+    config = alembic_config.Config(ALEMBIC_INI_PATH)
+    config.set_main_option("script_location", ALEMBIC_DIR)
+    return config
+
+
+def _stamp_legacy_database_if_needed(engine, config):
+    """Transition a sqlalchemy-migrate managed database to alembic.
+
+    If this database was previously managed by the old sqlalchemy-migrate
+    based migrate_repo, stamp it onto the equivalent alembic revision
+    instead of re-running the already-applied DDL.
+    """
+    inspector = sqlalchemy.inspect(engine)
+    if LEGACY_VERSION_TABLE not in inspector.get_table_names():
+        return
+
+    with engine.connect() as conn:
+        legacy_version = conn.execute(
+            sqlalchemy.text(
+                f"SELECT version FROM {LEGACY_VERSION_TABLE}")).scalar()
+
+    if legacy_version > LEGACY_FINAL_VERSION:
+        raise exception.CoriolisException(
+            _("This database was last migrated using the legacy "
+              "sqlalchemy-migrate based coriolis-dbsync (version %(cur)s), "
+              "which is newer than the last version known to alembic "
+              "(%(final)s).") % {
+                "cur": legacy_version, "final": LEGACY_FINAL_VERSION})
+
+    config.attributes["connection"] = engine.connect()
+    command.stamp(config, "%03d" % legacy_version)
 
 
 def db_sync(engine, version=None):
-    path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-                        'migrate_repo')
-    return oslo_migration.db_sync(engine, path, version,
-                                  init_version=INIT_VERSION)
+    config = _get_alembic_config()
+    _stamp_legacy_database_if_needed(engine, config)
+    config.attributes["connection"] = engine.connect()
+    return command.upgrade(config, version or "head")
 
 
 def db_version(engine):
-    path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-                        'migrate_repo')
-    return oslo_migration.db_version(engine, path, INIT_VERSION)
+    with engine.connect() as conn:
+        context = alembic_migration.MigrationContext.configure(conn)
+        return context.get_current_revision()
 
 
 def db_version_control(engine, version=None):
-    path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-                        'migrate_repo')
-    return oslo_migration.db_version_control(engine, path, version)
+    config = _get_alembic_config()
+    config.attributes["connection"] = engine.connect()
+    return command.stamp(config, version or "head")

coriolis/tests/db/sqlalchemy/test_api.py

@@ -94,9 +94,9 @@ class DatabaseSqlalchemyApiTestCase(test_base.CoriolisBaseTestCase):
         mock_db_version,
         mock_db_sync
     ):
-        mock_db_version.return_value = 1
+        mock_db_version.return_value = "001"
 
-        result = api.db_sync(mock.sentinel.engine, version=1)
+        result = api.db_sync(mock.sentinel.engine, version="002")
 
         self.assertEqual(
             mock_db_sync.return_value,
@@ -104,20 +104,39 @@ class DatabaseSqlalchemyApiTestCase(test_base.CoriolisBaseTestCase):
         )
         mock_db_version.assert_called_once_with(mock.sentinel.engine)
         mock_db_sync.assert_called_once_with(
-            mock.sentinel.engine, version=1)
+            mock.sentinel.engine, version="002")
+
+    @mock.patch.object(migration, 'db_sync')
+    @mock.patch.object(api, 'db_version')
+    def test_db_sync_version_no_current_version(
+        self,
+        mock_db_version,
+        mock_db_sync
+    ):
+        mock_db_version.return_value = None
+
+        result = api.db_sync(mock.sentinel.engine, version="001")
+
+        self.assertEqual(
+            mock_db_sync.return_value,
+            result
+        )
+        mock_db_version.assert_called_once_with(mock.sentinel.engine)
+        mock_db_sync.assert_called_once_with(
+            mock.sentinel.engine, version="001")
 
     @mock.patch.object(api, 'db_version')
     def test_db_sync_version_raise(
         self,
-        mock_db_version
+        mock_db_version,
     ):
-        mock_db_version.return_value = 2
+        mock_db_version.return_value = "003"
 
         self.assertRaises(
             exception.CoriolisException,
             api.db_sync,
             mock.sentinel.engine,
-            version=1
+            version="001"
         )
         mock_db_version.assert_called_once_with(mock.sentinel.engine)
 

coriolis/tests/db/sqlalchemy/test_migration.py

@@ -1,61 +1,113 @@
 # Copyright 2024 Cloudbase Solutions Srl
 # All Rights Reserved.
 
-import os
 from unittest import mock
 
-from oslo_db.sqlalchemy import migration as oslo_migration
-
 from coriolis.db.sqlalchemy import migration
+from coriolis import exception
 from coriolis.tests import test_base
 
 
 class DatabaseSqlalchemyMigrationTestCase(test_base.CoriolisBaseTestCase):
     """Test suite for the Coriolis Database Sqlalchemy migration."""
 
-    @mock.patch.object(os.path, 'abspath')
-    @mock.patch.object(oslo_migration, 'db_sync')
-    def test_db_sync(self, mock_db_sync, mock_abspath):
-        mock_abspath.return_value = "/abspath"
+    @mock.patch.object(migration, "sqlalchemy")
+    def test_stamp_legacy_database_if_needed_no_table(self, mock_sqlalchemy):
+        mock_sqlalchemy.inspect.return_value.get_table_names.return_value = [
+            "foo"]
+        mock_engine = mock.MagicMock()
+        mock_config = mock.MagicMock()
 
-        result = migration.db_sync(mock.sentinel.engine, mock.sentinel.version)
+        migration._stamp_legacy_database_if_needed(mock_engine, mock_config)
 
-        self.assertEqual(
-            mock_db_sync.return_value,
-            result
-        )
-        mock_db_sync.assert_called_once_with(
-            mock.sentinel.engine,
-            "/abspath/migrate_repo",
-            mock.sentinel.version,
-            init_version=0
-        )
+        mock_engine.connect.assert_not_called()
 
-    @mock.patch.object(os.path, 'abspath')
-    @mock.patch.object(oslo_migration, 'db_version')
-    def test_db_version(self, mock_db_version, mock_abspath):
-        mock_abspath.return_value = "/abspath"
+    @mock.patch.object(migration, "sqlalchemy")
+    def test_stamp_legacy_database_if_needed_raises(self, mock_sqlalchemy):
+        mock_sqlalchemy.inspect.return_value.get_table_names.return_value = [
+            migration.LEGACY_VERSION_TABLE]
+        mock_engine = mock.MagicMock()
+        mock_conn = mock_engine.connect.return_value.__enter__.return_value
+        mock_conn.execute.return_value.scalar.return_value = (
+            migration.LEGACY_FINAL_VERSION + 1)
+        mock_config = mock.MagicMock()
 
-        result = migration.db_version(mock.sentinel.engine)
+        self.assertRaises(
+            exception.CoriolisException,
+            migration._stamp_legacy_database_if_needed,
+            mock_engine, mock_config)
 
-        self.assertEqual(mock_db_version.return_value, result)
-        mock_db_version.assert_called_once_with(
-            mock.sentinel.engine,
-            "/abspath/migrate_repo",
-            0
-        )
+    @mock.patch.object(migration, "command")
+    @mock.patch.object(migration, "sqlalchemy")
+    def test_stamp_legacy_database_if_needed(
+        self, mock_sqlalchemy, mock_command,
+    ):
+        mock_sqlalchemy.inspect.return_value.get_table_names.return_value = [
+            migration.LEGACY_VERSION_TABLE]
+        mock_engine = mock.MagicMock()
+        mock_conn = mock_engine.connect.return_value.__enter__.return_value
+        mock_conn.execute.return_value.scalar.return_value = 10
+        mock_config = mock.MagicMock()
 
-    @mock.patch.object(os.path, 'abspath')
-    @mock.patch.object(oslo_migration, 'db_version_control')
-    def test_db_version_control(self, mock_db_version_control, mock_abspath):
-        mock_abspath.return_value = "/abspath"
+        migration._stamp_legacy_database_if_needed(mock_engine, mock_config)
 
-        result = migration.db_version_control(
-            mock.sentinel.engine, mock.sentinel.version)
+        mock_command.stamp.assert_called_once_with(mock_config, "010")
+
+    @mock.patch.object(migration, "command")
+    @mock.patch.object(migration, "sqlalchemy")
+    def test_stamp_legacy_database_if_needed_final_version(
+        self, mock_sqlalchemy, mock_command,
+    ):
+        mock_sqlalchemy.inspect.return_value.get_table_names.return_value = [
+            migration.LEGACY_VERSION_TABLE]
+        mock_engine = mock.MagicMock()
+        mock_conn = mock_engine.connect.return_value.__enter__.return_value
+        mock_conn.execute.return_value.scalar.return_value = (
+            migration.LEGACY_FINAL_VERSION)
+        mock_config = mock.MagicMock()
+
+        migration._stamp_legacy_database_if_needed(mock_engine, mock_config)
+
+        mock_command.stamp.assert_called_once_with(
+            mock_config, "%03d" % migration.LEGACY_FINAL_VERSION)
 
-        self.assertEqual(mock_db_version_control.return_value, result)
-        mock_db_version_control.assert_called_once_with(
-            mock.sentinel.engine,
-            "/abspath/migrate_repo",
-            mock.sentinel.version
+    @mock.patch.object(migration, "_stamp_legacy_database_if_needed")
+    @mock.patch.object(migration, "command")
+    @mock.patch.object(migration, "_get_alembic_config")
+    def test_db_sync(
+        self, mock_get_config, mock_command, mock_stamp_legacy,
+    ):
+        mock_engine = mock.MagicMock()
+
+        result = migration.db_sync(mock_engine, mock.sentinel.version)
+
+        self.assertEqual(mock_command.upgrade.return_value, result)
+        mock_get_config.assert_called_once_with()
+        mock_stamp_legacy.assert_called_once_with(
+            mock_engine, mock_get_config.return_value)
+        mock_command.upgrade.assert_called_once_with(
+            mock_get_config.return_value, mock.sentinel.version)
+
+    @mock.patch.object(migration.alembic_migration, "MigrationContext")
+    def test_db_version(self, mock_migration_context):
+        mock_engine = mock.MagicMock()
+
+        result = migration.db_version(mock_engine)
+
+        mock_conn = mock_engine.connect.return_value.__enter__.return_value
+        mock_configure = mock_migration_context.configure
+        mock_configure.assert_called_once_with(mock_conn)
+        self.assertEqual(
+            mock_configure.return_value.get_current_revision.return_value,
+            result,
         )
+
+    @mock.patch.object(migration, "command")
+    @mock.patch.object(migration, "_get_alembic_config")
+    def test_db_version_control(self, mock_get_config, mock_command):
+        result = migration.db_version_control(
+            mock.MagicMock(), mock.sentinel.version)
+
+        self.assertEqual(mock_command.stamp.return_value, result)
+        mock_command.stamp.assert_called_once_with(
+            mock_get_config.return_value, mock.sentinel.version)

requirements.txt

@@ -14,7 +14,7 @@ oslo.cache
 oslo.concurrency
 oslo.config<9.8.0
 oslo.context<6.0.0
-oslo.db<=12.3.2
+oslo.db
 oslo.i18n
 oslo.log
 oslo.messaging==12.2.0