integration: Adds tests for source minion pools

New tests: source pool CRUD / allocate / deallocate,
get_source_minion_pool_options, source-pool-backed transfer.

coriolis/tests/integration/README.md

@@ -114,9 +114,12 @@ pointing at the built-in test provider.
    pre-existing VM to migrate; the harness does not create or delete it (this
    is unlike the destination side, where resources are created and torn down
    per test). Merges `source.environment` into each transfer's
-   `source_environment`.
+   `source_environment`, and uses `source.minion_pool_environment` as the
+   `environment_options` when creating a source minion pool.
 4. Uses `destination.environment` as `destination_environment` and
-   `destination.storage_mappings` as `storage_mappings` for each transfer.
+   `destination.storage_mappings` as `storage_mappings` for each transfer, and
+   `destination.minion_pool_environment` as the `environment_options` when
+   creating a destination minion pool.
 
 ### Running
 

coriolis/tests/integration/base.py

@@ -70,7 +70,8 @@ class CoriolisIntegrationTestBase(test_base.CoriolisBaseTestCase):
         cls._imp_conn_info = cls._harness.imp_conn_info
         cls._imp_env_options = cls._harness.imp_env_options
         cls._storage_mappings = cls._harness.imp_storage_mappings
-        cls._pool_env = cls._harness.imp_minion_pool_environment
+        cls._imp_pool_env = cls._harness.imp_minion_pool_environment
+        cls._exp_pool_env = cls._harness.exp_minion_pool_environment
 
         cls._client = cls.get_client()
 
@@ -168,13 +169,19 @@ class CoriolisIntegrationTestBase(test_base.CoriolisBaseTestCase):
         name="test-pool",
         skip_allocation=True,
         wait_for_allocation=False,
+        platform=constants.PROVIDER_PLATFORM_DESTINATION,
     ):
+        env_options = (
+            cls._imp_pool_env
+            if platform == constants.PROVIDER_PLATFORM_DESTINATION
+            else cls._exp_pool_env
+        )
         pool = cls._client.minion_pools.create(
             name=name,
             endpoint=endpoint_id,
-            platform=constants.PROVIDER_PLATFORM_DESTINATION,
+            platform=platform,
             os_type=constants.OS_TYPE_LINUX,
-            environment_options=cls._pool_env,
+            environment_options=env_options,
             minimum_minions=1,
             maximum_minions=1,
             minion_max_idle_time=3600,
@@ -241,6 +248,39 @@ class CoriolisIntegrationTestBase(test_base.CoriolisBaseTestCase):
             is_admin=True,
         )
 
+    def assertPoolAllocated(self, pool_id):
+        """Assert the pool is healthy and still in ALLOCATED status."""
+        ctxt = self._get_db_context()
+        pool = db_api.get_minion_pool(ctxt, pool_id)
+        self.assertIsNotNone(pool, "Pool %s not found" % pool_id)
+        self.assertEqual(
+            constants.MINION_POOL_STATUS_ALLOCATED,
+            pool.status,
+            "Pool %s is not ALLOCATED (got %s)" % (pool_id, pool.status),
+        )
+
+    def assertMachinesAvailable(self, pool_id):
+        """Assert all machines in the pool are AVAILABLE and have been used."""
+        ctxt = self._get_db_context()
+        pool = db_api.get_minion_pool(ctxt, pool_id, include_machines=True)
+        self.assertIsNotNone(pool, "Pool %s not found" % pool_id)
+        self.assertTrue(
+            pool.minion_machines,
+            "Pool %s has no minion machines" % pool_id,
+        )
+        for machine in pool.minion_machines:
+            self.assertEqual(
+                constants.MINION_MACHINE_STATUS_AVAILABLE,
+                machine.allocation_status,
+                "Machine %s in pool %s is not AVAILABLE (got %s)"
+                % (machine.id, pool_id, machine.allocation_status),
+            )
+            self.assertIsNotNone(
+                machine.last_used_at,
+                "Machine %s in pool %s has no last_used_at; "
+                "it may not have been used by the transfer" % (machine.id, pool_id),
+            )
+
     @staticmethod
     def _ignoreExc(func, ignored_exc=Exception):
         """Wrap the given function, ignoring exceptions."""
@@ -255,7 +295,8 @@ class CoriolisIntegrationTestBase(test_base.CoriolisBaseTestCase):
 
 
 class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
-    _CREATE_MINION_POOLS = False
+    _CREATE_DST_MINION_POOL = False
+    _CREATE_SRC_MINION_POOL = False
     _SRC_DEVICE_SIZE_MB = 16
 
     # Extra source_environment entries merged into the default transfer's
@@ -281,15 +322,27 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
         )
 
         # Create minion pool if needed.
-        cls._pool_id = None
-        if cls._CREATE_MINION_POOLS:
+        cls._dst_pool_id = None
+        if cls._CREATE_DST_MINION_POOL:
             pool = cls._create_pool(
                 cls._dst_endpoint.id,
-                "transfer-pool",
+                "dst-transfer-pool",
                 skip_allocation=False,
                 wait_for_allocation=True,
             )
-            cls._pool_id = pool.id
+            cls._dst_pool_id = pool.id
+
+        # Create source minion pool if needed.
+        cls._src_pool_id = None
+        if cls._CREATE_SRC_MINION_POOL:
+            pool = cls._create_pool(
+                cls._src_endpoint.id,
+                "src-transfer-pool",
+                skip_allocation=False,
+                wait_for_allocation=True,
+                platform=constants.PROVIDER_PLATFORM_SOURCE,
+            )
+            cls._src_pool_id = pool.id
 
     def setUp(self):
         super().setUp()
@@ -303,7 +356,8 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
             self._src_endpoint.id,
             self._dst_endpoint.id,
             instances=[self._instance_name],
-            destination_minion_pool_id=self._pool_id,
+            destination_minion_pool_id=self._dst_pool_id,
+            origin_minion_pool_id=self._src_pool_id,
             source_environment={
                 **extra_source_env,
                 **self._exp_env_options,
@@ -591,7 +645,28 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
         self.addCleanup(patcher.stop)
 
 
-class MinionPoolTestBase(CoriolisIntegrationTestBase):
+class SourceMinionPoolTestBase(CoriolisIntegrationTestBase):
+    """Base class for source minion pool integration tests.
+
+    Skips the entire test class when the export provider does not advertise
+    ``PROVIDER_TYPE_SOURCE_MINION_POOL`` support.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        h = harness._IntegrationHarness.get()
+        available = providers_factory.get_available_providers()
+        exp_types = available.get(h.exp_provider_platform, {}).get("types", [])
+        if constants.PROVIDER_TYPE_SOURCE_MINION_POOL not in exp_types:
+            raise unittest.SkipTest(
+                "Export provider '%s' does not support minion pools"
+                % h.exp_provider_platform
+            )
+
+        super().setUpClass()
+
+
+class DestinationMinionPoolTestBase(CoriolisIntegrationTestBase):
     """Base class for minion pool integration tests.
 
     Skips the entire test class when the import provider does not advertise
@@ -615,59 +690,50 @@ class MinionPoolTestBase(CoriolisIntegrationTestBase):
         super().setUpClass()
 
 
-class MinionPoolReplicaTestBase(MinionPoolTestBase, ReplicaIntegrationTestBase):
-    """Base class for replica integration tests using minion pools.
+class MinionPoolReplicaTestBase(
+    DestinationMinionPoolTestBase, ReplicaIntegrationTestBase
+):
+    """Base class for replica integration tests using destination minion pools.
 
     Extends the assertions to also verify that the minions in the pool have
     been used, and that the minions and the pool returns to an available state.
     """
 
-    _CREATE_MINION_POOLS = True
+    _CREATE_DST_MINION_POOL = True
 
     def _execute_and_wait(self, transfer_id, timeout=600):
         super()._execute_and_wait(transfer_id, timeout=timeout)
-        self.assertPoolAllocated(self._pool_id)
-        self.assertMachinesAvailable(self._pool_id)
+        self.assertPoolAllocated(self._dst_pool_id)
+        self.assertMachinesAvailable(self._dst_pool_id)
 
     def assertExecutionCompleted(self, execution_id, timeout=600):
         super().assertExecutionCompleted(execution_id, timeout=timeout)
-        self.assertPoolAllocated(self._pool_id)
-        self.assertMachinesAvailable(self._pool_id)
+        self.assertPoolAllocated(self._dst_pool_id)
+        self.assertMachinesAvailable(self._dst_pool_id)
 
     def assertDeploymentCompleted(self, deployment_id, timeout=600):
         super().assertDeploymentCompleted(deployment_id, timeout=timeout)
-        self.assertPoolAllocated(self._pool_id)
-        self.assertMachinesAvailable(self._pool_id)
+        self.assertPoolAllocated(self._dst_pool_id)
+        self.assertMachinesAvailable(self._dst_pool_id)
 
-    def assertPoolAllocated(self, pool_id):
-        """Assert the pool is healthy and still in ALLOCATED status."""
-        ctxt = self._get_db_context()
-        pool = db_api.get_minion_pool(ctxt, pool_id)
-        self.assertIsNotNone(pool, "Pool %s not found" % pool_id)
-        self.assertEqual(
-            constants.MINION_POOL_STATUS_ALLOCATED,
-            pool.status,
-            "Pool %s is not ALLOCATED (got %s)" % (pool_id, pool.status),
-        )
 
-    def assertMachinesAvailable(self, pool_id):
-        """Assert all machines in the pool are AVAILABLE and have been used."""
-        ctxt = self._get_db_context()
-        pool = db_api.get_minion_pool(ctxt, pool_id, include_machines=True)
-        self.assertIsNotNone(pool, "Pool %s not found" % pool_id)
-        self.assertTrue(
-            pool.minion_machines,
-            "Pool %s has no minion machines" % pool_id,
-        )
-        for machine in pool.minion_machines:
-            self.assertEqual(
-                constants.MINION_MACHINE_STATUS_AVAILABLE,
-                machine.allocation_status,
-                "Machine %s in pool %s is not AVAILABLE (got %s)"
-                % (machine.id, pool_id, machine.allocation_status),
-            )
-            self.assertIsNotNone(
-                machine.last_used_at,
-                "Machine %s in pool %s has no last_used_at; "
-                "it may not have been used by the transfer" % (machine.id, pool_id),
-            )
+class SourceMinionPoolReplicaTestBase(
+    SourceMinionPoolTestBase, ReplicaIntegrationTestBase
+):
+    """Base class for replica integration tests using source minion pools.
+
+    Extends the assertions to also verify that the minions in the pool have
+    been used, and that the minions and the pool returns to an available state.
+    """
+
+    _CREATE_SRC_MINION_POOL = True
+
+    def _execute_and_wait(self, transfer_id, timeout=600):
+        super()._execute_and_wait(transfer_id, timeout=timeout)
+        self.assertPoolAllocated(self._src_pool_id)
+        self.assertMachinesAvailable(self._src_pool_id)
+
+    def assertExecutionCompleted(self, execution_id, timeout=600):
+        super().assertExecutionCompleted(execution_id, timeout=timeout)
+        self.assertPoolAllocated(self._src_pool_id)
+        self.assertMachinesAvailable(self._src_pool_id)

coriolis/tests/integration/deployments/test_osmorphing.py

@@ -194,7 +194,7 @@ class OsMorphingDeploymentTest(OsMorphingDeploymentTestBase):
 
 
 class OsMorphingMinionPoolDeploymentTest(
-    integration_base.MinionPoolTestBase, OsMorphingDeploymentTestBase
+    integration_base.DestinationMinionPoolTestBase, OsMorphingDeploymentTestBase
 ):
     """OS morphing deployment using a minion pool for the OS morphing phase."""
 

coriolis/tests/integration/harness.py

@@ -117,6 +117,7 @@ def _load_providers_config():
             "connection_info": src_config.get("connection_info"),
             "environment": src_config.get("environment") or {},
             "instance_name": src_config.get("instance_name"),
+            "minion_pool_environment": src_config.get("minion_pool_environment") or {},
         },
         "destination": {
             "provider": dest_provider_path,
@@ -400,6 +401,9 @@ class _IntegrationHarness:
         self.exp_provider.initialize(self.exp_conn_info, providers_config["source"])
         self.exp_provider.check_prerequisites()
         self.exp_env_options = providers_config["source"]["environment"]
+        self.exp_minion_pool_environment = providers_config["source"][
+            "minion_pool_environment"
+        ]
 
         # Init importer.
         imp_provider_cls = providers_config["destination"]["provider_cls"]

coriolis/tests/integration/test_endpoints.py

@@ -10,6 +10,7 @@ Exercises endpoint-related operations via the Coriolis REST API:
 - get_storage (list and default)
 - get_source_environment_options
 - get_target_environment_options
+- get_source_minion_pool_options
 - get_destination_minion_pool_options
 - get_inventory_csv
 - endpoint_instances.list and endpoint_instances.get
@@ -98,6 +99,15 @@ class EndpointCapabilitiesTest(base.CoriolisIntegrationTestBase):
         self.assertIsInstance(options, list)
         self.assertTrue(len(options) > 0, "Expected at least one destination option")
 
+    def test_list_source_minion_pool_options(self):
+        options = self._client.endpoint_source_minion_pool_options.list(
+            self._src_endpoint.id
+        )
+        self.assertIsInstance(options, list)
+        self.assertTrue(
+            len(options) > 0, "Expected at least one source minion pool option"
+        )
+
     def test_list_destination_minion_pool_options(self):
         if not isinstance(
             self._imp_provider, provider_base.BaseDestinationMinionPoolProvider

coriolis/tests/integration/test_failure_recovery.py

@@ -166,13 +166,13 @@ class MinionPoolAllocationFailureTest(base.MinionPoolReplicaTestBase):
         mock_create.assert_called()
 
         # The pool itself stays usable.
-        self.assertPoolAllocated(self._pool_id)
+        self.assertPoolAllocated(self._dst_pool_id)
 
         # Its only machine failed both the healthcheck and the recreation
         # attempt. ending up as UNINITIALIZED. It then gets deleted, rather
         # than left dangling in a broken intermediate status.
         ctxt = self._get_db_context()
-        pool = db_api.get_minion_pool(ctxt, self._pool_id, include_machines=True)
+        pool = db_api.get_minion_pool(ctxt, self._dst_pool_id, include_machines=True)
         self.assertEqual(
             [],
             pool.minion_machines,

coriolis/tests/integration/test_minion_pools.py

@@ -21,15 +21,8 @@ from coriolis.tests.integration import base
 CONF = cfg.CONF
 
 
-class MinionPoolLifecycleTest(base.MinionPoolTestBase):
-    def setUp(self):
-        super().setUp()
-
-        self._endpoint = self._create_endpoint(
-            name="pool-dst",
-            endpoint_type=self._imp_platform,
-            connection_info=self._imp_conn_info,
-        )
+class MinionPoolLifecycleTestMixin:
+    _MINION_PLATFORM = None
 
     def _wait_for_machine_status(self, pool_id, status, timeout=120):
         """Poll the DB until the pool's single machine reaches *status*."""
@@ -54,6 +47,7 @@ class MinionPoolLifecycleTest(base.MinionPoolTestBase):
         pool = self._create_pool(self._endpoint.id)
 
         self.assertEqual("test-pool", pool.name)
+        self.assertEqual(self._MINION_PLATFORM, pool.platform)
         self.assertEqual(constants.MINION_POOL_STATUS_DEALLOCATED, pool.status)
 
         # List
@@ -115,6 +109,22 @@ class MinionPoolLifecycleTest(base.MinionPoolTestBase):
             "Pool deallocation ended in unexpected status '%s'" % final.status,
         )
 
+
+class MinionPoolLifecycleTests(
+    MinionPoolLifecycleTestMixin, base.DestinationMinionPoolTestBase
+):
+    _MINION_PLATFORM = constants.PROVIDER_PLATFORM_DESTINATION
+
+    def setUp(self):
+        super().setUp()
+
+        self._endpoint = self._create_endpoint(
+            name="pool-dst",
+            endpoint_type=self._imp_platform,
+            connection_info=self._imp_conn_info,
+        )
+        self._pool_env = self._imp_pool_env
+
     def test_cron_triggered_refresh(self):
         """Cron-scheduled refresh.
 
@@ -163,3 +173,24 @@ class MinionPoolLifecycleTest(base.MinionPoolTestBase):
             "Minion pool machine '%s' was not refreshed by the automatic "
             "cron job in time" % pool.id,
         )
+
+
+class SourceMinionPoolLifecycleTests(
+    MinionPoolLifecycleTestMixin, base.SourceMinionPoolTestBase
+):
+    _MINION_PLATFORM = constants.PROVIDER_PLATFORM_SOURCE
+
+    def setUp(self):
+        super().setUp()
+
+        self._endpoint = self._create_endpoint(
+            name="pool-src",
+            endpoint_type=self._exp_platform,
+            connection_info=self._exp_conn_info,
+        )
+        self._pool_env = self._exp_pool_env
+
+    def _create_pool(self, endpoint_id, **kwargs):
+        return super()._create_pool(
+            endpoint_id, platform=constants.PROVIDER_PLATFORM_SOURCE, **kwargs
+        )

coriolis/tests/integration/transfers/test_transfer.py

@@ -436,8 +436,8 @@ class MinionPoolTransferTest(
 
     def test_transfer(self):
         super().test_transfer()
-        self.assertPoolAllocated(self._pool_id)
-        self.assertMachinesAvailable(self._pool_id)
+        self.assertPoolAllocated(self._dst_pool_id)
+        self.assertMachinesAvailable(self._dst_pool_id)
 
 
 class ReplicaTransferViaSSHTunnelTest(base.ReplicaIntegrationTestBase):
@@ -480,3 +480,14 @@ class ReplicaTransferViaSSHTunnelTest(base.ReplicaIntegrationTestBase):
                 test_utils.devices_match(self._src_device, self._dst_device),
                 "Devices do not match after transfer via SSH tunnel",
             )
+
+
+class SourceMinionPoolTransferTest(
+    base.SourceMinionPoolReplicaTestBase, _ReplicaTransferTestsMixin
+):
+    """Transfer execution that uses a pre-allocated source minion pool."""
+
+    def test_transfer(self):
+        super().test_transfer()
+        self.assertPoolAllocated(self._src_pool_id)
+        self.assertMachinesAvailable(self._src_pool_id)