integration: Add providers.yaml support for external providers

- `_load_providers_config()`: parses the YAML file present at the
  CORIOLIS_PROVIDERS_YAML environment variable, which describes the
  destination provider, its connection info, target environment, storage
  mappings. If CORIOLIS_PROVIDERS_YAML is unset, the default test
  provider will be used instead.
- `CORIOLIS_CONFIG_FILE`: optional `oslo.config` file path, so provider
  packages can register and read their own config options.
- `providers.yaml.sample`: sample file for declaring providers.
- `tox.ini`: pass all `CORIOLIS_*` env vars through; set `PBR_VERSION` to avoid
  pbr version-detection failures.
- Randomizes `instance_name` used for transfer tests to avoid conflict.
- Fixes `connection_info` reference in `test_minion_pools.py`.
- Machine Pool allocation, transfers, deployments may take longer for other
  providers. 600s gives sufficient headroom.

coriolis/tests/integration/README.md

@@ -98,6 +98,39 @@ sudo tox -e integration -- --no-discover coriolis.tests.integration.transfers.te
 > `sudo` is required because `tox` itself must run as root so that the
 > test process inherits root privileges.
 
+## Using an external destination provider
+
+By default, the harness uses the built-in Docker test provider for both source
+and destination. To run the integration suite against a real destination
+provider, install the provider package via `CORIOLIS_PROVIDER_PACKAGE` and
+supply provider configuration via `CORIOLIS_PROVIDERS_YAML`.
+
+### What the harness does with `providers.yaml`
+
+1. Registers the destination provider class with `oslo.config`.
+2. Creates a destination endpoint with `destination.connection_info`.
+3. Uses `destination.environment` as `destination_environment` and
+   `destination.storage_mappings` as `storage_mappings` for each transfer.
+
+### Running
+
+Set `CORIOLIS_PROVIDER_PACKAGE` to a local path or any pip-compatible specifier
+(`git+file://`, `git+https://`, etc.); tox installs it into the virtualenv
+before running the tests. Leave it unset to use only the built-in test provider.
+
+```bash
+sudo -E CORIOLIS_PROVIDER_PACKAGE=/path/to/provider \
+  CORIOLIS_PROVIDERS_YAML=./providers.yaml tox -e integration
+```
+
+Supply `CORIOLIS_CONFIG_FILE` when provider-specific configurations are required:
+
+```bash
+sudo -E CORIOLIS_PROVIDER_PACKAGE=/path/to/provider \
+  CORIOLIS_CONFIG_FILE=./provider.conf \
+  CORIOLIS_PROVIDERS_YAML=./providers.yaml tox -e integration
+```
+
 ## Test modules
 
 ### No block devices (extend `CoriolisIntegrationTestBase`)

coriolis/tests/integration/base.py

@@ -16,6 +16,7 @@ import os
 import time
 import unittest
 from unittest import mock
+import uuid
 
 from coriolisclient import client as coriolis_client
 from keystoneauth1 import session as ks_session
@@ -70,6 +71,7 @@ class CoriolisIntegrationTestBase(test_base.CoriolisBaseTestCase):
         cls._imp_platform = cls._harness.imp_provider_platform
         cls._imp_conn_info = cls._harness.imp_conn_info
         cls._imp_env_options = cls._harness.imp_env_options
+        cls._storage_mappings = cls._harness.imp_storage_mappings
 
         cls._client = cls.get_client()
 
@@ -136,15 +138,19 @@ class CoriolisIntegrationTestBase(test_base.CoriolisBaseTestCase):
             destination_environment=None, **kwargs):
         """Create a Replica transfer object and return its ID."""
 
+        destination_environment = (
+            destination_environment or self._imp_env_options
+        )
+
         transfer = self._client.transfers.create(
             origin_endpoint_id=src_id,
             destination_endpoint_id=dst_id,
             source_environment=source_environment or {},
-            destination_environment=destination_environment or {},
+            destination_environment=destination_environment,
             instances=instances,
             transfer_scenario=constants.TRANSFER_SCENARIO_REPLICA,
             network_map={},
-            storage_mappings={},
+            storage_mappings=self._storage_mappings,
             notes="integration test replica",
             skip_os_morphing=True,
             **kwargs,
@@ -193,7 +199,7 @@ class CoriolisIntegrationTestBase(test_base.CoriolisBaseTestCase):
             cls._client.minion_pools.delete(pool_id)
 
     @classmethod
-    def _wait_for_pool(cls, pool_id, terminal_statuses, timeout=180):
+    def _wait_for_pool(cls, pool_id, terminal_statuses, timeout=600):
         """Poll the DB until *pool_id* reaches one of *terminal_statuses*.
 
         :returns: minion pool ORM object.
@@ -290,7 +296,8 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
         # Create transfer replica.
         # Use basename as instance name; real VM names do not contain slashes,
         # and some providers use the name as is in resource indentifiers.
-        self._instance_name = os.path.basename(self._src_device)
+        self._instance_name = "%s-%s" % (
+            os.path.basename(self._src_device), uuid.uuid4().hex[:8])
         self._transfer = self._create_transfer(
             self._src_endpoint.id,
             self._dst_endpoint.id,
@@ -359,7 +366,7 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
             LOG.warn(
                 "Could not clean up provider dst devices. Ex: %s", ex)
 
-    def _execute_and_wait(self, transfer_id, timeout=300):
+    def _execute_and_wait(self, transfer_id, timeout=600):
         """Trigger one execution of *transfer_id* and wait for completion."""
         execution = self._client.transfer_executions.create(
             transfer_id, shutdown_instances=False)
@@ -390,7 +397,7 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
 
         self._client.transfer_executions.delete(transfer_id, execution_id)
 
-    def wait_for_execution(self, execution_id, timeout=300,
+    def wait_for_execution(self, execution_id, timeout=600,
                            desired_statuses=None):
         """Block until *execution_id* reaches a terminal state.
 
@@ -416,7 +423,7 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
             % (execution_id, desired_statuses, timeout, execution.status)
         )
 
-    def assertExecutionCompleted(self, execution_id, timeout=300):
+    def assertExecutionCompleted(self, execution_id, timeout=600):
         """Assert that *execution_id* completes successfully."""
         execution = self.wait_for_execution(execution_id, timeout=timeout)
         self.assertEqual(
@@ -437,7 +444,7 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
             ),
         )
 
-    def assertExecutionErrored(self, execution_id, timeout=300):
+    def assertExecutionErrored(self, execution_id, timeout=600):
         """Assert that *execution_id* ends in an error state."""
         execution = self.wait_for_execution(execution_id, timeout=timeout)
         self.assertIn(
@@ -489,7 +496,7 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
                     "Could not clean up deployed instance '%s': %s",
                     instance_name, ex)
 
-    def wait_for_deployment(self, deployment_id, timeout=300,
+    def wait_for_deployment(self, deployment_id, timeout=600,
                             desired_statuses=None):
         """Block until *deployment_id* reaches any terminal state.
 
@@ -513,7 +520,7 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
                deployment.last_execution_status)
         )
 
-    def assertDeploymentCompleted(self, deployment_id, timeout=300):
+    def assertDeploymentCompleted(self, deployment_id, timeout=600):
         """Assert that *deployment_id* finishes with a completed status."""
         deployment = self.wait_for_deployment(deployment_id, timeout=timeout)
         self.assertEqual(
@@ -573,17 +580,17 @@ class MinionPoolReplicaTestBase(
 
     _CREATE_MINION_POOLS = True
 
-    def _execute_and_wait(self, transfer_id, timeout=300):
+    def _execute_and_wait(self, transfer_id, timeout=600):
         super()._execute_and_wait(transfer_id, timeout=timeout)
         self.assertPoolAllocated(self._pool_id)
         self.assertMachinesAvailable(self._pool_id)
 
-    def assertExecutionCompleted(self, execution_id, timeout=300):
+    def assertExecutionCompleted(self, execution_id, timeout=600):
         super().assertExecutionCompleted(execution_id, timeout=timeout)
         self.assertPoolAllocated(self._pool_id)
         self.assertMachinesAvailable(self._pool_id)
 
-    def assertDeploymentCompleted(self, deployment_id, timeout=300):
+    def assertDeploymentCompleted(self, deployment_id, timeout=600):
         super().assertDeploymentCompleted(deployment_id, timeout=timeout)
         self.assertPoolAllocated(self._pool_id)
         self.assertMachinesAvailable(self._pool_id)

coriolis/tests/integration/harness.py

@@ -28,6 +28,8 @@ import tempfile
 from unittest import mock
 import uuid
 
+import yaml
+
 from cheroot.workers import threadpool as cheroot_threadpool
 from cheroot import wsgi as cheroot_wsgi
 from oslo_config import cfg
@@ -57,6 +59,7 @@ from coriolis.scheduler.rpc import server as scheduler_rpc_server
 from coriolis import service
 from coriolis.taskflow import runner as taskflow_runner
 from coriolis.tasks import factory as task_runners_factory
+from coriolis.tests.integration import provider_test_base
 from coriolis.tests.integration.test_provider import imp as test_provider_imp
 from coriolis.tests.integration import utils as test_utils
 from coriolis.transfer_cron.rpc import server as transfer_cron_rpc_server
@@ -78,6 +81,44 @@ _TEST_IMPORT_PROVIDER = (
 # Fixed project used for all test requests.
 _TEST_PROJECT_ID = 'integration-project'
 
+# Path to an optional YAML file that configures the destination provider,
+# its connection_info, environment, and storage mappings.
+_PROVIDERS_YAML = os.environ.get("CORIOLIS_PROVIDERS_YAML")
+
+
+def _load_providers_config():
+    """Returns the provider configurations.
+
+    If set, loads and returns the YAML file from the CORIOLIS_PROVIDERS_YAML
+    env variable.
+    If not set, returns a default configuration for the test providers.
+    """
+    providers_config = {}
+
+    if _PROVIDERS_YAML:
+        with open(_PROVIDERS_YAML) as f:
+            providers_config = yaml.safe_load(f) or {}
+
+    dest_config = providers_config.get("destination", {})
+    dest_provider_path = dest_config.get("provider") or _TEST_IMPORT_PROVIDER
+    dest_provider_cls = _get_provider(dest_provider_path)
+
+    if not issubclass(
+        dest_provider_cls, provider_test_base.BaseTestImportProvider
+    ):
+        raise TypeError(
+            "%s must subclass BaseTestImportProvider" % dest_provider_path)
+
+    return {
+        "destination": {
+            "provider": dest_provider_path,
+            "provider_cls": dest_provider_cls,
+            "connection_info": dest_config.get("connection_info"),
+            "environment": dest_config.get("environment") or {},
+            "storage_mappings": dest_config.get("storage_mappings") or {},
+        },
+    }
+
 
 def _get_provider(dotted_path):
     """Return the class at the *dotted_path*."""
@@ -276,11 +317,18 @@ class _IntegrationHarness:
         )
 
         coriolis_conf.init_common_opts()
-        cfg.CONF([], project='coriolis', version='1.0.0',
+        _config_file = os.environ.get("CORIOLIS_CONFIG_FILE")
+        _conf_args = (
+            ['--config-file', _config_file] if _config_file else []
+        )
+        cfg.CONF(_conf_args, project='coriolis', version='1.0.0',
                  default_config_files=[], default_config_dirs=[])
         cfg.CONF.set_override('messaging_transport_url', 'fake://')
+
+        providers_config = _load_providers_config()
+        imp_provider = providers_config["destination"]["provider"]
         cfg.CONF.set_override(
-            'providers', [_TEST_EXPORT_PROVIDER, _TEST_IMPORT_PROVIDER])
+            'providers', [_TEST_EXPORT_PROVIDER, imp_provider])
         db_url = ('mysql+pymysql://%(user)s:%(password)s'
                   '@localhost:13306/%(database)s') % {
             "user": self._mysql_username,
@@ -305,6 +353,7 @@ class _IntegrationHarness:
         # Policy enforcer: reset so it re-reads the new CONF (no policy file).
         policy_module.reset()
 
+        # Init exporter.
         self.exp_provider_class = _get_provider(_TEST_EXPORT_PROVIDER)
         self.exp_provider_platform = self.exp_provider_class.platform
         self.exp_conn_info = {
@@ -312,19 +361,25 @@ class _IntegrationHarness:
             "role": "source",
         }
 
-        self.imp_provider_class = _get_provider(_TEST_IMPORT_PROVIDER)
+        # Init importer.
+        imp_provider_cls = providers_config["destination"]["provider_cls"]
+        self.imp_provider_class = imp_provider_cls
         self.imp_provider_platform = self.imp_provider_class.platform
         self.imp_provider = providers_factory.get_provider(
             self.imp_provider_platform,
             constants.PROVIDER_TYPE_TRANSFER_IMPORT,
             event_handler=mock.MagicMock(),
         )
-        self.imp_conn_info = {
+        conn_info = providers_config["destination"]["connection_info"]
+        self.imp_conn_info = conn_info or {
             "pkey_path": self.ssh_key_path,
             "role": "destination",
         }
         self.imp_provider.initialize(self.imp_conn_info)
-        self.imp_env_options = {}
+        self.imp_env_options = providers_config["destination"]["environment"]
+        self.imp_storage_mappings = (
+            providers_config["destination"]["storage_mappings"]
+        )
 
         self._wsgi_server = None
         self._wsgi_server_thread = None

coriolis/tests/integration/providers.yaml.sample

@@ -0,0 +1,28 @@
+# Sample providers.yaml - built-in Docker test provider as destination.
+#
+# This sample uses the provider that ships with the test suite itself and
+# requires no external packages or credentials. It is the same provider the
+# harness uses when CORIOLIS_PROVIDERS_YAML is not set, so it is mainly useful
+# as a reference for the file format.
+#
+# To use a real destination provider, copy this file, adjust the ``provider``
+# dotted path to point at your provider's Import Provider, and fill in
+# ``connection_info``, ``environment``, and ``storage_mappings`` as required by
+# that provider.
+#
+# Run with:
+#   sudo -E CORIOLIS_PROVIDER_PACKAGE=/path/to/provider \
+#     CORIOLIS_PROVIDERS_YAML=./providers.yaml.sample tox -e integration
+
+destination:
+  # Dotted path to an Import Provider to test. Must also implement BaseTestImportProvider
+  provider: "coriolis.tests.integration.test_provider.imp.TestImportProvider"
+
+  # connection_info is passed to the destination endpoint.
+  connection_info: null
+
+  # destination_environment options forwarded to each transfer / deployment.
+  environment: {}
+
+  # Storage backend mapping (source identifier -> destination pool / datastore).
+  storage_mappings: {}

coriolis/tests/integration/test_minion_pools.py

@@ -22,10 +22,7 @@ class MinionPoolLifecycleTest(base.MinionPoolTestBase):
         self._endpoint = self._create_endpoint(
             name="pool-dst",
             endpoint_type=self._imp_platform,
-            connection_info={
-                "devices": [],
-                "pkey_path": self._harness.ssh_key_path,
-            },
+            connection_info=self._imp_conn_info,
         )
 
     def test_minion_pool_crud(self):

tox.ini

@@ -30,12 +30,19 @@ commands =
 # Must be run as root: sudo -E tox -e integration
 # Requires the scsi_debug kernel module: modinfo scsi_debug
 # Requires kernel version 5.11 or newer (scsi_debug: per_host_store=1 parameter)
-setenv = {[testenv]setenv}
+#
+# To test with an external provider, set CORIOLIS_PROVIDER_PACKAGE to a local
+# path or pip-compatible specifier (git+file://, git+https://, etc.) and run:
+#   sudo -E CORIOLIS_PROVIDER_PACKAGE=/path/to/provider tox -e integration
+setenv =
+  {[testenv]setenv}
+  PBR_VERSION = 0.0.1
 passenv =
-  CORIOLIS_TEST_SSH_KEY_PATH
+  CORIOLIS_*
 deps =
   {[testenv]deps}
   git+https://github.com/cloudbase/python-coriolisclient.git
+  {env:CORIOLIS_PROVIDER_PACKAGE:}
 commands = stestr run --slowest --concurrency=1 --test-path coriolis/tests/integration/ {posargs}
 
 [testenv:venv]