integration: Replace scsi_debug with loop devices

coriolis-replicator refuses loop devices as exportable block devices by
default. The integration tests' fake source / destination providers used
scsi_debug to work around that. scsi_debug backs every host added under a
given module load with the same dev_size_mb, so tests needing bigger devices
(OS morphing, LUKS) had to unload / reload the kernel module around them.
This essentially means that the tests have to run sequentially on the
same host.

Loop devices are backed by an individually-sized sparse file per device,
so this constraint goes away. The test export provider now starts the replicator
with -allow-loop-devices=true.

The test suite has been updated to use loop devices instead of
scsi_debug devices.

Updated the vendored replicator binary, which now accepts the
-allow-loop-devices flag.

.github/workflows/integration-tests.yml

@@ -31,11 +31,6 @@ jobs:
       run: |
         sudo apt install tox
 
-    - name: Install scsi_debug kernel module
-      shell: bash
-      run: |
-        sudo apt-get install -y linux-modules-extra-$(uname -r)
-
     - name: Build write_data resource binary
       shell: bash
       run: |

coriolis/osmorphing/osmount/base.py

@@ -492,11 +492,16 @@ class BaseLinuxOSMountTools(luks_mixin.LinuxLUKSMixin, BaseSSHOSMountTools):
         raw = self._exec_cmd("lsblk -lnao KNAME,TYPE")
         LOG.debug("All block devices: %s", raw)
 
-        # Include top-level disk devices only.
+        block_dev_types = _VOLUME_BLOCK_DEVICE_TYPES
+        if self._osmorphing_info.get("_include_loop_devices"):
+            # the test provider sets up loop devices.
+            block_dev_types |= {"loop"}
+
+        # Include top-level disk (and, if opted in, loop) devices only.
         volume_devs = []
         for line in raw.splitlines():
             parts = line.split()
-            if len(parts) >= 2 and parts[1] in _VOLUME_BLOCK_DEVICE_TYPES:
+            if len(parts) >= 2 and parts[1] in block_dev_types:
                 volume_devs.append("/dev/%s" % parts[0])
 
         LOG.debug("Ignoring block devices: %s", self._ignore_devices)

coriolis/providers/replicator.py

@@ -370,6 +370,7 @@ class Replicator(object):
         watch_devices=True,
         chunk_size=10485760,
         use_tunnel=False,
+        _allow_loop_devices=False,
     ):
         self._event_manager = event_manager
         self._repl_state = replica_state
@@ -386,6 +387,7 @@ class Replicator(object):
         self._hash_method = hash_method
         self._ignore_mounted = ignore_mounted
         self._chunk_size = chunk_size
+        self._allow_loop_devices = _allow_loop_devices
         self._ssh = self._setup_ssh()
         self._credentials = None
         self._cli = None
@@ -779,6 +781,7 @@ class Replicator(object):
             "-listen-port=%(listen_port)s "
             "-chunk-size=%(chunk_size)s "
             "-watch-devices=%(watch_devs)s "
+            "-allow-loop-devices=%(allow_loop_devices)s "
             "-state-file=%(state_file)s "
             "-ca-cert=%(ca_cert)s -cert=%(srv_cert)s "
             "-key=%(srv_key)s"
@@ -787,6 +790,7 @@ class Replicator(object):
                 "hash_method": self._hash_method,
                 "ignore_mounted": json.dumps(self._ignore_mounted),
                 "watch_devs": json.dumps(self._watch_devices),
+                "allow_loop_devices": json.dumps(self._allow_loop_devices),
                 "listen_port": str(port),
                 "state_file": state_file,
                 "chunk_size": self._chunk_size,

coriolis/tests/integration/README.md

@@ -26,7 +26,8 @@ The test harness (`harness.py`) performs a one-time setup per process:
    provider.
 
 Teardown (registered with `atexit`) stops all services, removes the Docker
-container, removes the working directory, and unloads `scsi_debug`.
+container, removes the working directory, and detaches any leftover loop
+devices.
 
 ## Prerequisites
 
@@ -34,17 +35,14 @@ container, removes the working directory, and unloads `scsi_debug`.
 
 | Package | Why |
 |---------|-----|
-| `scsi_debug` kernel module | Virtual block devices used as source / destination storage |
-| `lsblk`, `udevadm` | Device discovery after hot-adding a `scsi_debug` host |
+| `losetup`, `truncate` | Sparse-file-backed loop devices used as source / destination storage |
 | `dd`, `sync`, `cmp` | Test-pattern writes and device comparison |
-| `modprobe` | Loading and unloading `scsi_debug` |
 | `docker` | MariaDB database container; data-minion container image |
 | `ssh-keygen` | Generates the ephemeral SSH key pair used by the test provider |
 
 On Ubuntu / Debian:
 ```bash
-sudo apt-get install util-linux kmod
-# scsi_debug ships with the standard kernel; no extra package is needed
+sudo apt-get install util-linux coreutils
 ```
 
 ### Docker image - data-minion
@@ -74,9 +72,7 @@ Key packages used by the harness:
 ### Root access
 
 The tests must run as root because:
-- `modprobe` requires root to load/unload `scsi_debug`.
-- Writing to the `scsi_debug` sysfs add-host knob (`/sys/bus/pseudo/…`)
-  requires root.
+- `losetup` requires root to attach / detach loop devices.
 - Raw block-device reads/writes (`dd`, `cmp`) require root.
 
 Tests that extend `CoriolisIntegrationTestBase` call `os.geteuid()` in
@@ -162,7 +158,7 @@ sudo -E CORIOLIS_PROVIDER_PACKAGE=/path/to/provider \
 | Class | Module | Use when |
 |-------|--------|----------|
 | `CoriolisIntegrationTestBase` | `base.py` | API-level tests; no block devices needed. |
-| `ReplicaIntegrationTestBase` | `base.py` | Tests that exercise the transfer / deployment pipeline with real disk I/O via `scsi_debug`. Requires the `coriolis-data-minion:test` Docker image. |
+| `ReplicaIntegrationTestBase` | `base.py` | Tests that exercise the transfer / deployment pipeline with real disk I/O via loop devices. Requires the `coriolis-data-minion:test` Docker image. |
 | `MinionPoolTestBase` | `base.py` | Like `CoriolisIntegrationTestBase`; skips when the import provider does not advertise minion-pool support. |
 | `MinionPoolReplicaTestBase` | `base.py` | Like `ReplicaIntegrationTestBase` with a pre-allocated minion pool; also asserts the pool and its machines return to a healthy state after each execution. |
 
@@ -180,7 +176,7 @@ sudo -E CORIOLIS_PROVIDER_PACKAGE=/path/to/provider \
 integration/
 ├── base.py                     # base test classes
 ├── harness.py                  # _IntegrationHarness singleton
-├── utils.py                    # scsi_debug helpers, device I/O, OS image utilities
+├── utils.py                    # loop device helpers, device I/O, OS image utilities
 ├── test_smoke.py
 ├── test_endpoints.py
 ├── test_failure_recovery.py
@@ -198,7 +194,7 @@ integration/
 │   ├── test_providers.py
 │   ├── test_region.py
 │   └── test_service.py
-├── test_provider/              # built-in fake cloud provider (scsi_debug backed)
+├── test_provider/              # built-in fake cloud provider
 │   ├── __init__.py
 │   ├── exp.py                  # Export provider
 │   ├── imp.py                  # Import provider

coriolis/tests/integration/base.py

@@ -256,7 +256,7 @@ class CoriolisIntegrationTestBase(test_base.CoriolisBaseTestCase):
 
 class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
     _CREATE_MINION_POOLS = False
-    _SCSI_DEBUG_SIZE_MB = 16
+    _SRC_DEVICE_SIZE_MB = 16
 
     # Extra source_environment entries merged into the default transfer's
     # source_environment.
@@ -293,20 +293,16 @@ class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase):
             )
             cls._pool_id = pool.id
 
-        # (re)init the scsi_debug module.
-        test_utils.destroy_scsi_debug()
-        test_utils.init_scsi_debug(size_mb=cls._SCSI_DEBUG_SIZE_MB)
-
     def setUp(self):
         super().setUp()
 
-        self._src_device = test_utils.add_scsi_debug_device()
-        self.addCleanup(test_utils.remove_scsi_debug_device)
+        self._src_device = test_utils.create_loop_device(
+            self._SRC_DEVICE_SIZE_MB * 1024 * 1024
+        )
+        self.addCleanup(test_utils.remove_loop_device, self._src_device)
 
         # Write a test pattern on the src device.
         # Incremental transfer tests update the second chunk (offset=4096).
-        # We need to reset any residual data left in the scsi_debug backing
-        # store from a previous test run.
         test_utils.write_test_pattern(self._src_device, 8192)
 
         # Create transfer replica.

coriolis/tests/integration/deployments/test_deployment.py

@@ -10,7 +10,7 @@ replica and asserts that the deployment execution reaches COMPLETED.
 Covers deployments.list(), get(), list(detail=True), clone_disks=False,
 deployments.cancel().
 
-Must be run as root; requires the scsi_debug kernel module.
+Must be run as root; requires losetup support (loop devices).
 """
 
 from coriolis import constants

coriolis/tests/integration/deployments/test_luks_osmorphing.py

@@ -13,7 +13,7 @@ it. OS families tested:
   update-initramfs.
 - Rocky Linux 9 (dracut-based): initramfs is regenerated via dracut.
 
-Must be run as root; requires the scsi_debug kernel module and cryptsetup.
+Must be run as root; requires losetup support (loop devices) and cryptsetup.
 """
 
 import os
@@ -47,7 +47,7 @@ class _LUKSOSMorphingMixin:
 
     # Extra space for initramfs-tools and cryptsetup-initramfs packages that
     # the LUKS morphing tools install on top of the base OS image.
-    _SCSI_DEBUG_SIZE_MB = 512
+    _SRC_DEVICE_SIZE_MB = 512
     _CONTAINER_IMAGE = "ubuntu:24.04"
 
     @classmethod
@@ -218,7 +218,7 @@ class LUKSRockyLinuxOSMorphingDeploymentTest(
 
     # kernel-core (~150 MB installed) needs extra room on top of the base
     # container image and the other morphing packages.
-    _SCSI_DEBUG_SIZE_MB = 777
+    _SRC_DEVICE_SIZE_MB = 777
     _CONTAINER_IMAGE = "rockylinux:9"
 
     def _assert_firstboot_setup(self):

coriolis/tests/integration/deployments/test_osmorphing.py

@@ -22,7 +22,7 @@ from coriolis.tests.integration import osmorphing_utils
 class OsMorphingDeploymentTestBase(integration_base.ReplicaIntegrationTestBase):
     # NOTE(claudiub): Size must be high enough to contain the tested OS and
     # any new packages to be added during OS morphing.
-    _SCSI_DEBUG_SIZE_MB = 256
+    _SRC_DEVICE_SIZE_MB = 256
 
     @classmethod
     def setUpClass(cls):

coriolis/tests/integration/harness.py

@@ -14,7 +14,7 @@ fake:// oslo.messaging transport is in-memory and process-local; subprocess
 tasks would initialise their own isolated transport with no conductor listener,
 causing every event-handler RPC call from the task to block indefinitely.
 
-Must be run as root (scsi_debug block device setup requires it).
+Must be run as root (loop device setup requires it).
 """
 
 import atexit
@@ -593,9 +593,9 @@ class _IntegrationHarness:
 
         shutil.rmtree(self.workdir, True)
         try:
-            test_utils.destroy_scsi_debug()
+            test_utils.destroy_leaked_loop_devices()
         except Exception:
-            LOG.exception("Failed to destroy the scsi_debug device.")
+            LOG.exception("Failed to destroy leaked loop devices.")
 
     def uses_core_test_import_provider(self):
         """Returns True when the test import provider is being used."""

coriolis/tests/integration/test_failure_recovery.py

@@ -16,7 +16,7 @@ Tests two symmetrical error-path scenarios:
   destination cleanup (DELETE_TRANSFER_TARGET_RESOURCES) must run,
   target_resources zeroed out.
 
-Must be run as root; requires the scsi_debug kernel module.
+Must be run as root; requires losetup support (loop devices).
 """
 
 import time

coriolis/tests/integration/test_provider/exp.py

@@ -51,7 +51,7 @@ class TestExportProvider(
     BaseReplicaExportProvider,
     BaseReplicaExportValidationProvider,
 ):
-    """Source-side provider backed by a local `scsi_debug` block device.
+    """Source-side provider backed by a local loop device.
 
     ``connection_info`` (the source endpoint's connection info) has the form::
 
@@ -97,6 +97,7 @@ class TestExportProvider(
             volumes_info,
             repl_state,
             use_tunnel=conn_info.get("use_tunnel", False),
+            _allow_loop_devices=True,
         )
 
     # BaseProvider / BaseEndpointProvider
@@ -358,7 +359,7 @@ class TestExportProvider(
     def delete_replica_source_snapshots(
         self, ctxt, connection_info, source_environment, volumes_info
     ):
-        # scsi_debug devices have no snapshots.
+        # not implemented for loop devices.
         return volumes_info
 
     def shutdown_instance(

coriolis/tests/integration/test_provider/imp.py

@@ -57,7 +57,7 @@ class TestImportProvider(
     BaseDestinationMinionPoolProvider,
     provider_test_base.BaseTestImportProvider,
 ):
-    """Destination-side provider backed by a local `scsi_debug` block device.
+    """Destination-side provider backed by a local loop device.
 
     ``connection_info`` (the destination endpoint's connection info) has the
     form::
@@ -206,7 +206,7 @@ class TestImportProvider(
             result.append(
                 {
                     "disk_id": disk["id"],
-                    "volume_dev": test_utils.add_scsi_debug_device(),
+                    "volume_dev": test_utils.create_loop_device(disk["size_bytes"]),
                 }
             )
 
@@ -307,13 +307,13 @@ class TestImportProvider(
         for vol in volumes_info:
             device = vol.get('volume_dev')
             if device and os.path.exists(device):
-                test_utils.remove_scsi_debug_device()
+                test_utils.remove_loop_device(device)
         return volumes_info
 
     def create_replica_disk_snapshots(
         self, ctxt, connection_info, target_environment, volumes_info
     ):
-        # scsi_debug has no snapshot support.
+        # not implemented for loop devices.
         return volumes_info
 
     def delete_replica_target_disk_snapshots(
@@ -423,6 +423,7 @@ class TestImportProvider(
             "osmorphing_info": {
                 "os_type": instance_deployment_info.get("os_type", "linux"),
                 "ignore_devices": ignore_devices,
+                "_include_loop_devices": True,
                 constants.ENCRYPTED_DISKS_PASS: passphrase,
             },
         }
@@ -608,5 +609,6 @@ class TestImportProvider(
             "osmorphing_info": {
                 "os_type": instance_deployment_info.get("os_type", "linux"),
                 "ignore_devices": ignore_devices,
+                "_include_loop_devices": True,
             }
         }

coriolis/tests/integration/test_smoke.py

@@ -6,7 +6,7 @@ Smoke tests for the integration harness itself.
 
 Verifies that the in-process service stack (API, conductor, scheduler, worker)
 initialises correctly and that basic resource CRUD works end-to-end, without
-any cloud provider, block device, or scsi_debug involvement.
+any cloud provider or block device involvement.
 """
 
 from coriolis.tests.integration import base

coriolis/tests/integration/transfers/test_transfer.py

@@ -237,12 +237,16 @@ class ClusteredTransferIntegrationTest(base.ReplicaIntegrationTestBase):
         super().setUp()
 
         self._own_device_a = self._src_device
-        self._own_device_b = test_utils.add_scsi_debug_device()
-        self.addCleanup(test_utils.remove_scsi_debug_device)
+        self._own_device_b = test_utils.create_loop_device(
+            self._SRC_DEVICE_SIZE_MB * 1024 * 1024
+        )
+        self.addCleanup(test_utils.remove_loop_device, self._own_device_b)
         test_utils.write_test_pattern(self._own_device_b, 8192)
 
-        self._shared_device = test_utils.add_scsi_debug_device()
-        self.addCleanup(test_utils.remove_scsi_debug_device)
+        self._shared_device = test_utils.create_loop_device(
+            self._SRC_DEVICE_SIZE_MB * 1024 * 1024
+        )
+        self.addCleanup(test_utils.remove_loop_device, self._shared_device)
         test_utils.write_test_pattern(self._shared_device, 4096)
 
         self._instance_a = "%s-%s-clustered" % (

coriolis/tests/integration/utils.py

@@ -19,97 +19,59 @@ from coriolis import utils as coriolis_utils
 
 LOG = logging.getLogger(__name__)
 
-_SETTLE_TIMEOUT = 15
-_POLL_INTERVAL = 1
-
-# Sysfs knob for adding / removing scsi_debug hosts. Writing "1" adds a new
-# host with its own independent backing store (requires per_host_store=1);
-# writing "-1" removes the most-recently added host (LIFO).
-_SCSI_DEBUG_ADD_HOST = "/sys/bus/pseudo/drivers/scsi_debug/add_host"
-
 DATA_MINION_IMAGE = "coriolis-data-minion:test"
 
-
-def get_host_disk_devices() -> set:
-    """Return the /dev paths of disk-type block devices visible on the host."""
-    disk_names = _lsblk_disk_names()
-    return {"/dev/" + disk_name for disk_name in disk_names}
+# device_path: backing_sparse_file, for devices created by create_loop_device().
+_loop_backing_files = {}
 
 
-def _lsblk_disk_names() -> set:
-    """Return the set of disk-type block device names visible to lsblk."""
+def get_host_disk_devices() -> set:
+    """Return /dev paths of disk / loop block devices visible on the host."""
     result = _run(["lsblk", "-Jb", "-o", "NAME,TYPE"], check=False)
     if result.returncode != 0:
         return set()
 
     data = json.loads(result.stdout)
-    return {d["name"] for d in data.get("blockdevices", []) if d["type"] == "disk"}
-
-
-def _poll_for_new_disks(before, count, timeout=_SETTLE_TIMEOUT):
-    """Block until *count* new disk names appear beyond *before*.
-
-    :returns: sorted list of new names.
-    :raises: ``AssertionError`` on timeout.
-    """
-    deadline = time.monotonic() + timeout
-    while time.monotonic() < deadline:
-        subprocess.call(["udevadm", "settle"])
-        new = sorted(_lsblk_disk_names() - before)
-        if len(new) >= count:
-            return new[:count]
-        time.sleep(_POLL_INTERVAL)
-    raise AssertionError(
-        "Only %d new disk(s) appeared within %ds (expected %d)"
-        % (len(sorted(_lsblk_disk_names() - before)), timeout, count)
-    )
+    return {
+        "/dev/%s" % d["name"]
+        for d in data.get("blockdevices", [])
+        if d["type"] in ("disk", "loop")
+    }
 
 
-def init_scsi_debug(size_mb=16):
-    """Load scsi_debug with per_host_store=1 and size_mb per device.
+def create_loop_device(size_bytes) -> str:
+    """Create a *size_bytes* sparse file and attach it as a loop device.
 
-    Call ``destroy_scsi_debug`` first if the module is already loaded with a
-    different size. With ``per_host_store=1`` every host added via the sysfs
-    knob gets its own independent backing store, so devices never share
-    storage.
+    :returns: the /dev/loopN path.
     """
-    _run(
-        [
-            "modprobe",
-            "scsi_debug",
-            "per_host_store=1",
-            "num_tgts=1",
-            f"dev_size_mb={size_mb}",
-        ]
-    )
-
+    fd, backing_file = tempfile.mkstemp(prefix="coriolis-loopdev-")
+    os.close(fd)
+    _run(["truncate", "-s", str(size_bytes), backing_file])
 
-def destroy_scsi_debug():
-    """Unload the scsi_debug module."""
-    _run(["modprobe", "-r", "scsi_debug"])
+    result = _run(["losetup", "--find", "--show", backing_file])
+    device_path = result.stdout.decode().strip()
+    _loop_backing_files[device_path] = backing_file
+    LOG.info("loop device created: %s (%d bytes)", device_path, size_bytes)
 
+    return device_path
 
-def add_scsi_debug_device() -> str:
-    """Add one scsi_debug host and return its /dev/sdX path.
 
-    Each call creates an independent backing store (per_host_store=1), so
-    writing to one device is never visible through another.
-    """
-    before = _lsblk_disk_names()
-    with open(_SCSI_DEBUG_ADD_HOST, "w") as fh:
-        fh.write("1\n")
+def remove_loop_device(device_path):
+    """Detach *device_path* and remove its backing file."""
+    _run(["losetup", "-d", device_path], check=False)
 
-    new = _poll_for_new_disks(before, count=1)
-    path = os.path.join("/dev", new[0])
-    LOG.info("scsi_debug device added: %s", path)
-
-    return path
+    backing_file = _loop_backing_files.pop(device_path, None)
+    if backing_file:
+        try:
+            os.unlink(backing_file)
+        except OSError:
+            pass
 
 
-def remove_scsi_debug_device():
-    """Remove the most-recently added scsi_debug host."""
-    with open(_SCSI_DEBUG_ADD_HOST, "w") as fh:
-        fh.write("-1\n")
+def destroy_leaked_loop_devices():
+    """Detach and remove any loop devices left over from a previous run."""
+    for device_path in list(_loop_backing_files):
+        remove_loop_device(device_path)
 
 
 def write_test_pattern(device_path, chunk_size=4096):

coriolis/tests/osmorphing/osmount/test_base.py

@@ -846,10 +846,30 @@ class BaseLinuxOSMountToolsTestCase(test_base.CoriolisBaseTestCase):
         result = self.base_os_mount_tools._get_volume_block_devices()
 
         mock_exec_cmd.assert_called_once_with("lsblk -lnao KNAME,TYPE")
+        # loop devices are never considered unless the provider opts in via
+        # the '_include_loop_devices' osmorphing_info flag.
         expected_result = ["/dev/sda", "/dev/sdb"]
 
         self.assertEqual(result, expected_result)
 
+    @mock.patch.object(base.utils, 'test_ssh_path', return_value=True)
+    @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd')
+    def test__get_volume_block_devices_include_loop_devices(
+        self, mock_exec_cmd, _mock_test_path
+    ):
+        lsblk_output = "sda  disk\nsdb  disk\nsdb1 part\nloop0 loop\nloop1 loop\n"
+        mock_exec_cmd.return_value = lsblk_output
+
+        self.base_os_mount_tools._ignore_devices = ["/dev/sda", "/dev/loop1"]
+        self.base_os_mount_tools._osmorphing_info = {"_include_loop_devices": True}
+
+        result = self.base_os_mount_tools._get_volume_block_devices()
+
+        mock_exec_cmd.assert_called_once_with("lsblk -lnao KNAME,TYPE")
+        expected_result = ["/dev/sdb", "/dev/loop0"]
+
+        self.assertEqual(result, expected_result)
+
     @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd')
     @mock.patch.object(base.utils, 'list_ssh_dir')
     def test__find_dev_with_contents(self, mock_list_ssh_dir, mock_exec_cmd):

tox.ini

@@ -48,8 +48,7 @@ commands =
 
 [testenv:integration]
 # Must be run as root: sudo -E tox -e integration
-# Requires the scsi_debug kernel module: modinfo scsi_debug
-# Requires kernel version 5.11 or newer (scsi_debug: per_host_store=1 parameter)
+# Requires losetup support (loop devices).
 #
 # To test with an external provider, set CORIOLIS_PROVIDER_PACKAGE to a local
 # path or pip-compatible specifier (git+file://, git+https://, etc.) and run: