docs: a running VM can be cloned

README.md

@@ -605,6 +605,7 @@ When the restored version predates the worker, `supervisorctl` reports that `web
 - Taking and reverting a snapshot run as jobs, like deleting an external snapshot: the page returns at once and the job panel follows them. Deleting an internal snapshot stays immediate.
 - Migrating a VM runs as a job: the page returns at once, the job panel shows its progress, and a running migration can be cancelled there (the VM keeps running on its source host). The 540 s limit is gone. The API's migrate answers 202 with the job and needs a new `submission_key` per request.
 - Uploading an ISO is tracked as a job: the storage page's job panel shows it and can cancel it, and the page no longer reloads when it ends. The upload dialog closes once the upload starts; the browser still sends the file, so stay on the page until it ends. An upload left idle for an hour is ended and its partial file deleted.
+- A running or paused VM can be cloned: libvirt copies its disks from one instant while it keeps running (file systems are frozen for a moment when the guest agent is ready). It needs libvirt 11.10 or later on the host, local file disks and no TPM; the VM cannot be changed from the panel until the copy ends, and the copy can be cancelled. Cancelling a clone of a shut-off VM has no effect: it shows as cancelling until the copy ends. A target pool that fills up during the copy can pause the source VM; a clone that ends uncertain keeps the VM locked until a superuser resolves the job.
 - Manual installs that keep their own `settings.py` copy the SQLite `OPTIONS` and `TEST` entries from `settings.py.template` and add `"operations"` to `INSTALLED_APPS`.
 - The SQLite database must be on a local file system (WAL).
 - Outside compose, stop the Docker container with `docker stop -t 95` so the worker can finish running jobs.

instances/tests/livetest.py

@@ -86,13 +86,13 @@ def inventory(conn):
     return domains, volumes
 
 
-def create_volume(conn, name, size_mib=64):
-    """A qcow2 volume in the test pool; returns its path."""
+def create_volume(conn, name, size_mib=64, fmt="qcow2"):
+    """A volume named name.fmt in the test pool; returns its path."""
     assert name.startswith(PREFIX)
     pool = conn.storagePoolLookupByName(POOL)
     vol = pool.createXML(
-        f"<volume><name>{name}.qcow2</name><capacity unit='MiB'>{size_mib}</capacity>"
-        "<target><format type='qcow2'/></target></volume>",
+        f"<volume><name>{name}.{fmt}</name><capacity unit='MiB'>{size_mib}</capacity>"
+        f"<target><format type='{fmt}'/></target></volume>",
         0,
     )
     return vol.path()
@@ -101,11 +101,11 @@ def create_volume(conn, name, size_mib=64):
 OVMF_CODE = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
 
 
-def define_vm(conn, name, disk_paths, uefi=False, extra_devices=""):
+def define_vm(conn, name, disk_paths, uefi=False, extra_devices="", disk_format="qcow2"):
     """A small q35 VM without an OS; disks are attached as vda, vdb, ..."""
     assert name.startswith(PREFIX)
     disks = "".join(
-        f"<disk type='file' device='disk'><driver name='qemu' type='qcow2'/>"
+        f"<disk type='file' device='disk'><driver name='qemu' type='{disk_format}'/>"
         f"<source file='{path}'/><target dev='vd{chr(ord('a') + i)}' bus='virtio'/></disk>"
         for i, path in enumerate(disk_paths)
     )

instances/tests/test_live_clone.py

@@ -1,15 +1,27 @@
 """Live checks for cloning. Runs only with TEST_LIBVIRT_HOST set."""
 
+import hashlib
 import os
+import time
 import unittest
 from xml.etree import ElementTree
 
 import libvirt
-from django.test import SimpleTestCase
+from django.contrib.auth import get_user_model
+from django.test import SimpleTestCase, TransactionTestCase
 from vrtManager.connection import connection_manager
 from vrtManager.instance import wvmInstance
 
+from appsettings.settings import get_settings
+from computes.models import Compute
+from instances.models import Instance
+from operations import jobs
+from operations.models import Operation
+from operations.tests import fakes
+from operations.worker import Worker
+
 from . import livetest
+from .test_live_jobs import wait_for
 
 P = livetest.PREFIX
 
@@ -115,3 +127,112 @@ class LiveCloneTestCase(SimpleTestCase):
         clone = self.conn.lookupByUUIDString(uuid)
         self.assertEqual(clone.name(), P + "clone-ok-copy")
         self.assertTrue({P + "clone-ok-copy-a.qcow2", P + "clone-ok-copy-b.qcow2"} <= self.pool_volumes())
+
+
+MIB = 1024 * 1024
+
+
+class LiveCloneJobTestCase(TransactionTestCase):
+    """The instance.clone job copies a running VM through the worker."""
+
+    @classmethod
+    def setUpClass(cls):
+        if not livetest.enabled():
+            raise unittest.SkipTest("Set TEST_LIBVIRT_HOST to run the live libvirt tests")
+        cls.args = (
+            os.environ["TEST_LIBVIRT_HOST"],
+            os.environ.get("TEST_LIBVIRT_LOGIN", ""),
+            os.environ.get("TEST_LIBVIRT_PASSWORD", ""),
+            int(os.environ.get("TEST_LIBVIRT_TYPE", 4)),
+        )
+        cls.conn = connection_manager.get_connection(*cls.args)
+        livetest.cleanup(cls.conn)
+        livetest.ensure_pool(cls.conn)
+        cls.inventory = livetest.inventory(cls.conn)
+        super().setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            livetest.remove_pool(cls.conn)
+        finally:
+            super().tearDownClass()
+        if livetest.inventory(cls.conn) != cls.inventory:
+            raise AssertionError("Live tests changed objects outside the test namespace")
+
+    def setUp(self):
+        get_settings()
+        host, login, password, conn_type = self.args
+        self.compute = Compute.objects.create(
+            name=P + "compute", hostname=host, login=login, password=password, type=conn_type
+        )
+        self.admin = get_user_model().objects.create_superuser("clone_live_admin", "a@example.com", "pw")
+        self.worker = Worker(concurrency=1, tick=0.1, heartbeat_every=1, progress_every=1, exit_on_lost_lease=False)
+        thread = fakes.start_worker(self.worker)
+        self.addCleanup(fakes.stop_worker, self.worker, thread)
+
+    def tearDown(self):
+        livetest.cleanup(self.conn)
+
+    def pool(self):
+        pool = self.conn.storagePoolLookupByName(livetest.POOL)
+        pool.refresh(0)
+        return pool
+
+    def upload(self, vol, data):
+        stream = self.conn.newStream(0)
+        vol.upload(stream, 0, len(data), 0)
+        for offset in range(0, len(data), MIB):
+            stream.send(data[offset : offset + MIB])
+        stream.finish()
+
+    def sha256(self, vol):
+        stream = self.conn.newStream(0)
+        vol.download(stream, 0, 0, 0)
+        digest = hashlib.sha256()
+        while chunk := stream.recv(MIB):
+            digest.update(chunk)
+        stream.finish()
+        return digest.hexdigest()
+
+    def submit(self, source, name, mac, key):
+        fields = {"instance": source.pk, "name": P + name, "disk-vda": P + name + ".raw", "clone-net-mac-0": mac}
+        return jobs.submit("instance.clone", self.admin, fields, key)
+
+    def ended(self, op):
+        self.assertTrue(wait_for(lambda: Operation.objects.get(pk=op.pk).state in Operation.TERMINAL, 300))
+        op.refresh_from_db()
+        return op
+
+    def test_a_running_vm_is_cloned_and_a_cancelled_clone_leaves_nothing(self):
+        path = livetest.create_volume(self.conn, P + "live-src", fmt="raw")
+        self.upload(self.pool().storageVolLookupByName(P + "live-src.raw"), os.urandom(64 * MIB))
+        dom = livetest.define_vm(self.conn, P + "live-src", [path], disk_format="raw")
+        dom.create()
+        xml = dom.XMLDesc(0)
+        source = Instance.objects.create(compute=self.compute, name=dom.name(), uuid=dom.UUIDString())
+
+        op = self.ended(self.submit(source, "live-copy", "52:54:00:00:32:01", "key-liveclone1"))
+        self.assertEqual(op.state, Operation.SUCCEEDED, op.error)
+        clone = self.conn.lookupByUUIDString(op.intent["uuid"])
+        self.assertEqual(clone.name(), P + "live-copy")
+        self.assertEqual(clone.info()[0], libvirt.VIR_DOMAIN_SHUTOFF)
+        pool = self.pool()
+        self.assertEqual(
+            self.sha256(pool.storageVolLookupByName(P + "live-copy.raw")),
+            self.sha256(pool.storageVolLookupByName(P + "live-src.raw")),
+        )
+        self.assertEqual(dom.info()[0], libvirt.VIR_DOMAIN_RUNNING)
+        self.assertEqual(dom.XMLDesc(0), xml)
+
+        op = self.submit(source, "live-cancel", "52:54:00:00:32:02", "key-liveclone2")
+        end = time.monotonic() + 60
+        while Operation.objects.get(pk=op.pk).state == Operation.QUEUED and time.monotonic() < end:
+            time.sleep(0.05)
+        jobs.cancel(op.pk, self.admin)
+        op = self.ended(op)
+        self.assertEqual(op.state, Operation.CANCELLED, op.error)
+        self.assertNotIn(P + "live-cancel.raw", self.pool().listVolumes())
+        with self.assertRaises(libvirt.libvirtError):
+            self.conn.lookupByName(P + "live-cancel")
+        self.assertEqual(dom.info()[0], libvirt.VIR_DOMAIN_RUNNING)