diff --git a/README.md b/README.md
index 4d193165..1e480a96 100644
--- a/README.md
+++ b/README.md
@@ -605,6 +605,7 @@ When the restored version predates the worker, `supervisorctl` reports that `web
- Taking and reverting a snapshot run as jobs, like deleting an external snapshot: the page returns at once and the job panel follows them. Deleting an internal snapshot stays immediate.
- Migrating a VM runs as a job: the page returns at once, the job panel shows its progress, and a running migration can be cancelled there (the VM keeps running on its source host). The 540 s limit is gone. The API's migrate answers 202 with the job and needs a new `submission_key` per request.
- Uploading an ISO is tracked as a job: the storage page's job panel shows it and can cancel it, and the page no longer reloads when it ends. The upload dialog closes once the upload starts; the browser still sends the file, so stay on the page until it ends. An upload left idle for an hour is ended and its partial file deleted.
+- A running or paused VM can be cloned: libvirt copies its disks from one instant while it keeps running (file systems are frozen for a moment when the guest agent is ready). It needs libvirt 11.10 or later on the host, local file disks and no TPM; the VM cannot be changed from the panel until the copy ends, and the copy can be cancelled. Cancelling a clone of a shut-off VM has no effect: it shows as cancelling until the copy ends. A target pool that fills up during the copy can pause the source VM; a clone that ends uncertain keeps the VM locked until a superuser resolves the job.
- Manual installs that keep their own `settings.py` copy the SQLite `OPTIONS` and `TEST` entries from `settings.py.template` and add `"operations"` to `INSTALLED_APPS`.
- The SQLite database must be on a local file system (WAL).
- Outside compose, stop the Docker container with `docker stop -t 95` so the worker can finish running jobs.
diff --git a/instances/tests/livetest.py b/instances/tests/livetest.py
index 41363f7d..c94b1211 100644
--- a/instances/tests/livetest.py
+++ b/instances/tests/livetest.py
@@ -86,13 +86,13 @@ def inventory(conn):
return domains, volumes
-def create_volume(conn, name, size_mib=64):
- """A qcow2 volume in the test pool; returns its path."""
+def create_volume(conn, name, size_mib=64, fmt="qcow2"):
+ """A volume named name.fmt in the test pool; returns its path."""
assert name.startswith(PREFIX)
pool = conn.storagePoolLookupByName(POOL)
vol = pool.createXML(
- f"{name}.qcow2{size_mib}"
- "",
+ f"{name}.{fmt}{size_mib}"
+ f"",
0,
)
return vol.path()
@@ -101,11 +101,11 @@ def create_volume(conn, name, size_mib=64):
OVMF_CODE = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
-def define_vm(conn, name, disk_paths, uefi=False, extra_devices=""):
+def define_vm(conn, name, disk_paths, uefi=False, extra_devices="", disk_format="qcow2"):
"""A small q35 VM without an OS; disks are attached as vda, vdb, ..."""
assert name.startswith(PREFIX)
disks = "".join(
- f""
+ f""
f""
for i, path in enumerate(disk_paths)
)
diff --git a/instances/tests/test_live_clone.py b/instances/tests/test_live_clone.py
index 49265276..7c05fbe6 100644
--- a/instances/tests/test_live_clone.py
+++ b/instances/tests/test_live_clone.py
@@ -1,15 +1,27 @@
"""Live checks for cloning. Runs only with TEST_LIBVIRT_HOST set."""
+import hashlib
import os
+import time
import unittest
from xml.etree import ElementTree
import libvirt
-from django.test import SimpleTestCase
+from django.contrib.auth import get_user_model
+from django.test import SimpleTestCase, TransactionTestCase
from vrtManager.connection import connection_manager
from vrtManager.instance import wvmInstance
+from appsettings.settings import get_settings
+from computes.models import Compute
+from instances.models import Instance
+from operations import jobs
+from operations.models import Operation
+from operations.tests import fakes
+from operations.worker import Worker
+
from . import livetest
+from .test_live_jobs import wait_for
P = livetest.PREFIX
@@ -115,3 +127,112 @@ class LiveCloneTestCase(SimpleTestCase):
clone = self.conn.lookupByUUIDString(uuid)
self.assertEqual(clone.name(), P + "clone-ok-copy")
self.assertTrue({P + "clone-ok-copy-a.qcow2", P + "clone-ok-copy-b.qcow2"} <= self.pool_volumes())
+
+
+MIB = 1024 * 1024
+
+
+class LiveCloneJobTestCase(TransactionTestCase):
+ """The instance.clone job copies a running VM through the worker."""
+
+ @classmethod
+ def setUpClass(cls):
+ if not livetest.enabled():
+ raise unittest.SkipTest("Set TEST_LIBVIRT_HOST to run the live libvirt tests")
+ cls.args = (
+ os.environ["TEST_LIBVIRT_HOST"],
+ os.environ.get("TEST_LIBVIRT_LOGIN", ""),
+ os.environ.get("TEST_LIBVIRT_PASSWORD", ""),
+ int(os.environ.get("TEST_LIBVIRT_TYPE", 4)),
+ )
+ cls.conn = connection_manager.get_connection(*cls.args)
+ livetest.cleanup(cls.conn)
+ livetest.ensure_pool(cls.conn)
+ cls.inventory = livetest.inventory(cls.conn)
+ super().setUpClass()
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+ livetest.remove_pool(cls.conn)
+ finally:
+ super().tearDownClass()
+ if livetest.inventory(cls.conn) != cls.inventory:
+ raise AssertionError("Live tests changed objects outside the test namespace")
+
+ def setUp(self):
+ get_settings()
+ host, login, password, conn_type = self.args
+ self.compute = Compute.objects.create(
+ name=P + "compute", hostname=host, login=login, password=password, type=conn_type
+ )
+ self.admin = get_user_model().objects.create_superuser("clone_live_admin", "a@example.com", "pw")
+ self.worker = Worker(concurrency=1, tick=0.1, heartbeat_every=1, progress_every=1, exit_on_lost_lease=False)
+ thread = fakes.start_worker(self.worker)
+ self.addCleanup(fakes.stop_worker, self.worker, thread)
+
+ def tearDown(self):
+ livetest.cleanup(self.conn)
+
+ def pool(self):
+ pool = self.conn.storagePoolLookupByName(livetest.POOL)
+ pool.refresh(0)
+ return pool
+
+ def upload(self, vol, data):
+ stream = self.conn.newStream(0)
+ vol.upload(stream, 0, len(data), 0)
+ for offset in range(0, len(data), MIB):
+ stream.send(data[offset : offset + MIB])
+ stream.finish()
+
+ def sha256(self, vol):
+ stream = self.conn.newStream(0)
+ vol.download(stream, 0, 0, 0)
+ digest = hashlib.sha256()
+ while chunk := stream.recv(MIB):
+ digest.update(chunk)
+ stream.finish()
+ return digest.hexdigest()
+
+ def submit(self, source, name, mac, key):
+ fields = {"instance": source.pk, "name": P + name, "disk-vda": P + name + ".raw", "clone-net-mac-0": mac}
+ return jobs.submit("instance.clone", self.admin, fields, key)
+
+ def ended(self, op):
+ self.assertTrue(wait_for(lambda: Operation.objects.get(pk=op.pk).state in Operation.TERMINAL, 300))
+ op.refresh_from_db()
+ return op
+
+ def test_a_running_vm_is_cloned_and_a_cancelled_clone_leaves_nothing(self):
+ path = livetest.create_volume(self.conn, P + "live-src", fmt="raw")
+ self.upload(self.pool().storageVolLookupByName(P + "live-src.raw"), os.urandom(64 * MIB))
+ dom = livetest.define_vm(self.conn, P + "live-src", [path], disk_format="raw")
+ dom.create()
+ xml = dom.XMLDesc(0)
+ source = Instance.objects.create(compute=self.compute, name=dom.name(), uuid=dom.UUIDString())
+
+ op = self.ended(self.submit(source, "live-copy", "52:54:00:00:32:01", "key-liveclone1"))
+ self.assertEqual(op.state, Operation.SUCCEEDED, op.error)
+ clone = self.conn.lookupByUUIDString(op.intent["uuid"])
+ self.assertEqual(clone.name(), P + "live-copy")
+ self.assertEqual(clone.info()[0], libvirt.VIR_DOMAIN_SHUTOFF)
+ pool = self.pool()
+ self.assertEqual(
+ self.sha256(pool.storageVolLookupByName(P + "live-copy.raw")),
+ self.sha256(pool.storageVolLookupByName(P + "live-src.raw")),
+ )
+ self.assertEqual(dom.info()[0], libvirt.VIR_DOMAIN_RUNNING)
+ self.assertEqual(dom.XMLDesc(0), xml)
+
+ op = self.submit(source, "live-cancel", "52:54:00:00:32:02", "key-liveclone2")
+ end = time.monotonic() + 60
+ while Operation.objects.get(pk=op.pk).state == Operation.QUEUED and time.monotonic() < end:
+ time.sleep(0.05)
+ jobs.cancel(op.pk, self.admin)
+ op = self.ended(op)
+ self.assertEqual(op.state, Operation.CANCELLED, op.error)
+ self.assertNotIn(P + "live-cancel.raw", self.pool().listVolumes())
+ with self.assertRaises(libvirt.libvirtError):
+ self.conn.lookupByName(P + "live-cancel")
+ self.assertEqual(dom.info()[0], libvirt.VIR_DOMAIN_RUNNING)