diff --git a/README.md b/README.md index 4d193165..1e480a96 100644 --- a/README.md +++ b/README.md @@ -605,6 +605,7 @@ When the restored version predates the worker, `supervisorctl` reports that `web - Taking and reverting a snapshot run as jobs, like deleting an external snapshot: the page returns at once and the job panel follows them. Deleting an internal snapshot stays immediate. - Migrating a VM runs as a job: the page returns at once, the job panel shows its progress, and a running migration can be cancelled there (the VM keeps running on its source host). The 540 s limit is gone. The API's migrate answers 202 with the job and needs a new `submission_key` per request. - Uploading an ISO is tracked as a job: the storage page's job panel shows it and can cancel it, and the page no longer reloads when it ends. The upload dialog closes once the upload starts; the browser still sends the file, so stay on the page until it ends. An upload left idle for an hour is ended and its partial file deleted. +- A running or paused VM can be cloned: libvirt copies its disks from one instant while it keeps running (file systems are frozen for a moment when the guest agent is ready). It needs libvirt 11.10 or later on the host, local file disks and no TPM; the VM cannot be changed from the panel until the copy ends, and the copy can be cancelled. Cancelling a clone of a shut-off VM has no effect: it shows as cancelling until the copy ends. A target pool that fills up during the copy can pause the source VM; a clone that ends uncertain keeps the VM locked until a superuser resolves the job. - Manual installs that keep their own `settings.py` copy the SQLite `OPTIONS` and `TEST` entries from `settings.py.template` and add `"operations"` to `INSTALLED_APPS`. - The SQLite database must be on a local file system (WAL). - Outside compose, stop the Docker container with `docker stop -t 95` so the worker can finish running jobs. diff --git a/instances/tests/livetest.py b/instances/tests/livetest.py index 41363f7d..c94b1211 100644 --- a/instances/tests/livetest.py +++ b/instances/tests/livetest.py @@ -86,13 +86,13 @@ def inventory(conn): return domains, volumes -def create_volume(conn, name, size_mib=64): - """A qcow2 volume in the test pool; returns its path.""" +def create_volume(conn, name, size_mib=64, fmt="qcow2"): + """A volume named name.fmt in the test pool; returns its path.""" assert name.startswith(PREFIX) pool = conn.storagePoolLookupByName(POOL) vol = pool.createXML( - f"{name}.qcow2{size_mib}" - "", + f"{name}.{fmt}{size_mib}" + f"", 0, ) return vol.path() @@ -101,11 +101,11 @@ def create_volume(conn, name, size_mib=64): OVMF_CODE = "/usr/share/edk2/ovmf/OVMF_CODE.fd" -def define_vm(conn, name, disk_paths, uefi=False, extra_devices=""): +def define_vm(conn, name, disk_paths, uefi=False, extra_devices="", disk_format="qcow2"): """A small q35 VM without an OS; disks are attached as vda, vdb, ...""" assert name.startswith(PREFIX) disks = "".join( - f"" + f"" f"" for i, path in enumerate(disk_paths) ) diff --git a/instances/tests/test_live_clone.py b/instances/tests/test_live_clone.py index 49265276..7c05fbe6 100644 --- a/instances/tests/test_live_clone.py +++ b/instances/tests/test_live_clone.py @@ -1,15 +1,27 @@ """Live checks for cloning. Runs only with TEST_LIBVIRT_HOST set.""" +import hashlib import os +import time import unittest from xml.etree import ElementTree import libvirt -from django.test import SimpleTestCase +from django.contrib.auth import get_user_model +from django.test import SimpleTestCase, TransactionTestCase from vrtManager.connection import connection_manager from vrtManager.instance import wvmInstance +from appsettings.settings import get_settings +from computes.models import Compute +from instances.models import Instance +from operations import jobs +from operations.models import Operation +from operations.tests import fakes +from operations.worker import Worker + from . import livetest +from .test_live_jobs import wait_for P = livetest.PREFIX @@ -115,3 +127,112 @@ class LiveCloneTestCase(SimpleTestCase): clone = self.conn.lookupByUUIDString(uuid) self.assertEqual(clone.name(), P + "clone-ok-copy") self.assertTrue({P + "clone-ok-copy-a.qcow2", P + "clone-ok-copy-b.qcow2"} <= self.pool_volumes()) + + +MIB = 1024 * 1024 + + +class LiveCloneJobTestCase(TransactionTestCase): + """The instance.clone job copies a running VM through the worker.""" + + @classmethod + def setUpClass(cls): + if not livetest.enabled(): + raise unittest.SkipTest("Set TEST_LIBVIRT_HOST to run the live libvirt tests") + cls.args = ( + os.environ["TEST_LIBVIRT_HOST"], + os.environ.get("TEST_LIBVIRT_LOGIN", ""), + os.environ.get("TEST_LIBVIRT_PASSWORD", ""), + int(os.environ.get("TEST_LIBVIRT_TYPE", 4)), + ) + cls.conn = connection_manager.get_connection(*cls.args) + livetest.cleanup(cls.conn) + livetest.ensure_pool(cls.conn) + cls.inventory = livetest.inventory(cls.conn) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + try: + livetest.remove_pool(cls.conn) + finally: + super().tearDownClass() + if livetest.inventory(cls.conn) != cls.inventory: + raise AssertionError("Live tests changed objects outside the test namespace") + + def setUp(self): + get_settings() + host, login, password, conn_type = self.args + self.compute = Compute.objects.create( + name=P + "compute", hostname=host, login=login, password=password, type=conn_type + ) + self.admin = get_user_model().objects.create_superuser("clone_live_admin", "a@example.com", "pw") + self.worker = Worker(concurrency=1, tick=0.1, heartbeat_every=1, progress_every=1, exit_on_lost_lease=False) + thread = fakes.start_worker(self.worker) + self.addCleanup(fakes.stop_worker, self.worker, thread) + + def tearDown(self): + livetest.cleanup(self.conn) + + def pool(self): + pool = self.conn.storagePoolLookupByName(livetest.POOL) + pool.refresh(0) + return pool + + def upload(self, vol, data): + stream = self.conn.newStream(0) + vol.upload(stream, 0, len(data), 0) + for offset in range(0, len(data), MIB): + stream.send(data[offset : offset + MIB]) + stream.finish() + + def sha256(self, vol): + stream = self.conn.newStream(0) + vol.download(stream, 0, 0, 0) + digest = hashlib.sha256() + while chunk := stream.recv(MIB): + digest.update(chunk) + stream.finish() + return digest.hexdigest() + + def submit(self, source, name, mac, key): + fields = {"instance": source.pk, "name": P + name, "disk-vda": P + name + ".raw", "clone-net-mac-0": mac} + return jobs.submit("instance.clone", self.admin, fields, key) + + def ended(self, op): + self.assertTrue(wait_for(lambda: Operation.objects.get(pk=op.pk).state in Operation.TERMINAL, 300)) + op.refresh_from_db() + return op + + def test_a_running_vm_is_cloned_and_a_cancelled_clone_leaves_nothing(self): + path = livetest.create_volume(self.conn, P + "live-src", fmt="raw") + self.upload(self.pool().storageVolLookupByName(P + "live-src.raw"), os.urandom(64 * MIB)) + dom = livetest.define_vm(self.conn, P + "live-src", [path], disk_format="raw") + dom.create() + xml = dom.XMLDesc(0) + source = Instance.objects.create(compute=self.compute, name=dom.name(), uuid=dom.UUIDString()) + + op = self.ended(self.submit(source, "live-copy", "52:54:00:00:32:01", "key-liveclone1")) + self.assertEqual(op.state, Operation.SUCCEEDED, op.error) + clone = self.conn.lookupByUUIDString(op.intent["uuid"]) + self.assertEqual(clone.name(), P + "live-copy") + self.assertEqual(clone.info()[0], libvirt.VIR_DOMAIN_SHUTOFF) + pool = self.pool() + self.assertEqual( + self.sha256(pool.storageVolLookupByName(P + "live-copy.raw")), + self.sha256(pool.storageVolLookupByName(P + "live-src.raw")), + ) + self.assertEqual(dom.info()[0], libvirt.VIR_DOMAIN_RUNNING) + self.assertEqual(dom.XMLDesc(0), xml) + + op = self.submit(source, "live-cancel", "52:54:00:00:32:02", "key-liveclone2") + end = time.monotonic() + 60 + while Operation.objects.get(pk=op.pk).state == Operation.QUEUED and time.monotonic() < end: + time.sleep(0.05) + jobs.cancel(op.pk, self.admin) + op = self.ended(op) + self.assertEqual(op.state, Operation.CANCELLED, op.error) + self.assertNotIn(P + "live-cancel.raw", self.pool().listVolumes()) + with self.assertRaises(libvirt.libvirtError): + self.conn.lookupByName(P + "live-cancel") + self.assertEqual(dom.info()[0], libvirt.VIR_DOMAIN_RUNNING)