feat(instances): move a VM's disk to another storage pool of its host

The Disks tab has a Move button for each disk: it opens a dialog to
choose another storage pool of the same host, and the move runs as a job.

- Running or paused VM: libvirt copies the disk with a block job
  (TRANSIENT_JOB, as the VM is persistent) into a file it creates in the
  pool, then pivots. The pivot names the copy in both the running and
  the next start's definition.
- Shut-off VM: the volume is copied into the pool and the copy named in
  the definition.

The source file goes then, unless a VM or volume of any compute uses it
(shared storage holds it for every host that mounts it), a compute
cannot be read, or the delete fails: the job lists such a file under
"Left behind", and the move stands.

A disk moves when:
- the VM is defined and running, paused or shut off, without snapshots,
  a saved state or a block job;
- the disk is a writable, unshared, unencrypted raw or qcow2 file of a
  storage pool, without a backing file, and the same file in both
  definitions;
- the pool keeps disk files, is another directory, and has no file of
  that name (libvirt would write into it).
Otherwise it is refused with the reason before anything changes.

The job claims the VM and the file's name in both pools. A cancel aborts
the block copy, and ends the job cancelled once the VM is back on its
file. A failed move deletes the copy it made, unless a definition names
it (a lost reply after the pivot); a file there before the copy began is
never deleted. Recovery after a stopped worker
ends a copy left mirroring, matched by its destination, and reads the
outcome from the definitions.

Superusers only, like the other disk edits.

vrtManager/connection.py

@@ -921,6 +921,16 @@ class wvmConnect(object):
                 return pool
         return None
 
+    def file_pools(self):
+        """The active pools of this host that keep disks as files, a copy's
+        destination: {name: (pool, directory)}."""
+        pools = {}
+        for pool in self.wvm.listAllStoragePools(libvirt.VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE):
+            tree = etree.fromstring(pool.XMLDesc(0))
+            if tree.get("type") in ("dir", "fs", "netfs") and tree.findtext("target/path"):
+                pools[pool.name()] = (pool, tree.findtext("target/path").rstrip("/"))
+        return pools
+
     @contextlib.contextmanager
     def _nvram_pool(self):
         """An active, refreshed pool on NVRAM_DIR: libvirt reads and deletes

vrtManager/instance.py

@@ -15,6 +15,7 @@ try:
         VIR_DOMAIN_AFFECT_CONFIG,
         VIR_DOMAIN_AFFECT_LIVE,
         VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_ARP,
+        VIR_DOMAIN_PAUSED,
         VIR_DOMAIN_RUNNING,
         VIR_DOMAIN_SHUTOFF,
         VIR_DOMAIN_START_PAUSED,
@@ -26,6 +27,8 @@ try:
         VIR_ERR_NO_STORAGE_POOL,
         VIR_ERR_NO_STORAGE_VOL,
         VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE,
+        VIR_DOMAIN_BLOCK_COPY_TRANSIENT_JOB,
+        VIR_DOMAIN_BLOCK_JOB_ABORT_PIVOT,
         VIR_DOMAIN_BLOCK_RESIZE_BYTES,
         VIR_STORAGE_VOL_FILE,
         VIR_MIGRATE_AUTO_CONVERGE,
@@ -288,16 +291,6 @@ class wvmInstances(wvmConnect):
             return "a disk with a backing file is not copied"
         return None
 
-    def file_pools(self):
-        """The active pools of this host that keep disks as files, a copy's
-        destination: {name: (pool, directory)}."""
-        pools = {}
-        for pool in self.wvm.listAllStoragePools(VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE):
-            tree = ElementTree.fromstring(pool.XMLDesc(0))
-            if tree.get("type") in ("dir", "fs", "netfs") and tree.findtext("target/path"):
-                pools[pool.name()] = (pool, tree.findtext("target/path").rstrip("/"))
-        return pools
-
     def disk_copies(self, conn, definitions, pools):
         """The copies of the disks pools names a pool here for: {target, path,
         dest, pool}, from the first of definitions (XML, the one the migration
@@ -2107,6 +2100,125 @@ class wvmInstance(wvmConnect):
                 jobs.append({"dev": dev, "cur": info["cur"], "end": info["end"]})
         return jobs
 
+    def disk_move_plan(self, dev, pool_name):
+        """Where the disk dev of the VM goes in the pool pool_name of this
+        host: {dev, path, dest, format, pool, volume, running}. Refused with
+        the reason unless the VM is persistent, running, paused or shut off,
+        without snapshots, a saved state or a block job; the disk a raw or
+        qcow2 file of a storage pool, without a backing file, not read-only,
+        shareable or encrypted, the same file in both definitions; the pool
+        one of disk files, another directory, without a file of that name."""
+        dom = self.instance
+        state = dom.info()[0]
+        if not dom.isPersistent() or state not in (VIR_DOMAIN_RUNNING, VIR_DOMAIN_PAUSED, VIR_DOMAIN_SHUTOFF):
+            raise util.OperationError("A disk moves when its VM is defined and running, paused or shut off")
+        if dom.snapshotNum(0) or dom.hasManagedSaveImage(0) or self.get_snapshot_jobs():
+            raise util.OperationError("A disk of a VM with snapshots, a saved state or a running block job does not move")
+        # the running definition (the persistent one of a shut-off VM) first
+        disks = [ElementTree.fromstring(xml).find(f"devices/disk/target[@dev='{dev}']/..")
+                 for xml in (self._XMLDesc(0), self._XMLDesc(PERSISTENT_XML))]
+        sources = [disk.find("source") if disk is not None else None for disk in disks]
+        disk, source = disks[0], sources[0]
+        if disk is None or disk.get("type") != "file" or disk.get("device") != "disk" or None in sources \
+                or len({s.get("file") for s in sources}) != 1:
+            raise util.OperationError(f"{dev} is not a disk file of the VM, the same in its running and next start's "
+                                      "definition")
+        driver = disk.find("driver")
+        fmt = driver.get("type") if driver is not None else None
+        if fmt not in ("raw", "qcow2") or disk.find("readonly") is not None or disk.find("shareable") is not None \
+                or disk.find("encryption") is not None or source.find("encryption") is not None:
+            raise util.OperationError(f"{dev} does not move: only writable raw and qcow2 disks do")
+        path = source.get("file")
+        try:
+            volume = self.wvm.storageVolLookupByPath(path)
+        except libvirtError as err:
+            if err.get_error_code() != VIR_ERR_NO_STORAGE_VOL:
+                raise
+            raise util.OperationError(f"{dev} does not move: no storage pool has {path}") from err
+        if ElementTree.fromstring(volume.XMLDesc(0)).findtext("backingStore/path"):
+            raise util.OperationError(f"{dev} does not move: it has a backing file")
+        pools = self.file_pools()
+        if pool_name not in pools:
+            raise util.OperationError(f"{pool_name} is not an active pool of disk files of this host")
+        pool, directory = pools[pool_name]
+        if directory == os.path.dirname(path):
+            raise util.OperationError(f"{dev} is in pool {pool_name} already")
+        pool.refresh(0)
+        name = os.path.basename(path)
+        if self._find_volume(pool, name) is not None:
+            raise util.OperationError(f"Pool {pool_name} already has a file {name}")
+        return {"dev": dev, "path": path, "dest": f"{directory}/{name}", "format": fmt, "pool": pool,
+                "volume": volume, "running": state != VIR_DOMAIN_SHUTOFF}
+
+    def move_disk(self, move):
+        """Moves the disk as disk_move_plan planned it. A running or paused
+        VM's is copied by a block job (TRANSIENT_JOB: the VM is persistent)
+        into a file libvirt creates; at the pivot libvirt names the copy in
+        both definitions. A shut-off VM's volume is copied into the pool and
+        the copy named in the definition. The source file stays (see
+        MoveDisk). A failed move deletes the copy it made, unless a
+        definition names it; a file there before the copy began is never
+        ours."""
+        dom, dev, dest = self.instance, move["dev"], move["dest"]
+        made = False
+        try:
+            if move["running"]:
+                xml = (f"<disk type='file'><driver type='{move['format']}'/>"
+                       f"<source file='{util.xml_escape(dest)}'/></disk>")
+                dom.blockCopy(dev, xml, None, VIR_DOMAIN_BLOCK_COPY_TRANSIENT_JOB)
+                made = True
+                while True:  # mirrored once the copy caught up: cur == end
+                    info = dom.blockJobInfo(dev, 0)
+                    if not info:
+                        raise util.OperationError(f"The copy of {dev} ended before it was complete")
+                    if info["end"] and info["cur"] == info["end"]:
+                        break
+                    time.sleep(1)
+                dom.blockJobAbort(dev, VIR_DOMAIN_BLOCK_JOB_ABORT_PIVOT)
+                move["pool"].refresh(0)  # libvirt made the file past the pool: it lists it now
+            else:
+                move["pool"].createXMLFrom(
+                    f"<volume><name>{util.xml_escape(os.path.basename(dest))}</name>"
+                    f"<capacity>{move['volume'].info()[1]}</capacity>"
+                    f"<target><format type='{move['format']}'/></target></volume>", move["volume"], 0)
+                made = True
+                tree = ElementTree.fromstring(self._XMLDesc(PERSISTENT_XML))
+                tree.find(f"devices/disk/target[@dev='{dev}']/../source").set("file", dest)
+                self._defineXML(ElementTree.tostring(tree, encoding="unicode"))
+        except libvirtError as error:
+            if made:
+                self.remove_disk_copy(move, error)
+            raise
+
+    def disk_named(self, dev):
+        """The files the running (if active) and next start's definitions name for dev."""
+        flags = [0, PERSISTENT_XML] if self.instance.isActive() else [PERSISTENT_XML]
+        files = set()
+        for flag in flags:
+            disk = ElementTree.fromstring(self.instance.XMLDesc(flag)).find(f"devices/disk/target[@dev='{dev}']/..")
+            source = disk.find("source") if disk is not None else None
+            files.add(source.get("file") if source is not None else None)
+        return files
+
+    def remove_disk_copy(self, move, error):
+        """Deletes the copy of a failed move once nothing uses it: no block
+        job runs on the disk and no definition names it. Otherwise it stays,
+        and the error says so."""
+        dev, dest = move["dev"], move["dest"]
+        try:
+            if move["running"] and self.instance.blockJobInfo(dev, 0):
+                self.instance.blockJobAbort(dev, 0)
+            named = dest in self.disk_named(dev)
+            if not named:
+                move["pool"].refresh(0)
+                copy = self._find_volume(move["pool"], os.path.basename(dest))
+                if copy is not None:
+                    copy.delete(0)
+        except libvirtError as cleanup_error:
+            raise util.OperationError(f"{error}; remove {dest} if it is there: {cleanup_error}") from error
+        if named:
+            raise util.OperationError(f"{error}; the VM may use {dest} now: check its disk {dev}") from error
+
     def _snapshot(self, name, external):
         """The named snapshot; a snapshot of the other kind is refused."""
         flag = VIR_DOMAIN_SNAPSHOT_LIST_EXTERNAL if external else VIR_DOMAIN_SNAPSHOT_LIST_INTERNAL

vrtManager/tests/test_disk_move.py

@@ -0,0 +1,156 @@
+"""A disk moves to another storage pool of its host: a running or paused
+VM's by a block copy and a pivot, a shut-off VM's by a volume copy and a
+redefinition; the source file stays. Disks that cannot move are refused
+before anything changes, and a failed move deletes the copy unless a
+definition names it."""
+
+import unittest
+from unittest import mock
+
+import libvirt
+from django.conf import settings
+
+if not settings.configured:
+    settings.configure(MAC_OUI="52:54:10")
+
+from vrtManager import util
+from vrtManager.instance import wvmInstance
+
+IMAGES = "/var/lib/libvirt/images"
+FAST = "/srv/fast"
+
+
+def error(code, message="error"):
+    err = libvirt.libvirtError(message)
+    err.err = (code, 0, message, 2, None, None, None, 0, 0)
+    return err
+
+
+def domain(path=f"{IMAGES}/vm.qcow2", fmt="qcow2", extra=""):
+    return (f"<domain><devices><disk type='file' device='disk'><driver name='qemu' type='{fmt}'/>"
+            f"<source file='{path}'/><target dev='vda'/>{extra}</disk></devices></domain>")
+
+
+class Pool:
+    def __init__(self, files=()):
+        self.files = set(files)
+        self.deleted = []
+
+    refreshes = 0
+
+    def refresh(self, flags=0):
+        self.refreshes += 1
+
+    def storageVolLookupByName(self, name):
+        if name not in self.files:
+            raise error(libvirt.VIR_ERR_NO_STORAGE_VOL)
+        volume = mock.Mock()
+        volume.delete.side_effect = lambda flags: self.deleted.append(name)
+        return volume
+
+
+def vm(state=1, xml=None, config=None, backing=None, fast=None, snapshots=0):
+    """A wvmInstance of a VM in state whose vda is in IMAGES; pool fast is on FAST."""
+    proxy = wvmInstance.__new__(wvmInstance)
+    proxy.instance = dom = mock.Mock()
+    dom.info.return_value = [state, 0, 0, 1, 0]
+    dom.isPersistent.return_value = True
+    dom.isActive.return_value = state != 5
+    dom.snapshotNum.return_value = snapshots
+    dom.hasManagedSaveImage.return_value = 0
+    dom.blockJobInfo.return_value = {}
+    xml = xml or domain()
+    dom.XMLDesc.side_effect = lambda flags: (config or xml) if flags & libvirt.VIR_DOMAIN_XML_INACTIVE else xml
+    proxy.wvm = mock.Mock()
+    volume = proxy.wvm.storageVolLookupByPath.return_value
+    volume.info.return_value = [0, 1 << 30, 1 << 20]
+    volume.XMLDesc.return_value = f"<volume><backingStore><path>{backing}</path></backingStore></volume>" \
+        if backing else "<volume/>"
+    proxy.fast = fast or Pool()
+    proxy.file_pools = lambda: {"fast": (proxy.fast, FAST), "default": (Pool(), IMAGES)}
+    proxy.get_snapshot_jobs = lambda: []
+    return proxy
+
+
+class DiskMovePlanTestCase(unittest.TestCase):
+    def test_a_disk_file_moves_into_another_pool(self):
+        move = vm().disk_move_plan("vda", "fast")
+        self.assertEqual((move["path"], move["dest"], move["format"], move["running"]),
+                         (f"{IMAGES}/vm.qcow2", f"{FAST}/vm.qcow2", "qcow2", True))
+
+    def test_refusals(self):
+        cases = (
+            (vm(state=4), "fast", "defined and running, paused or shut off"),
+            (vm(snapshots=1), "fast", "snapshots"),
+            (vm(xml=domain(extra="<readonly/>")), "fast", "only writable raw and qcow2"),
+            (vm(xml=domain(fmt="vmdk")), "fast", "only writable raw and qcow2"),
+            (vm(config=domain(path=f"{IMAGES}/next.qcow2")), "fast", "the same in its running and next start"),
+            (vm(backing=f"{IMAGES}/base.qcow2"), "fast", "backing file"),
+            (vm(), "rbd", "not an active pool of disk files"),
+            (vm(), "default", "in pool default already"),
+            (vm(fast=Pool(files={"vm.qcow2"})), "fast", "Pool fast already has a file vm.qcow2"),
+        )
+        for proxy, pool, reason in cases:
+            with self.subTest(reason=reason):
+                with self.assertRaisesRegex(util.OperationError, reason):
+                    proxy.disk_move_plan("vda", pool)
+
+
+class MoveDiskTestCase(unittest.TestCase):
+    def test_a_running_vm_is_copied_and_pivoted(self):
+        proxy = vm()
+        proxy.instance.blockJobInfo.side_effect = [{"cur": 1, "end": 2}, {"cur": 2, "end": 2}]
+        with mock.patch("vrtManager.instance.time.sleep"):
+            proxy.move_disk(proxy.disk_move_plan("vda", "fast"))
+        dev, xml, params, flags = proxy.instance.blockCopy.call_args.args
+        self.assertEqual((dev, flags), ("vda", libvirt.VIR_DOMAIN_BLOCK_COPY_TRANSIENT_JOB))
+        self.assertIn(f"<source file='{FAST}/vm.qcow2'/>", xml)
+        proxy.instance.blockJobAbort.assert_called_once_with("vda", libvirt.VIR_DOMAIN_BLOCK_JOB_ABORT_PIVOT)
+        self.assertEqual(proxy.fast.refreshes, 2)  # the plan's, and one so that the pool lists the copy
+
+    def test_a_shut_off_vm_gets_a_copy_and_a_new_definition(self):
+        proxy = vm(state=5)
+        proxy.fast.createXMLFrom = mock.Mock()
+        proxy.move_disk(proxy.disk_move_plan("vda", "fast"))
+        volume_xml, source, flags = proxy.fast.createXMLFrom.call_args.args
+        self.assertIn("<name>vm.qcow2</name><capacity>1073741824</capacity>", volume_xml)
+        self.assertIn(f'<source file="{FAST}/vm.qcow2" />', proxy.wvm.defineXML.call_args.args[0])
+        proxy.instance.blockCopy.assert_not_called()
+
+    def test_a_copy_that_ends_early_is_deleted(self):
+        proxy = vm()
+        proxy.instance.blockJobInfo.return_value = {}  # aborted by a cancel
+        move = proxy.disk_move_plan("vda", "fast")
+        proxy.fast.files.add("vm.qcow2")  # libvirt made it
+        with self.assertRaisesRegex(util.OperationError, "ended before it was complete"):
+            proxy.move_disk(move)
+        self.assertEqual(proxy.fast.deleted, ["vm.qcow2"])
+
+    def test_a_copy_a_definition_names_is_kept(self):
+        """The pivot may have happened before its reply was lost."""
+        proxy = vm()
+        proxy.instance.blockJobInfo.side_effect = [{"cur": 2, "end": 2}, {}]
+        proxy.instance.blockJobAbort.side_effect = libvirt.libvirtError("Cannot recv data")
+        move = proxy.disk_move_plan("vda", "fast")
+        proxy.instance.XMLDesc.side_effect = lambda flags: domain(path=f"{FAST}/vm.qcow2")
+        proxy.fast.files.add("vm.qcow2")
+        with self.assertRaisesRegex(util.OperationError, f"the VM may use {FAST}/vm.qcow2 now"):
+            proxy.move_disk(move)
+        self.assertEqual(proxy.fast.deleted, [])
+
+    def test_a_file_made_by_someone_else_is_never_deleted(self):
+        """It appeared after the plan, and the copy failed on it."""
+        proxy = vm(state=5)
+        move = proxy.disk_move_plan("vda", "fast")
+        proxy.fast.files.add("vm.qcow2")
+        proxy.fast.createXMLFrom = mock.Mock(side_effect=error(libvirt.VIR_ERR_STORAGE_VOL_EXIST))
+        with self.assertRaises(libvirt.libvirtError):
+            proxy.move_disk(move)
+        self.assertEqual(proxy.fast.deleted, [])
+        running = vm()
+        move = running.disk_move_plan("vda", "fast")
+        running.fast.files.add("vm.qcow2")
+        running.instance.blockCopy.side_effect = error(libvirt.VIR_ERR_OPERATION_FAILED)
+        with self.assertRaises(libvirt.libvirtError):
+            running.move_disk(move)
+        self.assertEqual(running.fast.deleted, [])