mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-08 19:26:48 +00:00
043c24a208
A VM that changes its memory faster than the network carries it never finishes a live migration without auto converge: measured with a guest rewriting about 1.2 GB/s over an 8.5 Gbit/s link. The migration then ran until gunicorn ended the request after 600 s. libvirt cancelled it when the request went away and the VM stayed on the source, but the user got an error page and nothing was logged. The app now cancels a migration that has not finished after 540 s (MIGRATION_TIME_LIMIT, below the 600 s of gunicorn.conf.py; a test keeps it below). The failure is reported and logged like any other: the migration was cancelled, the VM stays on the source host, and a VM like that needs auto converge. A timer that fires as the migration ends finishes before the compute locks are released, and an error the cancel did not cause keeps its own message.
465 lines
20 KiB
Python
465 lines
20 KiB
Python
"""Migration: the source definition never stays behind; a shut-off VM's
|
|
definition is moved by the app, not by libvirt's offline migration."""
|
|
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import libvirt
|
|
from django.conf import settings
|
|
|
|
if not settings.configured:
|
|
settings.configure(MAC_OUI="52:54:10")
|
|
|
|
from vrtManager import util
|
|
from vrtManager.instance import wvmInstances
|
|
|
|
|
|
def no_domain():
|
|
error = libvirt.libvirtError("Domain not found")
|
|
error.err = (libvirt.VIR_ERR_NO_DOMAIN,)
|
|
return error
|
|
|
|
|
|
class FakeDomain:
|
|
def __init__(self, xml="<domain/>", snapshots=0, saved=False, undefine_fails=False, undefine_lost=False,
|
|
started=False):
|
|
self.calls = []
|
|
self.xml = xml
|
|
self.snapshots = snapshots
|
|
self.saved = saved
|
|
self.undefine_fails = undefine_fails
|
|
self.undefine_lost = undefine_lost # undefined, but the reply is lost
|
|
self.started = started # started by another client during the move
|
|
self.undefined = None
|
|
self.redefined = []
|
|
self.restored_autostart = None
|
|
|
|
def migrate(self, dconn, flags, dname, uri, bandwidth):
|
|
self.calls.append((flags, uri))
|
|
|
|
def name(self):
|
|
return "vm"
|
|
|
|
def UUIDString(self):
|
|
return "u-1"
|
|
|
|
inactive_xml = None # the persistent definition, when it differs
|
|
|
|
def XMLDesc(self, flags):
|
|
if flags & libvirt.VIR_DOMAIN_XML_INACTIVE and self.inactive_xml:
|
|
return self.inactive_xml
|
|
return self.xml
|
|
|
|
def snapshotNum(self, flags=0):
|
|
return self.snapshots
|
|
|
|
def hasManagedSaveImage(self, flags=0):
|
|
return self.saved
|
|
|
|
def undefineFlags(self, flags):
|
|
if self.undefine_fails:
|
|
raise libvirt.libvirtError("undefine failed")
|
|
self.undefined = flags
|
|
if self.undefine_lost:
|
|
raise libvirt.libvirtError("Cannot recv data")
|
|
|
|
def isPersistent(self):
|
|
return self.undefined is None or bool(self.redefined)
|
|
|
|
def isActive(self):
|
|
if not self.started and self.undefined is not None:
|
|
raise no_domain() # libvirt: an undefined shut-off VM is gone
|
|
return self.started
|
|
|
|
def autostart(self):
|
|
return 1
|
|
|
|
def setAutostart(self, value):
|
|
self.restored_autostart = value
|
|
|
|
def connect(self):
|
|
return self
|
|
|
|
def defineXML(self, xml):
|
|
self.redefined.append(xml)
|
|
return self
|
|
|
|
|
|
def no_volume():
|
|
error = libvirt.libvirtError("Storage volume not found")
|
|
error.err = (libvirt.VIR_ERR_NO_STORAGE_VOL,)
|
|
return error
|
|
|
|
|
|
class FakeVolume:
|
|
def __init__(self, backing):
|
|
self.backing = backing
|
|
|
|
def XMLDesc(self, flags=0):
|
|
if self.backing is None:
|
|
return "<volume><target><path>x</path></target></volume>"
|
|
return f"<volume><backingStore><path>{self.backing}</path></backingStore></volume>"
|
|
|
|
|
|
class FakePool:
|
|
def __init__(self, host, active=True):
|
|
self.host = host
|
|
self.active = active
|
|
|
|
def isActive(self):
|
|
return self.active
|
|
|
|
def refresh(self, flags=0):
|
|
self.host.refreshes += 1
|
|
self.host.volumes |= self.host.unlisted
|
|
self.host.unlisted = set()
|
|
|
|
def storageVolLookupByName(self, name):
|
|
return self.host.storageVolLookupByPath(name)
|
|
|
|
|
|
class FakeHost:
|
|
"""The destination's libvirt connection. volumes: the paths its pools
|
|
list; unlisted: files its pools show after a refresh."""
|
|
|
|
def __init__(self, existing=(), volumes=(), unlisted=(), backing=None, inactive_pools=()):
|
|
self.existing = set(existing)
|
|
self.defined = []
|
|
self.new = []
|
|
self.volumes = set(volumes)
|
|
self.unlisted = set(unlisted)
|
|
self.backing = backing or {} # path: the path of its backing file
|
|
self.inactive_pools = set(inactive_pools)
|
|
self.refreshes = 0
|
|
|
|
def storageVolLookupByPath(self, path):
|
|
if path not in self.volumes:
|
|
raise no_volume()
|
|
return FakeVolume(self.backing.get(path))
|
|
|
|
def storagePoolLookupByName(self, name):
|
|
return FakePool(self, active=name not in self.inactive_pools)
|
|
|
|
def listAllStoragePools(self, flags=0):
|
|
return [FakePool(self)]
|
|
|
|
def lookup(self, key):
|
|
if key in self.existing:
|
|
return FakeDomain()
|
|
raise no_domain()
|
|
|
|
lookupByName = lookupByUUIDString = lookup
|
|
|
|
def defineXML(self, xml):
|
|
self.defined.append(xml)
|
|
self.new.append(FakeDomain())
|
|
return self.new[-1]
|
|
|
|
|
|
class FakeSource:
|
|
def __init__(self, state, **domain):
|
|
self.state = state
|
|
self.instance = FakeDomain(**domain)
|
|
|
|
reads = 0
|
|
|
|
def get_status(self):
|
|
self.reads += 1
|
|
return self.state
|
|
|
|
def get_arch(self):
|
|
return "x86_64"
|
|
|
|
def get_dom_emulator(self):
|
|
return "/usr/bin/qemu-system-x86_64"
|
|
|
|
|
|
def destination(host=None):
|
|
dest = wvmInstances.__new__(wvmInstances)
|
|
dest.wvm = host or FakeHost()
|
|
dest.get_emulator = lambda arch: "/usr/bin/qemu-system-x86_64"
|
|
return dest
|
|
|
|
|
|
class MigrationFlagsTestCase(unittest.TestCase):
|
|
def test_the_source_definition_is_always_removed(self):
|
|
for state, live, offline in ((1, True, False), (3, False, False)):
|
|
with self.subTest(state=state):
|
|
source = FakeSource(state)
|
|
destination().moveto(source, "vm", live=live, unsafe=False, offline=offline)
|
|
flags = source.instance.calls[0][0]
|
|
self.assertTrue(flags & libvirt.VIR_MIGRATE_UNDEFINE_SOURCE)
|
|
self.assertTrue(flags & libvirt.VIR_MIGRATE_PERSIST_DEST)
|
|
|
|
def test_the_migration_uri_is_passed_on(self):
|
|
source = FakeSource(1)
|
|
destination().moveto(source, "vm", live=True, unsafe=False, offline=False, uri="tcp://192.0.2.12")
|
|
self.assertEqual(source.instance.calls[0][1], "tcp://192.0.2.12")
|
|
source = FakeSource(1)
|
|
destination().moveto(source, "vm", live=True, unsafe=False, offline=False)
|
|
self.assertIsNone(source.instance.calls[0][1])
|
|
|
|
|
|
class ShutOffMigrationTestCase(unittest.TestCase):
|
|
"""libvirt's offline migration fails on some builds (RHEL-156800); a
|
|
shut-off VM is only its definition, so the app moves that itself."""
|
|
|
|
def test_the_definition_moves_to_the_destination(self):
|
|
host = FakeHost()
|
|
source = FakeSource(5, xml="<domain><name>vm</name></domain>")
|
|
destination(host).moveto(source, "vm", live=False, unsafe=False, offline=True)
|
|
self.assertEqual(host.defined, ["<domain><name>vm</name></domain>"])
|
|
self.assertEqual(source.instance.calls, [])
|
|
self.assertEqual(source.instance.undefined, libvirt.VIR_DOMAIN_UNDEFINE_KEEP_NVRAM)
|
|
|
|
def test_tpm_state_stays_on_the_source(self):
|
|
source = FakeSource(5, xml="<domain><devices><tpm model='tpm-crb'/></devices></domain>")
|
|
destination().moveto(source, "vm", live=False, unsafe=False, offline=True)
|
|
self.assertEqual(
|
|
source.instance.undefined,
|
|
libvirt.VIR_DOMAIN_UNDEFINE_KEEP_NVRAM | libvirt.VIR_DOMAIN_UNDEFINE_KEEP_TPM,
|
|
)
|
|
|
|
def test_refused_before_anything_changes(self):
|
|
cases = {
|
|
"snapshots": (FakeHost(), {"snapshots": 2}),
|
|
"saved state": (FakeHost(), {"saved": True}),
|
|
"same uuid on the destination": (FakeHost(existing={"u-1"}), {}),
|
|
"same name on the destination": (FakeHost(existing={"vm"}), {}),
|
|
}
|
|
for case, (host, domain) in cases.items():
|
|
with self.subTest(case=case):
|
|
source = FakeSource(5, **domain)
|
|
with self.assertRaises(util.OperationError):
|
|
destination(host).moveto(source, "vm", live=False, unsafe=False, offline=True)
|
|
self.assertEqual(host.defined, [])
|
|
self.assertIsNone(source.instance.undefined)
|
|
|
|
def test_a_failed_source_undefine_removes_the_new_definition(self):
|
|
host = FakeHost()
|
|
source = FakeSource(5, undefine_fails=True)
|
|
with self.assertRaises(libvirt.libvirtError):
|
|
destination(host).moveto(source, "vm", live=False, unsafe=False, offline=True)
|
|
self.assertIsNotNone(host.new[0].undefined)
|
|
|
|
def test_the_destination_definition_stays_when_the_source_has_none(self):
|
|
host = FakeHost()
|
|
source = FakeSource(5, undefine_lost=True)
|
|
with self.assertRaisesRegex(util.OperationError, "only on the destination"):
|
|
destination(host).moveto(source, "vm", live=False, unsafe=False, offline=True)
|
|
self.assertIsNone(host.new[0].undefined)
|
|
|
|
def test_a_failed_rollback_is_reported(self):
|
|
host = FakeHost()
|
|
source = FakeSource(5, undefine_fails=True)
|
|
host.defineXML = lambda xml: host.new.append(FakeDomain(undefine_fails=True)) or host.new[-1]
|
|
with self.assertRaisesRegex(util.OperationError, "undefine failed; the VM is now defined on both hosts"):
|
|
destination(host).moveto(source, "vm", live=False, unsafe=False, offline=True)
|
|
|
|
def test_a_vm_started_meanwhile_stays_on_the_source(self):
|
|
host = FakeHost()
|
|
source = FakeSource(5, xml="<domain><name>vm</name></domain>", started=True)
|
|
with self.assertRaisesRegex(util.OperationError, "started on the source"):
|
|
destination(host).moveto(source, "vm", live=False, unsafe=False, offline=True)
|
|
self.assertEqual(source.instance.redefined, ["<domain><name>vm</name></domain>"])
|
|
self.assertEqual(source.instance.restored_autostart, 1)
|
|
self.assertIsNotNone(host.new[0].undefined)
|
|
|
|
|
|
class MigrationModeTestCase(unittest.TestCase):
|
|
"""The state is read once; a request that does not fit it is refused
|
|
before anything changes, and moveto names the mode it used."""
|
|
|
|
def migrate(self, state, **request):
|
|
source = FakeSource(state)
|
|
options = {"live": False, "unsafe": False, "offline": False, **request}
|
|
mode = destination().moveto(source, "vm", **options)
|
|
return mode, source.instance.calls[0][0] if source.instance.calls else None
|
|
|
|
def test_refused_requests(self):
|
|
for state, request in ((5, {"live": True}), (1, {"offline": True}), (3, {"offline": True}),
|
|
(1, {"live": True, "offline": True}), (6, {}), (4, {}), (7, {}), (0, {})):
|
|
with self.subTest(state=state, request=request):
|
|
source = FakeSource(state)
|
|
with self.assertRaises(util.OperationError):
|
|
destination().moveto(source, "vm", unsafe=False, **{"live": False, "offline": False, **request})
|
|
self.assertEqual(source.instance.calls, [])
|
|
|
|
def test_modes(self):
|
|
live, unsafe = libvirt.VIR_MIGRATE_LIVE, libvirt.VIR_MIGRATE_UNSAFE
|
|
converge, compressed = libvirt.VIR_MIGRATE_AUTO_CONVERGE, libvirt.VIR_MIGRATE_COMPRESSED
|
|
mode, flags = self.migrate(1, live=True, autoconverge=True, compress=True, unsafe=True)
|
|
self.assertEqual(mode, "live, unsafe, auto converge, compressed")
|
|
self.assertEqual(flags & (live | unsafe | converge | compressed), live | unsafe | converge | compressed)
|
|
source = FakeSource(1)
|
|
destination().moveto(source, "vm", live=True, unsafe=False, offline=False)
|
|
self.assertEqual(source.reads, 1)
|
|
mode, flags = self.migrate(1, compress=True, autoconverge=True)
|
|
self.assertEqual(mode, "non-live, compressed")
|
|
self.assertEqual(flags & (live | converge | compressed), compressed)
|
|
# a paused VM stays paused; the tuning options are for running VMs
|
|
mode, flags = self.migrate(3, live=True, autoconverge=True, compress=True, unsafe=True)
|
|
self.assertEqual(mode, "live")
|
|
self.assertEqual(flags & (live | unsafe | converge | compressed), live)
|
|
self.assertEqual(self.migrate(5), ("offline", None))
|
|
self.assertEqual(self.migrate(5, offline=True), ("offline", None))
|
|
|
|
|
|
def with_disks(*disks):
|
|
return "<domain><devices>%s</devices></domain>" % "".join(disks)
|
|
|
|
|
|
FILE_DISK = "<disk type='file' device='disk'><source file='/pool/vm.qcow2'/><target dev='vda'/></disk>"
|
|
|
|
|
|
class DestinationDisksTestCase(unittest.TestCase):
|
|
"""Every disk of the VM must be a volume the destination knows; the
|
|
VM is refused before anything changes otherwise, naming the disks."""
|
|
|
|
def migrate(self, host, xml, state=1, inactive_xml=None):
|
|
source = FakeSource(state, xml=xml)
|
|
source.instance.inactive_xml = inactive_xml
|
|
live = state != 5
|
|
destination(host).moveto(source, "vm", live=live, unsafe=False, offline=not live)
|
|
return source
|
|
|
|
def test_known_volumes_migrate(self):
|
|
host = FakeHost(volumes={"/pool/vm.qcow2", "/pool/base.qcow2", "/iso/os.iso"})
|
|
xml = with_disks(
|
|
"<disk type='file' device='disk'><source file='/pool/vm.qcow2'/><target dev='vda'/>"
|
|
"<backingStore type='file'><source file='/pool/base.qcow2'/><backingStore/></backingStore></disk>",
|
|
"<disk type='file' device='cdrom'><source file='/iso/os.iso'/><target dev='sda'/></disk>",
|
|
"<disk type='file' device='cdrom'><target dev='sdb'/></disk>", # empty
|
|
"<disk type='network' device='disk'><source protocol='rbd' name='rbd/vm'/><target dev='vdb'/></disk>",
|
|
"<disk type='volume' device='disk'><source pool='p' volume='/pool/vm.qcow2'/><target dev='vdc'/></disk>",
|
|
)
|
|
self.assertEqual(len(self.migrate(host, xml).instance.calls), 1)
|
|
self.assertEqual(host.defined, [])
|
|
self.migrate(host, xml, state=5)
|
|
self.assertEqual(len(host.defined), 1)
|
|
|
|
def test_missing_disks_are_refused_and_named(self):
|
|
cases = {
|
|
"vda \\(/pool/vm.qcow2\\)": with_disks(FILE_DISK),
|
|
"sda \\(/iso/os.iso\\)": with_disks(
|
|
"<disk type='file' device='cdrom'><source file='/iso/os.iso'/><target dev='sda'/></disk>"),
|
|
"vda \\(/pool/base.qcow2\\)": with_disks(
|
|
"<disk type='file' device='disk'><source file='/pool/vm.qcow2'/><target dev='vda'/>"
|
|
"<backingStore type='file'><source file='/pool/base.qcow2'/></backingStore></disk>"),
|
|
"vdb \\(/dev/sdz\\)": with_disks(
|
|
"<disk type='block' device='disk'><source dev='/dev/sdz'/><target dev='vdb'/></disk>"),
|
|
"vdc \\(p/v\\)": with_disks(
|
|
"<disk type='volume' device='disk'><source pool='p' volume='v'/><target dev='vdc'/></disk>"),
|
|
}
|
|
for missing, xml in cases.items():
|
|
for state in (1, 5):
|
|
with self.subTest(missing=missing, state=state):
|
|
host = FakeHost(volumes=set() if "vm.qcow2" in missing else {"/pool/vm.qcow2"})
|
|
source = FakeSource(state, xml=xml)
|
|
with self.assertRaisesRegex(util.OperationError, missing):
|
|
destination(host).moveto(source, "vm", live=state == 1, unsafe=False, offline=state == 5)
|
|
self.assertEqual((source.instance.calls, host.defined), ([], []))
|
|
|
|
def test_a_disk_of_the_persistent_definition_is_checked_too(self):
|
|
host = FakeHost(volumes={"/pool/vm.qcow2"})
|
|
inactive = with_disks(FILE_DISK, "<disk type='file' device='disk'><source file='/pool/data.img'/>"
|
|
"<target dev='vdb'/></disk>")
|
|
with self.assertRaisesRegex(util.OperationError, "vdb \\(/pool/data.img\\)"):
|
|
self.migrate(host, with_disks(FILE_DISK), inactive_xml=inactive)
|
|
|
|
def test_pools_are_refreshed_before_a_disk_counts_as_missing(self):
|
|
# a file another host made on shared storage is listed after a refresh
|
|
host = FakeHost(unlisted={"/pool/vm.qcow2"})
|
|
self.assertEqual(len(self.migrate(host, with_disks(FILE_DISK)).instance.calls), 1)
|
|
self.assertEqual(host.refreshes, 1)
|
|
host = FakeHost(volumes={"/pool/vm.qcow2"})
|
|
self.migrate(host, with_disks(FILE_DISK))
|
|
self.assertEqual(host.refreshes, 0)
|
|
|
|
def test_backing_files_the_definition_does_not_list_are_checked(self):
|
|
# an inactive definition has no <backingStore>: the volume tells
|
|
for state in (1, 5):
|
|
with self.subTest(state=state):
|
|
host = FakeHost(volumes={"/pool/vm.qcow2", "/pool/mid.qcow2"},
|
|
backing={"/pool/vm.qcow2": "/pool/mid.qcow2", "/pool/mid.qcow2": "/local/base.qcow2"})
|
|
with self.assertRaisesRegex(util.OperationError, "vda \\(/local/base.qcow2\\)"):
|
|
self.migrate(host, with_disks(FILE_DISK), state=state)
|
|
host.volumes.add("/local/base.qcow2")
|
|
self.migrate(host, with_disks(FILE_DISK), state=state)
|
|
|
|
def test_a_network_backing_image_is_not_checked(self):
|
|
host = FakeHost(volumes={"/pool/vm.qcow2"}, backing={"/pool/vm.qcow2": "nbd://storage.example.org:10809/base"})
|
|
self.assertEqual(len(self.migrate(host, with_disks(FILE_DISK)).instance.calls), 1)
|
|
|
|
def test_disks_that_cannot_be_checked_are_refused(self):
|
|
xml = with_disks("<disk type='dir' device='disk'><source dir='/srv/vm'/><target dev='vdb'/></disk>")
|
|
with self.assertRaisesRegex(util.OperationError, "vdb \\(dir disk, cannot be checked\\)"):
|
|
self.migrate(FakeHost(), xml, state=5)
|
|
|
|
def test_a_volume_of_an_inactive_pool_is_missing(self):
|
|
host = FakeHost(volumes={"v"}, inactive_pools={"p"})
|
|
xml = with_disks("<disk type='volume' device='disk'><source pool='p' volume='v'/><target dev='vdc'/></disk>")
|
|
with self.assertRaisesRegex(util.OperationError, "vdc \\(p/v\\)"):
|
|
self.migrate(host, xml)
|
|
|
|
|
|
class SlowDomain(FakeDomain):
|
|
"""A migration that does not finish until its job is aborted."""
|
|
|
|
def __init__(self, **kw):
|
|
super().__init__(**kw)
|
|
self.aborted = threading.Event()
|
|
|
|
def migrate(self, dconn, flags, dname, uri, bandwidth):
|
|
if not self.aborted.wait(5):
|
|
raise AssertionError("the migration was not aborted")
|
|
raise libvirt.libvirtError("operation aborted: job 'migration out' canceled by client")
|
|
|
|
def abortJob(self):
|
|
self.aborted.set()
|
|
|
|
|
|
class MigrationTimeLimitTestCase(unittest.TestCase):
|
|
"""A migration that does not converge is cancelled after the time limit,
|
|
before the web server ends the request; the VM stays on the source."""
|
|
|
|
def test_a_migration_over_the_limit_is_cancelled(self):
|
|
source = FakeSource(1)
|
|
source.instance = SlowDomain()
|
|
with self.assertRaisesRegex(util.OperationError, "did not finish within 0.05 s and was cancelled.*auto converge"):
|
|
destination().moveto(source, "vm", live=True, unsafe=False, offline=False, timeout=0.05)
|
|
self.assertTrue(source.instance.aborted.is_set())
|
|
|
|
def test_a_migration_within_the_limit_stops_its_timer(self):
|
|
timers = []
|
|
|
|
class Timer(threading.Timer):
|
|
def __init__(self, *args):
|
|
super().__init__(*args)
|
|
self.joined = False
|
|
timers.append(self)
|
|
|
|
def join(self, timeout=None):
|
|
self.joined = True
|
|
super().join(timeout)
|
|
|
|
source = FakeSource(1)
|
|
with patch("vrtManager.instance.threading.Timer", Timer):
|
|
destination().moveto(source, "vm", live=True, unsafe=False, offline=False, timeout=5)
|
|
self.assertEqual(len(source.instance.calls), 1)
|
|
self.assertTrue(timers[0].finished.is_set() and timers[0].joined)
|
|
self.assertFalse(timers[0].is_alive())
|
|
|
|
def test_a_failure_the_cancel_did_not_cause_keeps_its_error(self):
|
|
source = FakeSource(1)
|
|
source.instance = SlowDomain()
|
|
|
|
def refused():
|
|
source.instance.aborted.set() # migrate() gives up, but the abort itself failed
|
|
raise libvirt.libvirtError("cannot abort migration in confirm phase")
|
|
|
|
source.instance.abortJob = refused
|
|
with self.assertRaisesRegex(libvirt.libvirtError, "operation aborted") as caught:
|
|
destination().moveto(source, "vm", live=True, unsafe=False, offline=False, timeout=0.05)
|
|
self.assertNotIsInstance(caught.exception, util.OperationError)
|