mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-09 11:46:49 +00:00
cf991adad5
Without a migration address libvirt sends a migrating VM to the
destination's own host name, which the source host often cannot resolve:
a live migration to a host added without one failed ("Unable to resolve
address ..."), also when the other host had one. The destination's
address on the Computes page, which the panel reaches it at, is now the
default; the migration address stays for another network. A host the
panel reaches on its loopback keeps libvirt's default.
214 lines
13 KiB
Python
214 lines
13 KiB
Python
"""A migration runs as a job: the page queues it, the worker migrates."""
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from django.contrib.auth import get_user_model
|
|
from django.test import TransactionTestCase
|
|
from django.urls import reverse
|
|
|
|
from appsettings.settings import get_settings
|
|
from computes.models import Compute
|
|
from instances.jobs import MigrateInstance
|
|
from instances.models import Instance
|
|
from libvirt import VIR_DOMAIN_XML_INACTIVE, VIR_ERR_SYSTEM_ERROR, VIR_FROM_RPC, libvirtError
|
|
from operations.jobs import Cancelled, Context, get_kind
|
|
from operations.models import Claim, Operation, acquire
|
|
from operations.utils import public_error_text
|
|
|
|
UUID = "31313131-4242-5353-6464-757575757575"
|
|
|
|
|
|
class MigrateJobTestCase(TransactionTestCase):
|
|
def setUp(self):
|
|
get_settings()
|
|
self.source = Compute.objects.create(name="mig-src", hostname="192.0.2.50", login="root", password="", type=1)
|
|
self.dest = Compute.objects.create(name="mig-dst", hostname="192.0.2.51", login="root", password="", type=1)
|
|
self.vm = Instance.objects.create(compute=self.source, name="mig-vm", uuid=UUID)
|
|
self.client.force_login(get_user_model().objects.create_superuser("mig_admin", "a@example.com", "pw"))
|
|
proxy = MagicMock()
|
|
proxy.get_autostart.return_value = False
|
|
proxy.get_nvram.return_value = None
|
|
patch.object(Compute, "status", True).start()
|
|
patch.object(Instance, "proxy", proxy).start()
|
|
# another host than the VM's
|
|
patch.object(Compute, "proxy", MagicMock(**{"host_uuid.return_value": "dddddddd-0000-0000-0000-000000000051"})).start()
|
|
self.dest_conn = patch("instances.utils.wvmInstances").start().return_value
|
|
self.dest_conn.moveto.return_value = ("live", [])
|
|
self.addCleanup(patch.stopall)
|
|
|
|
def queue(self, **form):
|
|
self.client.post(reverse("instances:migrate", args=[self.vm.pk]),
|
|
{"compute_id": self.dest.pk, "submission_key": "key-mig00001", **form})
|
|
return Operation.objects.get()
|
|
|
|
def start(self, op):
|
|
"""The job as the worker takes it."""
|
|
acquire("wvc-test:1:a", "wvc-test")
|
|
Operation.objects.filter(pk=op.pk).update(state=Operation.RUNNING, worker="wvc-test:1:a")
|
|
op.refresh_from_db()
|
|
return Context(op, SimpleNamespace(identity="wvc-test:1:a"))
|
|
|
|
def test_the_worker_migrates_and_moves_the_record(self):
|
|
op = self.queue(live_migrate="true")
|
|
self.assertEqual(set(Claim.objects.values_list("resource", "mode")), {
|
|
(f"instance:{UUID}", "exclusive"), ("name:mig-vm", "exclusive"),
|
|
(f"compute:{self.source.pk}", "shared"), (f"reconcile:{self.source.pk}", "exclusive"),
|
|
(f"compute:{self.dest.pk}", "shared"), (f"reconcile:{self.dest.pk}", "exclusive")})
|
|
self.dest_conn.moveto.assert_not_called() # nothing ran in the request
|
|
ctx = self.start(op)
|
|
get_kind(op.kind).precheck(op, ctx)
|
|
get_kind(op.kind).run(op, ctx)
|
|
self.assertTrue(self.dest_conn.moveto.call_args.args[2]) # live
|
|
self.vm.refresh_from_db()
|
|
self.assertEqual(self.vm.compute, self.dest)
|
|
|
|
def test_recovery_reads_where_the_vm_is(self):
|
|
op = Operation(params={"instance": self.vm.pk, "target": self.dest.pk},
|
|
intent={"compute": self.source.pk, "uuid": UUID})
|
|
dom = MagicMock()
|
|
dom.isActive.return_value = False
|
|
with patch("instances.jobs._domain", side_effect=[dom, None]): # source, destination
|
|
self.assertEqual(MigrateInstance().reconcile(op).state, Operation.FAILED)
|
|
dom.isActive.return_value = True
|
|
dom.jobInfo.return_value = [2] + [0] * 11 # VIR_DOMAIN_JOB_UNBOUNDED
|
|
with patch("instances.jobs._domain", side_effect=[dom, dom]):
|
|
self.assertEqual(MigrateInstance().reconcile(op).state, Operation.RUNNING)
|
|
with patch("instances.jobs._domain", side_effect=[None, dom]):
|
|
self.assertEqual(MigrateInstance().reconcile(op).state, Operation.SUCCEEDED)
|
|
self.vm.refresh_from_db()
|
|
self.assertEqual(self.vm.compute, self.dest)
|
|
|
|
def test_a_copy_claims_its_volume_on_both_computes(self):
|
|
Instance.proxy.get_disk_devices.return_value = [
|
|
{"dev": "vda", "storage": "default", "image": "mig-vm.qcow2", "path": "/i/mig-vm.qcow2"},
|
|
{"dev": "vdb", "storage": None, "image": "/dev/sdb", "path": "/dev/sdb"}]
|
|
op = self.queue(live_migrate="true", delete_source="true", **{"pool-vda": "fast", "pool-vdb": ""})
|
|
self.assertTrue({(f"volume:{self.source.pk}:default/mig-vm.qcow2", "exclusive"),
|
|
(f"volume:{self.dest.pk}:fast/mig-vm.qcow2", "exclusive")}
|
|
<= set(Claim.objects.values_list("resource", "mode")))
|
|
self.assertEqual((op.params["disk_pools"], op.params["delete_source"]), ({"vda": "fast"}, True))
|
|
ctx = self.start(op)
|
|
get_kind(op.kind).run(op, ctx)
|
|
self.assertEqual(self.dest_conn.moveto.call_args.kwargs["pools"], {"vda": "fast"})
|
|
|
|
def test_the_migrate_tab_lists_each_disk_with_where_it_can_go(self):
|
|
Instance.proxy.instance.XMLDesc.return_value = "<domain/>"
|
|
Instance.proxy.get_status.return_value = 5
|
|
with patch("instances.views.wvmInstances") as instances:
|
|
dest = instances.return_value
|
|
fast, default = MagicMock(), MagicMock()
|
|
fast.info.return_value = default.info.return_value = [2, 0, 0, 8 << 30]
|
|
dest.file_pools.return_value = {"fast": (fast, "/srv/fast"), "default": (default, "/i")}
|
|
dest.migration_plan.return_value = ([
|
|
{"target": "vda", "path": "/i/mig-vm.qcow2", "capacity": 1 << 30, "shared": False, "refusal": None,
|
|
"free_pools": ["default", "fast"], "default": "default"},
|
|
{"target": "vdb", "path": "/i/data.qcow2", "capacity": 1 << 30, "shared": False, "refusal": None,
|
|
"free_pools": ["default"], "default": "default"},
|
|
{"target": "sda", "path": "/i/os.iso", "capacity": 1 << 20, "shared": False,
|
|
"refusal": "an image is not copied; eject it first", "free_pools": [], "default": None}],
|
|
# the next start's disks the migration does not carry
|
|
[{"target": "vdc", "path": "/i/new.qcow2", "shared": False},
|
|
{"target": "vdd", "path": "/nfs/new.qcow2", "shared": True}])
|
|
page = self.client.get(reverse("instances:migrate_plan", args=[self.vm.pk]),
|
|
{"compute_id": self.dest.pk}).content.decode()
|
|
# the definitions it plans with: the one it carries and the next start's
|
|
self.assertEqual(dest.migration_plan.call_args.args[1:], ("<domain/>", "<domain/>"))
|
|
self.assertEqual(Instance.proxy.instance.XMLDesc.call_args_list[-1].args, (VIR_DOMAIN_XML_INACTIVE,))
|
|
# the pool on the disk's directory is chosen; one that has a file of the name cannot be
|
|
self.assertIn('<option value="default" selected>Copy to default (8.0\xa0GB free)</option>', page)
|
|
self.assertIn('<option value="fast">Copy to fast (8.0\xa0GB free)</option>', page)
|
|
self.assertIn('<option value="fast" disabled>fast has a file of this name</option>', page)
|
|
self.assertIn("Cannot be copied: an image is not copied; eject it first", page)
|
|
self.assertIn("A shut-off VM is started paused for a copy", page)
|
|
self.assertIn("/nfs/new.qcow2</span>:\n on storage both hosts share", page)
|
|
# a disk that cannot go keeps the form from being sent, as does a destination that does not answer
|
|
self.assertEqual(page.count("data-refused"), 2)
|
|
dest.close.assert_called_once_with()
|
|
with patch("instances.views.wvmInstances", side_effect=libvirtError("no answer")):
|
|
page = self.client.get(reverse("instances:migrate_plan", args=[self.vm.pk]),
|
|
{"compute_id": self.dest.pk}).content.decode()
|
|
self.assertIn("data-refused", page)
|
|
self.assertEqual(self.client.get(reverse("instances:migrate_plan", args=[self.vm.pk])).status_code, 404)
|
|
|
|
def test_the_destinations_that_cannot_be_chosen(self):
|
|
down = Compute.objects.create(name="mig-down", hostname="192.0.2.52", login="root", password="", type=1)
|
|
alias = Compute.objects.create(name="mig-alias", hostname="192.0.2.53", login="root", password="", type=1)
|
|
Instance.proxy.host_uuid.return_value = "aaaaaaaa-0000-0000-0000-000000000050"
|
|
# the alias is the source host under another record: the same libvirt host UUID
|
|
uuids = {alias.pk: "aaaaaaaa-0000-0000-0000-000000000050"}
|
|
with patch.object(Compute, "status", property(lambda c: c.pk != down.pk)), patch.object(
|
|
Compute, "proxy", property(lambda c: MagicMock(**{"host_uuid.return_value": uuids.get(c.pk, str(c.pk))}))):
|
|
refused = self.client.get(reverse("instances:migrate_targets", args=[self.vm.pk])).json()
|
|
self.assertEqual(refused, {str(down.pk): "mig-down is not reachable",
|
|
str(alias.pk): "mig-alias is the VM's own host under another name"})
|
|
# a request that names it anyway is refused too
|
|
self.client.post(reverse("instances:migrate", args=[self.vm.pk]),
|
|
{"compute_id": alias.pk, "submission_key": "key-mig00002"})
|
|
self.assertFalse(Operation.objects.exists())
|
|
|
|
def test_a_cancelled_migration_ends_cancelled_and_the_vm_stays(self):
|
|
op = self.queue(live_migrate="true")
|
|
ctx = self.start(op)
|
|
self.dest_conn.moveto.side_effect = libvirtError("operation aborted: migration out: canceled by client")
|
|
Operation.objects.filter(pk=op.pk).update(cancel_requested=True)
|
|
with self.assertRaises(Cancelled):
|
|
get_kind(op.kind).run(op, ctx)
|
|
self.vm.refresh_from_db()
|
|
self.assertEqual(self.vm.compute, self.source)
|
|
|
|
def test_a_failed_migration_says_what_libvirt_said(self):
|
|
# libvirt reports the source host's failure to reach the destination's migration
|
|
# port as a connection error; only superusers migrate, so its text is shown
|
|
op = self.queue(live_migrate="true")
|
|
ctx = self.start(op)
|
|
err = libvirtError("Unable to resolve address 'kvm2.example.com' service '49152': Name or service not known")
|
|
err.err = (VIR_ERR_SYSTEM_ERROR, VIR_FROM_RPC, str(err), 2, None, None, None, 0, 0)
|
|
self.dest_conn.moveto.side_effect = err
|
|
with self.assertRaises(Exception) as raised:
|
|
get_kind(op.kind).run(op, ctx)
|
|
self.assertIn("Unable to resolve address 'kvm2.example.com'", public_error_text(raised.exception))
|
|
|
|
def test_each_way_goes_to_the_destination_s_migration_address_or_its_panel_address(self):
|
|
# one host with a migration address, the other without: no name resolution either way
|
|
self.dest.migration_address = ""
|
|
self.dest.save()
|
|
self.source.migration_address = "198.51.100.50"
|
|
self.source.save()
|
|
for target, uri, key in ((self.dest, "tcp://192.0.2.51", "key-mig00011"), (self.source, "tcp://198.51.100.50", "key-mig00012")):
|
|
with self.subTest(target=target.name):
|
|
self.vm.compute = self.source if target == self.dest else self.dest
|
|
self.vm.save()
|
|
self.client.post(reverse("instances:migrate", args=[self.vm.pk]),
|
|
{"compute_id": target.pk, "submission_key": key, "live_migrate": "true"})
|
|
op = Operation.objects.latest("created_at")
|
|
ctx = self.start(op)
|
|
get_kind(op.kind).precheck(op, ctx)
|
|
get_kind(op.kind).run(op, ctx)
|
|
self.assertEqual(self.dest_conn.moveto.call_args.kwargs["uri"], uri)
|
|
Operation.objects.all().delete()
|
|
Claim.objects.all().delete()
|
|
|
|
def test_cancel_and_progress_ask_the_source_vm(self):
|
|
op = self.queue(live_migrate="true")
|
|
with patch("instances.jobs.wvmInstance") as proxy:
|
|
proxy.return_value.instance.jobInfo.return_value = [2, 0, 0, 1000, 250] + [0] * 7
|
|
self.assertEqual(get_kind(op.kind).progress(op), (250, 1000))
|
|
get_kind(op.kind).cancel(op)
|
|
proxy.return_value.instance.abortJob.assert_called_once_with()
|
|
|
|
def test_a_cancel_before_libvirt_starts_the_migration_waits_for_it(self):
|
|
op = self.queue(live_migrate="true")
|
|
self.start(op)
|
|
with patch("instances.jobs.wvmInstance") as proxy, patch("instances.jobs.time.sleep") as sleep:
|
|
dom = proxy.return_value.instance
|
|
dom.isActive.return_value = True
|
|
dom.jobInfo.side_effect = [[0] * 12, [2] + [0] * 11] # no job yet, then the migration
|
|
get_kind(op.kind).cancel(op)
|
|
sleep.assert_called_once()
|
|
dom.abortJob.assert_called_once_with()
|
|
Operation.objects.filter(pk=op.pk).update(state=Operation.SUCCEEDED) # ended before libvirt began
|
|
with patch("instances.jobs.wvmInstance") as proxy:
|
|
proxy.return_value.instance.isActive.return_value = False
|
|
get_kind(op.kind).cancel(op)
|
|
proxy.return_value.instance.abortJob.assert_not_called()
|