mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-09 11:46:49 +00:00
perf(instances): the VM page reads each disk, the state and host info once
Measured on a VM with two disks, a CD-ROM and a NIC: 70 libvirt calls per page, now 43. Within the page's cached_reads block the VM's state, the host's info and domain capabilities, and each disk path's volume are read once; the running and the persistent disk lists share a path's volume. The ISO list takes the active pools in one call instead of looking each up by name; it still refreshes them, so an image copied in by hand shows up. The opt-in DRBD status waits at most 10 s for the compute's ssh, then leaves the status unknown instead of holding the page.
This commit is contained in:
catborise
parent
95b4418c96
commit
439f24041e
6 changed files with 278 additions and 52 deletions
|
|
@ -480,9 +480,10 @@ class wvmInstance(wvmConnect):
|
|||
@contextlib.contextmanager
|
||||
def cached_reads(self):
|
||||
"""
|
||||
Within the block the domain XML (per flags value) and the host
|
||||
capabilities are read from libvirt once, for pages that only read: a
|
||||
VM page reads the XML in some 30 getters. Code that changes the VM
|
||||
Within the block the domain XML (per flags value), the VM's state,
|
||||
the host's capabilities and info, and each disk's volume are read
|
||||
from libvirt once, for pages that only read: a VM page reads the XML
|
||||
in some 30 getters and each disk in two lists. Code that changes the VM
|
||||
must not run inside: most changes (devices, QoS, memory) do not go
|
||||
through _defineXML, the only call that drops the cache.
|
||||
"""
|
||||
|
|
@ -494,27 +495,31 @@ class wvmInstance(wvmConnect):
|
|||
finally:
|
||||
self._read_cache = None
|
||||
|
||||
def _XMLDesc(self, flag):
|
||||
def _cached(self, key, read):
|
||||
cache = self._read_cache
|
||||
if cache is None:
|
||||
return self.instance.XMLDesc(flag)
|
||||
key = ("xml", flag)
|
||||
return read()
|
||||
if key not in cache:
|
||||
cache[key] = self.instance.XMLDesc(flag)
|
||||
cache[key] = read()
|
||||
return cache[key]
|
||||
|
||||
def _XMLDesc(self, flag):
|
||||
return self._cached(("xml", flag), lambda: self.instance.XMLDesc(flag))
|
||||
|
||||
def _defineXML(self, xml):
|
||||
if self._read_cache is not None:
|
||||
self._read_cache.clear()
|
||||
return self.wvm.defineXML(xml)
|
||||
|
||||
def get_cap_xml(self):
|
||||
cache = self._read_cache
|
||||
if cache is None:
|
||||
return super().get_cap_xml()
|
||||
if "capabilities" not in cache:
|
||||
cache["capabilities"] = super().get_cap_xml()
|
||||
return cache["capabilities"]
|
||||
return self._cached("capabilities", super().get_cap_xml)
|
||||
|
||||
def get_dom_cap_xml(self, arch, machine):
|
||||
read = super().get_dom_cap_xml
|
||||
return self._cached(("domain capabilities", arch, machine), lambda: read(arch, machine))
|
||||
|
||||
def _host_info(self):
|
||||
return self._cached("host info", self.wvm.getInfo)
|
||||
|
||||
def get_status(self):
|
||||
"""
|
||||
|
|
@ -523,7 +528,7 @@ class wvmInstance(wvmConnect):
|
|||
VIR_DOMAIN_PAUSED = 3
|
||||
VIR_DOMAIN_SHUTOFF = 5
|
||||
"""
|
||||
return self.instance.info()[0]
|
||||
return self._cached("status", lambda: self.instance.info()[0])
|
||||
|
||||
def get_autostart(self):
|
||||
return self.instance.autostart()
|
||||
|
|
@ -608,11 +613,11 @@ class wvmInstance(wvmConnect):
|
|||
return description or ""
|
||||
|
||||
def get_max_memory(self):
|
||||
return self.wvm.getInfo()[1] * 1048576
|
||||
return self._host_info()[1] * 1048576
|
||||
|
||||
def get_max_cpus(self):
|
||||
"""Get number of physical CPUs."""
|
||||
hostinfo = self.wvm.getInfo()
|
||||
hostinfo = self._host_info()
|
||||
pcpus = hostinfo[4] * hostinfo[5] * hostinfo[6] * hostinfo[7]
|
||||
return range(1, int(pcpus + 1))
|
||||
|
||||
|
|
@ -775,15 +780,10 @@ class wvmInstance(wvmConnect):
|
|||
else None
|
||||
)
|
||||
|
||||
try:
|
||||
vol = self.get_volume_by_path(src_file)
|
||||
volume = vol.name()
|
||||
|
||||
disk_size = vol.info()[1]
|
||||
used_size = vol.info()[2]
|
||||
stg = vol.storagePoolLookupByVolume()
|
||||
storage = stg.name()
|
||||
except libvirtError:
|
||||
details = self._volume_details(src_file)
|
||||
if details:
|
||||
volume, disk_size, used_size, storage = details
|
||||
else:
|
||||
volume = src_file
|
||||
except Exception as e:
|
||||
print(f"Exception: {e}")
|
||||
|
|
@ -813,6 +813,20 @@ class wvmInstance(wvmConnect):
|
|||
flags = VIR_DOMAIN_XML_INACTIVE | VIR_DOMAIN_XML_SECURE if config else 0
|
||||
return util.get_xml_path(self._XMLDesc(flags), func=disks)
|
||||
|
||||
def _volume_details(self, path):
|
||||
"""(name, capacity, allocation, pool name) of the volume at path, or
|
||||
None when no pool holds it."""
|
||||
|
||||
def read():
|
||||
try:
|
||||
vol = self.get_volume_by_path(path)
|
||||
info = vol.info()
|
||||
return vol.name(), info[1], info[2], vol.storagePoolLookupByVolume().name()
|
||||
except libvirtError:
|
||||
return None
|
||||
|
||||
return self._cached(("volume", path), read)
|
||||
|
||||
def get_media_devices(self):
|
||||
def disks(doc):
|
||||
result = []
|
||||
|
|
@ -1541,16 +1555,15 @@ class wvmInstance(wvmConnect):
|
|||
|
||||
def get_iso_media(self):
|
||||
iso = []
|
||||
storages = self.get_storages(only_actives=True)
|
||||
for storage in storages:
|
||||
stg = self.get_storage(storage)
|
||||
if stg.info()[0] != 0:
|
||||
with contextlib.suppress(Exception):
|
||||
stg.refresh(0)
|
||||
|
||||
for img in stg.listVolumes():
|
||||
if img.lower().endswith(".iso"):
|
||||
iso.append(img)
|
||||
for pool in self.wvm.listAllStoragePools(VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE):
|
||||
# refreshed so that an image copied to the pool by hand shows up
|
||||
with contextlib.suppress(libvirtError):
|
||||
pool.refresh(0)
|
||||
try:
|
||||
names = pool.listVolumes()
|
||||
except libvirtError: # stopped or removed meanwhile
|
||||
continue
|
||||
iso.extend(name for name in names if name.lower().endswith(".iso"))
|
||||
return iso
|
||||
|
||||
def paths_used_by_other_domains(self):
|
||||
|
|
|
|||
|
|
@ -6,7 +6,12 @@ from django.conf import settings
|
|||
if not settings.configured:
|
||||
settings.configure(MAC_OUI="52:54:10")
|
||||
|
||||
from libvirt import VIR_DOMAIN_AFFECT_CONFIG, VIR_DOMAIN_AFFECT_LIVE, libvirtError
|
||||
from libvirt import (
|
||||
VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE,
|
||||
VIR_DOMAIN_AFFECT_CONFIG,
|
||||
VIR_DOMAIN_AFFECT_LIVE,
|
||||
libvirtError,
|
||||
)
|
||||
from lxml import etree
|
||||
from vrtManager.instance import wvmInstance
|
||||
|
||||
|
|
@ -52,5 +57,43 @@ class IsoMediaTestCase(unittest.TestCase):
|
|||
inst.instance.updateDeviceFlags.assert_not_called()
|
||||
|
||||
|
||||
def pool(volumes, refresh_error=False, list_error=False):
|
||||
p = MagicMock()
|
||||
if refresh_error:
|
||||
p.refresh.side_effect = libvirtError("refresh failed")
|
||||
if list_error:
|
||||
p.listVolumes.side_effect = libvirtError("pool is not active")
|
||||
else:
|
||||
p.listVolumes.return_value = volumes
|
||||
return p
|
||||
|
||||
|
||||
class IsoListTestCase(unittest.TestCase):
|
||||
"""The CD-ROM choices are the .iso volumes of the active pools, refreshed
|
||||
so that an image copied in by hand shows up."""
|
||||
|
||||
def iso(self, *pools):
|
||||
inst = wvmInstance.__new__(wvmInstance)
|
||||
inst.wvm = MagicMock()
|
||||
inst.wvm.listAllStoragePools.return_value = list(pools)
|
||||
return inst, inst.get_iso_media()
|
||||
|
||||
def test_iso_volumes_of_the_active_pools(self):
|
||||
a, b = pool(["a.iso", "disk.qcow2"]), pool(["B.ISO"])
|
||||
inst, iso = self.iso(a, b)
|
||||
self.assertEqual(iso, ["a.iso", "B.ISO"])
|
||||
inst.wvm.listAllStoragePools.assert_called_once_with(VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE)
|
||||
a.refresh.assert_called_once_with(0)
|
||||
b.refresh.assert_called_once_with(0)
|
||||
|
||||
def test_a_failed_refresh_still_lists_the_pool(self):
|
||||
_, iso = self.iso(pool(["a.iso"], refresh_error=True))
|
||||
self.assertEqual(iso, ["a.iso"])
|
||||
|
||||
def test_a_pool_stopped_meanwhile_is_skipped(self):
|
||||
_, iso = self.iso(pool([], list_error=True), pool(["b.iso"]))
|
||||
self.assertEqual(iso, ["b.iso"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
"""wvmInstance.cached_reads(): inside the block the domain XML (per flags) and
|
||||
the host capabilities are read from libvirt once; outside it every read goes
|
||||
to libvirt, as code that changes the VM needs."""
|
||||
"""wvmInstance.cached_reads(): inside the block the domain XML (per flags),
|
||||
the VM's state, the host capabilities and info, and each disk's volume are
|
||||
read from libvirt once; outside it every read goes to libvirt, as code that
|
||||
changes the VM needs."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
if not settings.configured:
|
||||
settings.configure(MAC_OUI="52:54:10")
|
||||
|
||||
from libvirt import libvirtError
|
||||
from vrtManager.connection import wvmConnect
|
||||
from vrtManager.instance import wvmInstance
|
||||
|
||||
|
||||
|
|
@ -17,6 +21,10 @@ class FakeDomain:
|
|||
self.reads = []
|
||||
self.fail = False
|
||||
|
||||
def info(self):
|
||||
self.reads.append("info")
|
||||
return [1, 0, 0, 0, 0]
|
||||
|
||||
def XMLDesc(self, flags):
|
||||
if self.fail:
|
||||
raise RuntimeError("libvirt error")
|
||||
|
|
@ -29,6 +37,10 @@ class FakeConn:
|
|||
self.caps = 0
|
||||
self.defined = []
|
||||
|
||||
def getInfo(self):
|
||||
self.caps += 1
|
||||
return ["x86_64", 2048, 4, 0, 1, 2, 2, 1]
|
||||
|
||||
def getCapabilities(self):
|
||||
self.caps += 1
|
||||
return "<capabilities/>"
|
||||
|
|
@ -111,3 +123,100 @@ class CachedReadsTestCase(unittest.TestCase):
|
|||
b._XMLDesc(0)
|
||||
b._XMLDesc(0)
|
||||
self.assertEqual((a.instance.reads, b.instance.reads), ([0], [0, 0]))
|
||||
|
||||
def test_the_state_and_host_info_are_read_once(self):
|
||||
p = proxy()
|
||||
with p.cached_reads():
|
||||
for _ in range(2):
|
||||
p.get_status()
|
||||
p.get_max_memory()
|
||||
p.get_max_cpus()
|
||||
self.assertEqual((p.instance.reads, p.wvm.caps), (["info"], 1))
|
||||
p.get_status()
|
||||
self.assertEqual(p.instance.reads, ["info", "info"])
|
||||
|
||||
def test_domain_capabilities_are_read_once_per_arch_and_machine(self):
|
||||
p = proxy()
|
||||
with patch.object(wvmConnect, "get_dom_cap_xml", return_value="<domainCapabilities/>") as read:
|
||||
with p.cached_reads():
|
||||
for _ in range(2):
|
||||
p.get_dom_cap_xml("x86_64", "q35")
|
||||
p.get_dom_cap_xml("x86_64", "pc")
|
||||
p.get_dom_cap_xml("x86_64", "q35")
|
||||
self.assertEqual([c.args for c in read.call_args_list], [("x86_64", "q35"), ("x86_64", "pc"), ("x86_64", "q35")])
|
||||
|
||||
|
||||
def volume_proxy(paths):
|
||||
"""A proxy whose host has volumes at paths (pool "pool", size 10, used 4)."""
|
||||
p = proxy()
|
||||
p.lookups = []
|
||||
|
||||
def lookup(path):
|
||||
p.lookups.append(path)
|
||||
if path not in paths:
|
||||
raise libvirtError("no storage vol with matching path")
|
||||
vol = MagicMock()
|
||||
vol.name.return_value = path.rsplit("/", 1)[-1]
|
||||
vol.info.return_value = [0, 10, 4]
|
||||
vol.storagePoolLookupByVolume.return_value.name.return_value = "pool"
|
||||
return vol
|
||||
|
||||
p.get_volume_by_path = lookup
|
||||
return p
|
||||
|
||||
|
||||
class VolumeDetailsTestCase(unittest.TestCase):
|
||||
def test_inside_the_block_each_path_is_looked_up_once(self):
|
||||
p = volume_proxy({"/p/a.qcow2", "/p/b.raw"})
|
||||
with p.cached_reads():
|
||||
for _ in range(2):
|
||||
self.assertEqual(p._volume_details("/p/a.qcow2"), ("a.qcow2", 10, 4, "pool"))
|
||||
p._volume_details("/p/b.raw")
|
||||
self.assertIsNone(p._volume_details("/dev/sdx"))
|
||||
self.assertEqual(p.lookups, ["/p/a.qcow2", "/p/b.raw", "/dev/sdx"])
|
||||
|
||||
def test_outside_the_block_and_after_a_change_the_volume_is_read_again(self):
|
||||
p = volume_proxy({"/p/a.qcow2"})
|
||||
p._volume_details("/p/a.qcow2")
|
||||
with p.cached_reads():
|
||||
p._volume_details("/p/a.qcow2")
|
||||
p._defineXML("<domain/>")
|
||||
p._volume_details("/p/a.qcow2")
|
||||
p._volume_details("/p/a.qcow2")
|
||||
self.assertEqual(len(p.lookups), 4)
|
||||
|
||||
|
||||
def disk(path, cache):
|
||||
return (
|
||||
f"<disk type='file' device='disk'><driver name='qemu' type='qcow2' cache='{cache}'/>"
|
||||
f"<source file='{path}'/><target dev='vda' bus='virtio'/></disk>"
|
||||
)
|
||||
|
||||
|
||||
class DiskListsTestCase(unittest.TestCase):
|
||||
"""The page lists the running and the persistent disks; both share a
|
||||
path's volume, each keeps its own settings."""
|
||||
|
||||
def lists(self, live, config, volumes):
|
||||
p = volume_proxy(volumes)
|
||||
xml = {0: f"<domain><devices>{live}</devices></domain>", 3: f"<domain><devices>{config}</devices></domain>"}
|
||||
p.instance.XMLDesc = lambda flags: xml[flags]
|
||||
with p.cached_reads():
|
||||
return p, p.get_disk_devices(), p.get_disk_devices(config=True)
|
||||
|
||||
def test_one_path_in_both_lists_is_looked_up_once(self):
|
||||
p, live, config = self.lists(disk("/p/a.qcow2", "none"), disk("/p/a.qcow2", "writeback"), {"/p/a.qcow2"})
|
||||
self.assertEqual(p.lookups, ["/p/a.qcow2"])
|
||||
self.assertEqual([(d["image"], d["storage"], d["size"], d["cache"]) for d in live], [("a.qcow2", "pool", 10, "none")])
|
||||
self.assertEqual([d["cache"] for d in config], ["writeback"])
|
||||
self.assertIsNot(live[0], config[0])
|
||||
|
||||
def test_a_pending_disk_change_is_looked_up_on_its_own(self):
|
||||
p, live, config = self.lists(disk("/p/a.qcow2", "none"), disk("/p/b.qcow2", "none"), {"/p/a.qcow2", "/p/b.qcow2"})
|
||||
self.assertEqual(p.lookups, ["/p/a.qcow2", "/p/b.qcow2"])
|
||||
self.assertEqual((live[0]["image"], config[0]["image"]), ("a.qcow2", "b.qcow2"))
|
||||
|
||||
def test_a_disk_outside_any_pool_shows_its_path(self):
|
||||
p, live, _ = self.lists(disk("/srv/x.img", "none"), disk("/srv/x.img", "none"), set())
|
||||
self.assertEqual(p.lookups, ["/srv/x.img"])
|
||||
self.assertEqual((live[0]["image"], live[0]["storage"], live[0]["size"]), ("/srv/x.img", None, None))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue