mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-10 04:06:47 +00:00
perf(instances): the VM stats no longer wait a second per poll
Each stats answer carries a signed sample of the counters, bound to the VM, its compute and UUID. The next poll sends it back and the rates are the averages since then; a sample from another run of the VM (domain ID), another host CPU count or device list, or older than 30 s falls back to one second as before. Windows are measured on the monotonic clock. Now that a poll is cheap, the Stats tab polls every 5 s like the host graph, the first time as soon as it opens (pollWhileVisible takes the delay of the first request). A counter the host reports as unsupported (-1) or that went back gives no value instead of 0 or a jump; a NIC without a target keeps its chart, a pool volume disk is measured, and a disk or NIC attached after the page loaded no longer stops the other charts. Disk rates are MB/s (they were MiB/s labelled Mb/s), network Mbit/s; the CPU tooltip names the window.
This commit is contained in:
catborise
parent
27f4845086
commit
24231813d7
8 changed files with 501 additions and 152 deletions
|
|
@ -60,6 +60,16 @@ from vrtManager.storage import wvmStorage, wvmStorages
|
|||
# with secrets. Live XML would drop pending changes, non-secure XML the VNC
|
||||
# password.
|
||||
PERSISTENT_XML = VIR_DOMAIN_XML_INACTIVE | VIR_DOMAIN_XML_SECURE
|
||||
# the longest window wvmInstance.usage() averages over
|
||||
USAGE_MAX_WINDOW = 30
|
||||
|
||||
|
||||
def _rate(before, after, seconds, scale=1):
|
||||
"""Growth per second of a counter, or None when a reading is unsupported
|
||||
(-1) or the counter went back."""
|
||||
if before < 0 or after < before:
|
||||
return None
|
||||
return scale * (after - before) / seconds
|
||||
|
||||
|
||||
class wvmInstances(wvmConnect):
|
||||
|
|
@ -1266,68 +1276,91 @@ class wvmInstance(wvmConnect):
|
|||
mem_usage["total"] = 0
|
||||
return mem_usage
|
||||
|
||||
def usage(self):
|
||||
def usage(self, previous=None):
|
||||
"""
|
||||
CPU (percent of the host), disk read/write bytes and network rx/tx
|
||||
bits of the last second, for the stats view. All counters are read
|
||||
before and after one shared second: the view runs in a web worker,
|
||||
and a second per device kept it busy for 1 + disks + NICs seconds.
|
||||
bits per second, for the stats view. With previous, the "sample" of an
|
||||
earlier answer, they are averages since then and nothing waits; without
|
||||
a usable one, over one second. A sample is usable for the same run of
|
||||
the VM (domain ID) on a host with the same CPUs and devices, at most
|
||||
USAGE_MAX_WINDOW seconds old. A value the host does not report (-1) or
|
||||
whose counter went back is None.
|
||||
"""
|
||||
devices = []
|
||||
tree = ElementTree.fromstring(self._XMLDesc(0))
|
||||
disks = []
|
||||
for disk in tree.findall("devices/disk"):
|
||||
if disk.get("device") == "disk":
|
||||
dev_file = None
|
||||
dev_bus = None
|
||||
network_disk = True
|
||||
for elm in disk:
|
||||
if elm.tag == "source":
|
||||
if elm.get("protocol"):
|
||||
dev_file = elm.get("protocol")
|
||||
network_disk = True
|
||||
if elm.get("file"):
|
||||
dev_file = elm.get("file")
|
||||
if elm.get("dev"):
|
||||
dev_file = elm.get("dev")
|
||||
if elm.tag == "target":
|
||||
dev_bus = elm.get("dev")
|
||||
if (dev_file and dev_bus) is not None:
|
||||
if network_disk:
|
||||
dev_file = dev_bus
|
||||
devices.append([dev_file, dev_bus])
|
||||
source, target = disk.find("source"), disk.find("target")
|
||||
if (
|
||||
disk.get("device") == "disk"
|
||||
and source is not None
|
||||
and any(source.get(name) for name in ("protocol", "file", "dev", "volume"))
|
||||
and target is not None
|
||||
and target.get("dev")
|
||||
):
|
||||
disks.append(target.get("dev")) # blockStats takes the target name
|
||||
# all NICs, in the order of the page's charts; one without a target is not measured
|
||||
nics = []
|
||||
running = self.get_status() == 1
|
||||
if running:
|
||||
tree = ElementTree.fromstring(self._XMLDesc(0))
|
||||
for target in tree.findall("devices/interface/target"):
|
||||
nics.append(target.get("dev"))
|
||||
for nic in tree.findall("devices/interface"):
|
||||
mac, target = nic.find("mac"), nic.find("target")
|
||||
nics.append([
|
||||
"" if mac is None else mac.get("address", ""),
|
||||
"" if target is None else target.get("dev", ""),
|
||||
])
|
||||
|
||||
def sample():
|
||||
return (
|
||||
if self.get_status() != 1:
|
||||
return {
|
||||
"cpu": 0,
|
||||
"disks": [{"dev": dev, "rd": 0, "wr": 0} for dev in disks],
|
||||
"nics": [{"dev": i, "rx": 0, "tx": 0} for i in range(len(nics))],
|
||||
}
|
||||
|
||||
key = [self.instance.ID(), self._host_info()[2], disks, nics]
|
||||
unread = (-1,) * 8
|
||||
|
||||
def read():
|
||||
blocks = [self.instance.blockStats(dev) for dev in disks]
|
||||
links = [self.instance.interfaceStats(target) if target else unread for _, target in nics]
|
||||
return [
|
||||
self.instance.info()[4],
|
||||
[self.instance.blockStats(dev[0]) for dev in devices],
|
||||
[self.instance.interfaceStats(dev) for dev in nics],
|
||||
)
|
||||
[[stats[1], stats[3]] for stats in blocks],
|
||||
[[stats[0], stats[4]] for stats in links],
|
||||
time.monotonic(),
|
||||
]
|
||||
|
||||
cpu_usage = {"cpu": 0}
|
||||
dev_usage = [{"dev": dev[1], "rd": 0, "wr": 0} for dev in devices]
|
||||
if not running:
|
||||
net_usage = [{"dev": i, "rx": 0, "tx": 0} for i, _ in enumerate(self.get_net_devices())]
|
||||
return cpu_usage, dev_usage, net_usage
|
||||
def rates(before, after):
|
||||
seconds = after[3] - before[3]
|
||||
if before[0] > after[0] or not 0 < seconds <= USAGE_MAX_WINDOW:
|
||||
return None
|
||||
return {
|
||||
"cpu": 100 * (after[0] - before[0]) / (seconds * key[1] * 10**9),
|
||||
"disks": [
|
||||
{"dev": dev, "rd": _rate(old[0], new[0], seconds), "wr": _rate(old[1], new[1], seconds)}
|
||||
for dev, old, new in zip(disks, before[1], after[1])
|
||||
],
|
||||
"nics": [
|
||||
{"dev": i, "rx": _rate(old[0], new[0], seconds, 8), "tx": _rate(old[1], new[1], seconds, 8)}
|
||||
for i, (old, new) in enumerate(zip(before[2], after[2]))
|
||||
],
|
||||
"window": seconds,
|
||||
}
|
||||
|
||||
nbcore = self.wvm.getInfo()[2]
|
||||
cpu_ago, blk_ago, net_ago = sample()
|
||||
time.sleep(1)
|
||||
cpu_now, blk_now, net_now = sample()
|
||||
cpu_usage["cpu"] = 100 * (cpu_now - cpu_ago) / (1 * nbcore * 10**9)
|
||||
for usage, ago, now in zip(dev_usage, blk_ago, blk_now):
|
||||
usage["rd"] = now[1] - ago[1]
|
||||
usage["wr"] = now[3] - ago[3]
|
||||
net_usage = [
|
||||
{"dev": i, "rx": (now[0] - ago[0]) * 8, "tx": (now[4] - ago[4]) * 8}
|
||||
for i, (ago, now) in enumerate(zip(net_ago, net_now))
|
||||
]
|
||||
return cpu_usage, dev_usage, net_usage
|
||||
counters = read()
|
||||
usage = None
|
||||
if previous is not None and previous.get("key") == key:
|
||||
usage = rates(previous["counters"], counters)
|
||||
if usage is None:
|
||||
before = counters
|
||||
time.sleep(1)
|
||||
counters = read()
|
||||
# None everywhere if the VM restarted meanwhile
|
||||
usage = rates(before, counters) or {
|
||||
"cpu": None,
|
||||
"disks": [{"dev": dev, "rd": None, "wr": None} for dev in disks],
|
||||
"nics": [{"dev": i, "rx": None, "tx": None} for i in range(len(nics))],
|
||||
"window": None,
|
||||
}
|
||||
usage["sample"] = {"key": key, "counters": counters}
|
||||
return usage
|
||||
|
||||
def get_telnet_port(self):
|
||||
telnet_port = None
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
"""VM statistics sample CPU, disks and NICs over one shared second: the stats
|
||||
view runs in a sync worker, polled every 10 s by each open VM page."""
|
||||
"""VM statistics: rates are averages since the sample of the previous answer,
|
||||
without waiting, when it is from the same run of the VM on the same host and
|
||||
devices; otherwise over one second. The stats view runs in a sync worker,
|
||||
polled every 5 s by each open VM page."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
|
@ -9,69 +11,168 @@ from django.conf import settings
|
|||
if not settings.configured:
|
||||
settings.configure(MAC_OUI="52:54:10")
|
||||
|
||||
from vrtManager.instance import wvmInstance
|
||||
from vrtManager.instance import USAGE_MAX_WINDOW, wvmInstance
|
||||
|
||||
XML = """<domain><devices>
|
||||
<disk type='file' device='disk'><source file='/var/lib/libvirt/images/a.qcow2'/><target dev='vda'/></disk>
|
||||
<disk type='file' device='disk'><source file='/var/lib/libvirt/images/b.qcow2'/><target dev='vdb'/></disk>
|
||||
<disk type='volume' device='disk'><source pool='default' volume='b.qcow2'/><target dev='vdb'/></disk>
|
||||
<disk type='file' device='cdrom'><target dev='sda'/></disk>
|
||||
<interface type='network'><target dev='vnet0'/></interface>
|
||||
<interface type='hostdev'><mac address='52:54:00:00:00:01'/></interface>
|
||||
<interface type='network'><mac address='52:54:00:00:00:02'/><target dev='vnet0'/></interface>
|
||||
</devices></domain>"""
|
||||
|
||||
|
||||
class FakeDomain:
|
||||
"""Counters grow by a fixed amount per elapsed second (per sleep)."""
|
||||
"""Counters grow by a fixed amount per second of the clock."""
|
||||
|
||||
def __init__(self, state):
|
||||
def __init__(self, state, clock):
|
||||
self.state = state
|
||||
self.second = 0
|
||||
self.clock = clock
|
||||
self.domain_id = 7
|
||||
self.unsupported = False
|
||||
|
||||
def now(self):
|
||||
return self.clock[0]
|
||||
|
||||
def ID(self):
|
||||
return self.domain_id
|
||||
|
||||
def info(self):
|
||||
return [self.state, 0, 0, 2, self.second * 500_000_000] # 0.5 s of CPU time per second
|
||||
return [self.state, 0, 0, 2, int(self.now() * 500_000_000)] # 0.5 s of CPU time per second
|
||||
|
||||
def blockStats(self, dev):
|
||||
n = {"vda": 1, "vdb": 2}[dev] # disks are read by target name
|
||||
return (0, self.second * n * 1000, 0, self.second * n * 3000, 0)
|
||||
written = -1 if self.unsupported and dev == "vdb" else int(self.now() * n * 3000)
|
||||
return (0, int(self.now() * n * 1000), 0, written, 0)
|
||||
|
||||
def interfaceStats(self, dev):
|
||||
return (self.second * 100, 0, 0, 0, self.second * 50, 0, 0, 0)
|
||||
assert dev == "vnet0"
|
||||
return (int(self.now() * 100), 0, 0, 0, int(self.now() * 50), 0, 0, 0)
|
||||
|
||||
|
||||
class FakeConn:
|
||||
def getInfo(self):
|
||||
return ["x86_64", 0, 4] # 4 host CPUs
|
||||
|
||||
|
||||
def instance(state):
|
||||
def instance(state, clock):
|
||||
proxy = wvmInstance.__new__(wvmInstance) # no connection: only these calls
|
||||
proxy.instance = FakeDomain(state)
|
||||
proxy.wvm = FakeConn()
|
||||
proxy._read_cache = None
|
||||
proxy.instance = FakeDomain(state, clock)
|
||||
proxy._XMLDesc = lambda flags: XML
|
||||
proxy.get_net_devices = lambda: [{"mac": "52:54:00:00:00:01"}]
|
||||
proxy._host_info = lambda: ["x86_64", 0, 4] # 4 host CPUs
|
||||
proxy.get_status = lambda: state
|
||||
return proxy
|
||||
|
||||
|
||||
class UsageTestCase(unittest.TestCase):
|
||||
def sample(self, state):
|
||||
proxy = instance(state)
|
||||
def setUp(self):
|
||||
self.clock = [100.0]
|
||||
patcher = patch("vrtManager.instance.time")
|
||||
self.addCleanup(patcher.stop)
|
||||
self.time = patcher.start()
|
||||
self.time.monotonic.side_effect = lambda: self.clock[0]
|
||||
self.time.sleep.side_effect = self.advance
|
||||
|
||||
def sleep(seconds):
|
||||
proxy.instance.second += seconds
|
||||
def advance(self, seconds):
|
||||
self.clock[0] += seconds
|
||||
|
||||
with patch("vrtManager.instance.time.sleep", side_effect=sleep) as slept:
|
||||
result = proxy.usage()
|
||||
return result, slept
|
||||
def usage(self, proxy, previous=None):
|
||||
result = proxy.usage(previous)
|
||||
self.time.sleep.reset_mock()
|
||||
return result
|
||||
|
||||
def test_a_running_vm_is_sampled_over_one_second(self):
|
||||
(cpu, disks, nets), slept = self.sample(1)
|
||||
slept.assert_called_once_with(1)
|
||||
self.assertEqual(cpu, {"cpu": 12.5}) # 0.5 s of 4 host CPUs
|
||||
self.assertEqual(disks, [{"dev": "vda", "rd": 1000, "wr": 3000}, {"dev": "vdb", "rd": 2000, "wr": 6000}])
|
||||
self.assertEqual(nets, [{"dev": 0, "rx": 800, "tx": 400}]) # bits
|
||||
def test_without_a_sample_over_one_second(self):
|
||||
proxy = instance(1, self.clock)
|
||||
result = proxy.usage()
|
||||
self.time.sleep.assert_called_once_with(1)
|
||||
self.assertEqual((result["cpu"], result["window"]), (12.5, 1.0)) # 0.5 s of 4 host CPUs
|
||||
# a pool volume is measured like a file
|
||||
self.assertEqual(
|
||||
result["disks"], [{"dev": "vda", "rd": 1000, "wr": 3000}, {"dev": "vdb", "rd": 2000, "wr": 6000}]
|
||||
)
|
||||
# bits; the NIC without a target is not measured and keeps its place
|
||||
self.assertEqual(result["nics"], [{"dev": 0, "rx": None, "tx": None}, {"dev": 1, "rx": 800, "tx": 400}])
|
||||
self.assertEqual(result["sample"]["counters"][3], 101.0)
|
||||
|
||||
def test_a_vm_that_is_not_running_is_not_sampled(self):
|
||||
(cpu, disks, nets), slept = self.sample(5)
|
||||
slept.assert_not_called()
|
||||
self.assertEqual(cpu, {"cpu": 0})
|
||||
self.assertEqual(disks, [{"dev": "vda", "rd": 0, "wr": 0}, {"dev": "vdb", "rd": 0, "wr": 0}])
|
||||
self.assertEqual(nets, [{"dev": 0, "rx": 0, "tx": 0}])
|
||||
def test_since_the_previous_sample_without_waiting(self):
|
||||
proxy = instance(1, self.clock)
|
||||
sample = self.usage(proxy)["sample"]
|
||||
self.advance(9.5)
|
||||
result = proxy.usage(sample)
|
||||
self.time.sleep.assert_not_called()
|
||||
self.assertEqual((result["cpu"], result["window"]), (12.5, 9.5))
|
||||
self.assertEqual(result["disks"][1], {"dev": "vdb", "rd": 2000, "wr": 6000}) # per second
|
||||
self.assertEqual(result["nics"][1], {"dev": 1, "rx": 800, "tx": 400})
|
||||
self.assertEqual(result["sample"]["counters"][3], 110.5)
|
||||
|
||||
def test_a_sample_of_another_run_host_or_devices_is_not_used(self):
|
||||
def restarted(proxy):
|
||||
proxy.instance.domain_id = 8
|
||||
|
||||
def other_host_cpus(proxy):
|
||||
proxy._host_info = lambda: ["x86_64", 0, 8]
|
||||
|
||||
def nic_renamed(proxy):
|
||||
proxy._XMLDesc = lambda flags: XML.replace("vnet0", "vnet1")
|
||||
|
||||
for change in (restarted, other_host_cpus, nic_renamed):
|
||||
with self.subTest(change.__name__):
|
||||
proxy = instance(1, self.clock)
|
||||
sample = self.usage(proxy)["sample"]
|
||||
change(proxy)
|
||||
self.advance(5)
|
||||
proxy.instance.interfaceStats = lambda dev: (0,) * 8
|
||||
self.assertEqual(proxy.usage(sample)["window"], 1.0)
|
||||
self.time.sleep.assert_called_once_with(1)
|
||||
self.time.sleep.reset_mock()
|
||||
|
||||
def test_a_sample_too_old_from_the_future_or_counting_back_is_not_used(self):
|
||||
for name, seconds in (("too old", USAGE_MAX_WINDOW + 1), ("future", -5), ("same time", 0)):
|
||||
with self.subTest(name):
|
||||
proxy = instance(1, self.clock)
|
||||
sample = self.usage(proxy)["sample"]
|
||||
self.advance(seconds)
|
||||
self.assertEqual(proxy.usage(sample)["window"], 1.0)
|
||||
self.time.sleep.assert_called_once_with(1)
|
||||
self.time.sleep.reset_mock()
|
||||
proxy = instance(1, self.clock)
|
||||
sample = self.usage(proxy)["sample"]
|
||||
sample["counters"][0] += 10**12 # CPU time went back: a restart with the same domain ID
|
||||
self.advance(5)
|
||||
self.assertEqual(proxy.usage(sample)["cpu"], 12.5)
|
||||
self.time.sleep.assert_called_once_with(1)
|
||||
|
||||
def test_a_counter_the_host_does_not_report_has_no_value(self):
|
||||
proxy = instance(1, self.clock)
|
||||
proxy.instance.unsupported = True
|
||||
disks = proxy.usage()["disks"]
|
||||
self.assertEqual(disks[1], {"dev": "vdb", "rd": 2000, "wr": None})
|
||||
|
||||
def test_a_vm_restarted_during_the_second_has_no_values(self):
|
||||
proxy = instance(1, self.clock)
|
||||
domain = proxy.instance
|
||||
real_sleep = self.time.sleep.side_effect
|
||||
|
||||
def restart(seconds):
|
||||
real_sleep(seconds)
|
||||
domain.info = lambda: [1, 0, 0, 2, 0] # CPU time starts over
|
||||
|
||||
self.time.sleep.side_effect = restart
|
||||
result = proxy.usage()
|
||||
self.assertEqual((result["cpu"], result["window"]), (None, None))
|
||||
self.assertEqual(result["disks"][0], {"dev": "vda", "rd": None, "wr": None})
|
||||
self.assertIn("sample", result)
|
||||
|
||||
def test_a_paused_or_shut_off_vm_is_not_sampled(self):
|
||||
for state in (3, 5):
|
||||
with self.subTest(state=state):
|
||||
result = instance(state, self.clock).usage()
|
||||
self.time.sleep.assert_not_called()
|
||||
self.assertEqual(
|
||||
result,
|
||||
{
|
||||
"cpu": 0,
|
||||
"disks": [{"dev": "vda", "rd": 0, "wr": 0}, {"dev": "vdb", "rd": 0, "wr": 0}],
|
||||
"nics": [{"dev": 0, "rx": 0, "tx": 0}, {"dev": 1, "rx": 0, "tx": 0}],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue