perf(instances): VM statistics sample all devices in one second

Each open VM page asks for its statistics every 10 seconds. For a
running VM the stats view measured the CPU, then each disk, then each
NIC, sleeping one second for each. With two disks and a NIC a request
held a sync gunicorn worker for over 4 s, so a few dozen open VM pages
could keep every worker busy.

wvmInstance.usage() replaces cpu_usage, disk_usage and net_usage, the
stats view's only callers: it reads all counters, sleeps one second
once and reads them again. Values and units are unchanged (CPU percent
of the host, disk bytes, network bits per second); compared against the
previous methods on the same counters, the results are identical.

Measured on the test host for a running VM with two disks and a NIC:
the stats request took 4.1-4.3 s before, 1.1-1.3 s now. The new test
fails on the previous code.

instances/views.py

@@ -236,10 +236,8 @@ def stats(request, pk):
     json_net = []
 
     # TODO: stats are inaccurate
-    cpu_usage = instance.proxy.cpu_usage()
+    cpu_usage, blk_usage, net_usage = instance.proxy.usage()
     mem_usage = instance.proxy.mem_usage()
-    blk_usage = instance.proxy.disk_usage()
-    net_usage = instance.proxy.net_usage()
 
     current_time = time.strftime("%H:%M:%S")
     for blk in blk_usage:

vrtManager/instance.py

@@ -944,19 +944,6 @@ class wvmInstance(wvmConnect):
 
         self._defineXML(etree.tostring(tree).decode())
 
-    def cpu_usage(self):
-        cpu_usage = {}
-        if self.get_status() == 1:
-            nbcore = self.wvm.getInfo()[2]
-            cpu_use_ago = self.instance.info()[4]
-            time.sleep(1)
-            cpu_use_now = self.instance.info()[4]
-            diff_usage = cpu_use_now - cpu_use_ago
-            cpu_usage["cpu"] = 100 * diff_usage / (1 * nbcore * 10**9)
-        else:
-            cpu_usage["cpu"] = 0
-        return cpu_usage
-
     def set_vcpu(self, cpu_id, enabled):
         self.instance.setVcpu(str(cpu_id), enabled)
 
@@ -1005,9 +992,14 @@ class wvmInstance(wvmConnect):
             mem_usage["total"] = 0
         return mem_usage
 
-    def disk_usage(self):
+    def usage(self):
+        """
+        CPU (percent of the host), disk read/write bytes and network rx/tx
+        bits of the last second, for the stats view. All counters are read
+        before and after one shared second: the view runs in a web worker,
+        and a second per device kept it busy for 1 + disks + NICs seconds.
+        """
         devices = []
-        dev_usage = []
         tree = ElementTree.fromstring(self._XMLDesc(0))
         for disk in tree.findall("devices/disk"):
             if disk.get("device") == "disk":
@@ -1029,41 +1021,39 @@ class wvmInstance(wvmConnect):
                     if network_disk:
                         dev_file = dev_bus
                     devices.append([dev_file, dev_bus])
-        for dev in devices:
-            if self.get_status() == 1:
-                rd_use_ago = self.instance.blockStats(dev[0])[1]
-                wr_use_ago = self.instance.blockStats(dev[0])[3]
-                time.sleep(1)
-                rd_use_now = self.instance.blockStats(dev[0])[1]
-                wr_use_now = self.instance.blockStats(dev[0])[3]
-                rd_diff_usage = rd_use_now - rd_use_ago
-                wr_diff_usage = wr_use_now - wr_use_ago
-            else:
-                rd_diff_usage = 0
-                wr_diff_usage = 0
-            dev_usage.append({"dev": dev[1], "rd": rd_diff_usage, "wr": wr_diff_usage})
-        return dev_usage
-
-    def net_usage(self):
-        devices = []
-        dev_usage = []
-        if self.get_status() == 1:
+        nics = []
+        running = self.get_status() == 1
+        if running:
             tree = ElementTree.fromstring(self._XMLDesc(0))
             for target in tree.findall("devices/interface/target"):
-                devices.append(target.get("dev"))
-            for i, dev in enumerate(devices):
-                rx_use_ago = self.instance.interfaceStats(dev)[0]
-                tx_use_ago = self.instance.interfaceStats(dev)[4]
-                time.sleep(1)
-                rx_use_now = self.instance.interfaceStats(dev)[0]
-                tx_use_now = self.instance.interfaceStats(dev)[4]
-                rx_diff_usage = (rx_use_now - rx_use_ago) * 8
-                tx_diff_usage = (tx_use_now - tx_use_ago) * 8
-                dev_usage.append({"dev": i, "rx": rx_diff_usage, "tx": tx_diff_usage})
-        else:
-            for i, dev in enumerate(self.get_net_devices()):
-                dev_usage.append({"dev": i, "rx": 0, "tx": 0})
-        return dev_usage
+                nics.append(target.get("dev"))
+
+        def sample():
+            return (
+                self.instance.info()[4],
+                [self.instance.blockStats(dev[0]) for dev in devices],
+                [self.instance.interfaceStats(dev) for dev in nics],
+            )
+
+        cpu_usage = {"cpu": 0}
+        dev_usage = [{"dev": dev[1], "rd": 0, "wr": 0} for dev in devices]
+        if not running:
+            net_usage = [{"dev": i, "rx": 0, "tx": 0} for i, _ in enumerate(self.get_net_devices())]
+            return cpu_usage, dev_usage, net_usage
+
+        nbcore = self.wvm.getInfo()[2]
+        cpu_ago, blk_ago, net_ago = sample()
+        time.sleep(1)
+        cpu_now, blk_now, net_now = sample()
+        cpu_usage["cpu"] = 100 * (cpu_now - cpu_ago) / (1 * nbcore * 10**9)
+        for usage, ago, now in zip(dev_usage, blk_ago, blk_now):
+            usage["rd"] = now[1] - ago[1]
+            usage["wr"] = now[3] - ago[3]
+        net_usage = [
+            {"dev": i, "rx": (now[0] - ago[0]) * 8, "tx": (now[4] - ago[4]) * 8}
+            for i, (ago, now) in enumerate(zip(net_ago, net_now))
+        ]
+        return cpu_usage, dev_usage, net_usage
 
     def get_telnet_port(self):
         telnet_port = None

vrtManager/test_usage.py

@@ -0,0 +1,77 @@
+"""VM statistics sample CPU, disks and NICs over one shared second: the stats
+view runs in a sync worker, polled every 10 s by each open VM page."""
+
+import unittest
+from unittest.mock import patch
+
+from django.conf import settings
+
+if not settings.configured:
+    settings.configure(MAC_OUI="52:54:10")
+
+from vrtManager.instance import wvmInstance
+
+XML = """<domain><devices>
+<disk type='file' device='disk'><source file='/var/lib/libvirt/images/a.qcow2'/><target dev='vda'/></disk>
+<disk type='file' device='disk'><source file='/var/lib/libvirt/images/b.qcow2'/><target dev='vdb'/></disk>
+<disk type='file' device='cdrom'><target dev='sda'/></disk>
+<interface type='network'><target dev='vnet0'/></interface>
+</devices></domain>"""
+
+
+class FakeDomain:
+    """Counters grow by a fixed amount per elapsed second (per sleep)."""
+
+    def __init__(self, state):
+        self.state = state
+        self.second = 0
+
+    def info(self):
+        return [self.state, 0, 0, 2, self.second * 500_000_000]  # 0.5 s of CPU time per second
+
+    def blockStats(self, dev):
+        n = {"vda": 1, "vdb": 2}[dev]  # disks are read by target name
+        return (0, self.second * n * 1000, 0, self.second * n * 3000, 0)
+
+    def interfaceStats(self, dev):
+        return (self.second * 100, 0, 0, 0, self.second * 50, 0, 0, 0)
+
+
+class FakeConn:
+    def getInfo(self):
+        return ["x86_64", 0, 4]  # 4 host CPUs
+
+
+def instance(state):
+    proxy = wvmInstance.__new__(wvmInstance)  # no connection: only these calls
+    proxy.instance = FakeDomain(state)
+    proxy.wvm = FakeConn()
+    proxy._XMLDesc = lambda flags: XML
+    proxy.get_net_devices = lambda: [{"mac": "52:54:00:00:00:01"}]
+    return proxy
+
+
+class UsageTestCase(unittest.TestCase):
+    def sample(self, state):
+        proxy = instance(state)
+
+        def sleep(seconds):
+            proxy.instance.second += seconds
+
+        with patch("vrtManager.instance.time.sleep", side_effect=sleep) as slept:
+            result = proxy.usage()
+        return result, slept
+
+    def test_a_running_vm_is_sampled_over_one_second(self):
+        (cpu, disks, nets), slept = self.sample(1)
+        slept.assert_called_once_with(1)
+        self.assertEqual(cpu, {"cpu": 12.5})  # 0.5 s of 4 host CPUs
+        self.assertEqual(disks, [{"dev": "vda", "rd": 1000, "wr": 3000}, {"dev": "vdb", "rd": 2000, "wr": 6000}])
+        self.assertEqual(nets, [{"dev": 0, "rx": 800, "tx": 400}])  # bits
+
+    def test_a_vm_that_is_not_running_is_not_sampled(self):
+        (cpu, disks, nets), slept = self.sample(5)
+        slept.assert_not_called()
+        self.assertEqual(cpu, {"cpu": 0})
+        self.assertEqual(disks, [{"dev": "vda", "rd": 0, "wr": 0}, {"dev": "vdb", "rd": 0, "wr": 0}])
+        self.assertEqual(nets, [{"dev": 0, "rx": 0, "tx": 0}])