perf(instances): the VM stats no longer wait a second per poll

Each stats answer carries a signed sample of the counters, bound to the VM,
its compute and UUID. The next poll sends it back and the rates are the
averages since then; a sample from another run of the VM (domain ID),
another host CPU count or device list, or older than 30 s falls back to
one second as before. Windows are measured on the monotonic clock.

Now that a poll is cheap, the Stats tab polls every 5 s like the host
graph, the first time as soon as it opens (pollWhileVisible takes the
delay of the first request).

A counter the host reports as unsupported (-1) or that went back gives no
value instead of 0 or a jump; a NIC without a target keeps its chart, a
pool volume disk is measured, and a disk or NIC attached after the page
loaded no longer stops the other charts. Disk rates are MB/s (they were
MiB/s labelled Mb/s), network Mbit/s; the CPU tooltip names the window.

README.md

@@ -535,6 +535,7 @@ A rollback leaves the clone on a detached HEAD at the restored commit; the updat
 - The log rotates (10 MB × 5) in `data/webvirtcloud.log`.
 - A disk can be grown while its VM runs or is paused, not only when it is shut off.
 - A host's CPU graph shows the average since its previous point (normally 5 s) instead of a 1 s sample taken while a worker waits.
+- A VM's stats are polled every 5 s like the host graph, the first point as soon as the tab opens; each point is the average since the previous one, disk rates in MB/s (they were MiB/s labelled Mb/s); a counter the host does not report shows a gap instead of 0.
 - A running VM without a guest agent shows the IPv4 address the host learned by ARP (the lookup always failed before).
 - After a live migration the VM's NVRAM file is deleted on the source, when it is in libvirt's `/var/lib/libvirt/qemu/nvram` and the destination does not share that directory (a VM created there later with the same name would start with those UEFI variables).
 - gstfsd finds the guest's operating system the way libguestfs inspects it, so a root on LVM works, and it always answers (an error when it finds no or several operating systems). Update `/usr/local/bin/gstfsd` on every compute (see [Adding a Compute Node](#adding-a-compute-node), step 3).

instances/templates/instance.html

@@ -381,8 +381,13 @@
 <script src="{% static 'js/Chart.bundle.min.js' %}"></script>
 <script src="{% static 'js/poll.js' %}"></script>
 <script>
+    {% trans "average over" as average_over %}
     var stats_tab = document.querySelector('#graphics-tab');
     var stats_poll = null;
+    // the seconds each CPU point averages
+    var cpuWindows = [];
+    // the usage sample of the previous answer: the next one averages since then
+    var statsSample = null;
     // The charts are polled while the Stats tab is shown; they are made once.
     stats_tab.addEventListener('hidden.bs.tab', function (event) {
         stats_poll.stop();
@@ -436,7 +441,9 @@
                              if (label) {
                                 label += ': ';
                              }
-                             return label += tooltipItem.yLabel + ' %';
+                             label += tooltipItem.yLabel + ' %';
+                             var span = cpuWindows[tooltipItem.index];
+                             return span ? label + ' ({{ average_over|escapejs }} ' + span + ' s)' : label;
                          }
                      }
                 }
@@ -530,7 +537,7 @@
                                 suggestedmax: 100,
                                 suggestedMin: 0,
                                 callback: function(value, index, values) {
-                                    return value + ' Mb/s';
+                                    return value + ' MB/s';
                                 }
                             },
                         }],
@@ -542,7 +549,7 @@
                                  if (label) {
                                     label += ': ';
                                  }
-                                 return label += tooltipItem.yLabel + ' Mb/s';
+                                 return label += tooltipItem.yLabel + ' MB/s';
                              }
                          }
                     }
@@ -608,14 +615,21 @@
             });
         {% endfor %}
 
-        stats_poll = pollWhileVisible(10000, function () {
-            return $.getJSON("{% url 'instances:stats' instance.id %}");
+        // the first point at once, then every 5 s like the host graph
+        stats_poll = pollWhileVisible(5000, function () {
+            return $.getJSON("{% url 'instances:stats' instance.id %}", statsSample ? {sample: statsSample} : {})
+                .fail(function () {
+                    statsSample = null;
+                });
         }, function graph_usage(data) {
+            statsSample = data.sample || null;
             cpuChart.data.labels.push(data.timeline);
             cpuChart.data.datasets[0].data.push(data.cpudata);
+            cpuWindows.push(data.window);
             if (cpuChart.data.datasets[0].data.length > 10){
                 cpuChart.data.labels.shift();
                 cpuChart.data.datasets[0].data.shift();
+                cpuWindows.shift();
             }
             cpuChart.update();
 
@@ -629,32 +643,24 @@
             }
             memChart.update();
 
-            for (let j = 0; j < data.blkdata.length; j++) {
-                diskChart[data.blkdata[j].dev].data.labels.push(data.timeline);
-
-                diskChart[data.blkdata[j].dev].data.datasets[0].data.push(data.blkdata[j].data[0]);
-                diskChart[data.blkdata[j].dev].data.datasets[1].data.push(data.blkdata[j].data[1]);
-                if (diskChart[data.blkdata[j].dev].data.datasets[0].data.length > 10){
-                    diskChart[data.blkdata[j].dev].data.labels.shift();
-                    diskChart[data.blkdata[j].dev].data.datasets[0].data.shift();
-                    diskChart[data.blkdata[j].dev].data.datasets[1].data.shift();
+            // a disk or NIC added after the page loaded has no chart until a reload
+            function push(chart, values) {
+                if (!chart) {
+                    return;
                 }
-                diskChart[data.blkdata[j].dev].update();
-            }
-
-            for (let j = 0; j < data.netdata.length; j++) {
-                netChart[data.netdata[j].dev].data.labels.push(data.timeline);
-
-                netChart[data.netdata[j].dev].data.datasets[0].data.push(data.netdata[j].data[0]);
-                netChart[data.netdata[j].dev].data.datasets[1].data.push(data.netdata[j].data[1]);
-                if (netChart[data.netdata[j].dev].data.datasets[0].data.length > 10){
-                    netChart[data.netdata[j].dev].data.labels.shift();
-                    netChart[data.netdata[j].dev].data.datasets[0].data.shift();
-                    netChart[data.netdata[j].dev].data.datasets[1].data.shift();
+                chart.data.labels.push(data.timeline);
+                chart.data.datasets[0].data.push(values[0]);
+                chart.data.datasets[1].data.push(values[1]);
+                if (chart.data.datasets[0].data.length > 10){
+                    chart.data.labels.shift();
+                    chart.data.datasets[0].data.shift();
+                    chart.data.datasets[1].data.shift();
                 }
-                netChart[data.netdata[j].dev].update();
+                chart.update();
             }
-        });
+            data.blkdata.forEach(function (disk) { push(diskChart[disk.dev], disk.data); });
+            data.netdata.forEach(function (nic) { push(netChart[nic.dev], nic.data); });
+        }, 0);
     });
 </script>
 <script>

instances/test_stats_sample.py

@@ -0,0 +1,194 @@
+"""The VM stats: each answer carries a signed usage sample bound to the VM on
+its compute; the next poll sends it back so that the rates are averages since
+then, without waiting."""
+
+import json
+import re
+import shutil
+import subprocess
+import unittest
+from unittest.mock import patch
+
+import libvirt
+from computes.models import Compute
+from django.contrib.auth import get_user_model
+from django.core import signing
+from django.test import TestCase
+from django.urls import reverse
+from instances.models import Instance
+from vrtManager.instance import USAGE_MAX_WINDOW
+
+SAMPLE = {"key": [7, 4, ["vda"], [["52:54:00:00:00:01", "vnet0"]]], "counters": [1, [[2, 3]], [[4, 5]], 100.0]}
+
+
+class StatsViewTestCase(TestCase):
+    def setUp(self):
+        self.compute = Compute.objects.create(name="stats", hostname="192.0.2.1", login="u", password="p", type=1)
+        self.other_compute = Compute.objects.create(name="stats2", hostname="192.0.2.2", login="u", password="p", type=1)
+        uuid = "5d1b8a3e-0000-4000-8000-000000000001"
+        self.vm = Instance.objects.create(compute=self.compute, name="stats-vm", uuid=uuid)
+        self.other = Instance.objects.create(compute=self.compute, name="stats-vm2", uuid=uuid.replace("1", "2"))
+        self.client.force_login(get_user_model().objects.create_superuser("sv-admin", "sv@example.com", "pw"))
+        patcher = patch("instances.models.wvmInstance")
+        self.addCleanup(patcher.stop)
+        self.proxy = patcher.start().return_value
+        self.proxy.mem_usage.return_value = {"used": 1, "total": 2}
+        self.proxy.usage.return_value = {
+            "cpu": 12.345,
+            "disks": [{"dev": "vda", "rd": 2_000_000, "wr": None}],
+            "nics": [{"dev": 0, "rx": 8_000_000, "tx": 0}],
+            "window": 9.96,
+            "sample": SAMPLE,
+        }
+
+    def stats(self, vm=None, **params):
+        response = self.client.get(reverse("instances:stats", args=[(vm or self.vm).id]), params)
+        self.assertEqual(response.status_code, 200)
+        return json.loads(response.content)
+
+    def previous(self):
+        return self.proxy.usage.call_args.args[0]
+
+    def test_rates_in_mb_and_mbit_per_second_and_the_sample_round_trip(self):
+        data = self.stats()
+        self.assertIsNone(self.previous())
+        self.assertEqual((data["cpudata"], data["window"]), (12.3, 10.0))
+        self.assertEqual(data["blkdata"], [{"dev": "vda", "data": [2.0, None]}])
+        self.assertEqual(data["netdata"], [{"dev": 0, "data": [8.0, 0.0]}])
+        self.stats(sample=data["sample"])
+        self.assertEqual(self.previous(), SAMPLE)
+
+    def test_an_altered_old_or_foreign_sample_is_not_used(self):
+        token = self.stats()["sample"]
+        foreign = self.stats(self.other)["sample"]
+        altered = token[:-2] + ("xx" if not token.endswith("xx") else "yy")
+        for name, sample in (("altered", altered), ("garbage", "x"), ("other VM", foreign)):
+            with self.subTest(name):
+                self.stats(sample=sample)
+                self.assertIsNone(self.previous())
+        with patch("django.core.signing.time.time", return_value=signing.time.time() + USAGE_MAX_WINDOW + 1):
+            self.stats(sample=token)
+        self.assertIsNone(self.previous())
+
+    def test_a_sample_from_before_a_migration_is_not_used(self):
+        token = self.stats()["sample"]
+        Instance.objects.filter(pk=self.vm.pk).update(compute=self.other_compute)
+        self.stats(sample=token)
+        self.assertIsNone(self.previous())
+
+    def test_a_vm_that_is_not_running_gives_no_sample(self):
+        self.proxy.usage.return_value = {"cpu": 0, "disks": [], "nics": []}
+        data = self.stats()
+        self.assertEqual(data["cpudata"], 0)
+        self.assertNotIn("sample", data)
+        self.assertNotIn("window", data)
+
+
+# a VM with a disk and a NIC, on libvirt's test driver
+DOMAIN = """<domain type='test'><name>stats-script-vm</name><uuid>3c0e2b7a-91d4-4f5e-8a6b-0d2f4e6a8c10</uuid>
+<memory unit='MiB'>128</memory><vcpu>1</vcpu><os><type arch='i686'>hvm</type></os>
+<devices><disk type='file' device='disk'><source file='/default-pool/stats-script.img'/><target dev='vda' bus='virtio'/></disk>
+<interface type='network'><mac address='52:54:00:00:00:31'/><source network='default'/><model type='virtio'/></interface></devices>
+</domain>"""
+
+
+class TestDriverVM:
+    def setUp(self):
+        self.conn = libvirt.open("test:///default")
+        self.addCleanup(self.conn.close)
+        # the test host lives on while a libvirt object still refers to it
+        pool = self.conn.storagePoolLookupByName("default-pool")
+        if "stats-script.img" not in pool.listVolumes():
+            pool.createXML("<volume><name>stats-script.img</name><capacity>1048576</capacity></volume>", 0)
+        self.domain = self.conn.defineXML(DOMAIN)
+        compute = Compute.objects.create(name="ss", hostname="localhost", login="", password="", type=4)
+        self.vm = Instance.objects.create(compute=compute, name=self.domain.name(), uuid=self.domain.UUIDString())
+        self.client.force_login(get_user_model().objects.create_superuser("ss-admin", "ss@example.com", "pw"))
+        for target, value in (
+            ("vrtManager.connection.connection_manager.get_connection", self.conn),
+            ("vrtManager.connection.wvmConnect.get_machine_types", ["pc"]),
+            ("vrtManager.connection.wvmConnect.get_nwfilters", []),
+        ):
+            patcher = patch(target, return_value=value)
+            patcher.start()
+            self.addCleanup(patcher.stop)
+
+
+class SampleRoundTripTestCase(TestDriverVM, TestCase):
+    def test_a_real_sample_is_used_by_the_next_poll(self):
+        self.domain.create()
+        self.addCleanup(self.domain.destroy)
+        url = reverse("instances:stats", args=[self.vm.id])
+        with patch("vrtManager.instance.time.sleep") as sleep:
+            first = json.loads(self.client.get(url).content)
+            sleep.assert_called_once_with(1)
+            sleep.reset_mock()
+            second = json.loads(self.client.get(url, {"sample": first["sample"]}).content)
+        sleep.assert_not_called()
+        self.assertEqual([disk["dev"] for disk in second["blkdata"]], ["vda"])
+        self.assertIn("sample", second)
+
+
+@unittest.skipUnless(shutil.which("node"), "node is not installed")
+class StatsScriptTestCase(TestDriverVM, TestCase):
+    """The VM page's script sends each answer's sample with the next poll and
+    drops it after a failed one."""
+
+    def page(self):
+        response = self.client.get(reverse("instances:instance", args=[self.vm.id]))
+        self.assertEqual(response.status_code, 200)
+        return response.content.decode()
+
+    def test_the_next_poll_sends_the_previous_sample(self):
+        script = next(s for s in re.findall(r"<script>(.*?)</script>", self.page(), re.S) if "statsSample" in s)
+        js = """
+        const vm = require("vm");
+        const charts = [], requests = [], handlers = {};
+        let poll, fail = false;
+        const canvas = {get: () => ({getContext: () => ({})})};
+        const $ = () => canvas;
+        $.getJSON = (url, params) => {
+            requests.push(params);
+            return {fail: (callback) => { if (fail) callback(); return {}; }};
+        };
+        const page = {
+            $,
+            document: {querySelector: () => ({addEventListener: (name, handler) => { handlers[name] = handler; }})},
+            // Chart.js adds the lists a config leaves out
+            Chart: function (ctx, config) {
+                this.data = Object.assign({labels: []}, config.data);
+                this.data.datasets.forEach((set) => { set.data = set.data || []; });
+                this.options = config.options;
+                this.update = () => {};
+                charts.push(this);
+            },
+            pollWhileVisible: (interval, request, onData, first) => { poll = {interval, first, request, onData}; return {}; },
+        };
+        vm.runInNewContext(SCRIPT, vm.createContext(page));
+        handlers["shown.bs.tab"]();
+        const answer = (cpu, sample) => ({cpudata: cpu, window: 9.9, sample: sample, timeline: "t",
+                                          memdata: {used: 0, total: 1024},
+                                          // vdb was attached after the page loaded
+                                          blkdata: [{dev: "vdb", data: [9, 9]}, {dev: "vda", data: [1.5, null]}],
+                                          netdata: [{dev: 0, data: [2, 3]}]});
+        poll.request();
+        poll.onData(answer(3, "token-1"));
+        poll.request();
+        fail = true;
+        poll.request();
+        fail = false;
+        poll.request();
+        const label = charts[0].options.tooltips.callbacks.label({datasetIndex: 0, index: 0, yLabel: 3}, charts[0].data);
+        console.log(JSON.stringify({requests, label, every: [poll.interval, poll.first], cpu: charts[0].data.datasets[0].data,
+                                    disk: charts[2].data.datasets[1].data, net: charts[3].data.datasets[0].data}));
+        """.replace("SCRIPT", json.dumps(script))
+        out = subprocess.run(["node", "-e", js], capture_output=True, text=True, timeout=30)
+        self.assertEqual(out.returncode, 0, out.stderr)
+        result = json.loads(out.stdout)
+        # the first poll at once, then every 5 s
+        self.assertEqual(result["every"], [5000, 0])
+        # the sample goes with the next poll; a failed poll drops it
+        self.assertEqual(result["requests"], [{}, {"sample": "token-1"}, {"sample": "token-1"}, {}])
+        # a disk without a chart is skipped, the other charts still update
+        self.assertEqual((result["cpu"], result["disk"], result["net"]), ([3], [None], [2]))
+        self.assertTrue(result["label"].endswith(": 3 % (average over 9.9 s)"), result["label"])

instances/views.py

@@ -22,6 +22,7 @@ from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import permission_required
 from django.contrib.auth.models import User
+from django.core import signing
 from django.core.exceptions import PermissionDenied, ValidationError
 from django.http import Http404, HttpResponse, JsonResponse
 from django.db.models import prefetch_related_objects
@@ -41,7 +42,7 @@ from logs.views import addlogmsg
 from webvirtcloud.middleware import error_text
 from vrtManager import util
 from vrtManager.create import wvmCreate
-from vrtManager.instance import wvmInstances
+from vrtManager.instance import USAGE_MAX_WINDOW, wvmInstances
 from vrtManager.interface import wvmInterface
 from vrtManager.storage import wvmStorage
 from vrtManager.util import randomPasswd
@@ -54,6 +55,8 @@ from .models import Flavor
 
 # the VM page waits this long (seconds) for the compute's drbdadm status
 DRBD_STATUS_TIMEOUT = 10
+# the stats tab sends the sample of each answer back with its next poll
+STATS_SAMPLE_SIGNER = signing.TimestampSigner(salt="instances.stats-sample")
 
 
 def index(request):
@@ -269,41 +272,42 @@ def drbd_status(request, pk):
     return result
 
 
+def _stats_sample(instance, token):
+    """The usage sample in a token of this VM's stats on its current compute,
+    or None when there is none, it is altered or too old."""
+    if not token:
+        return None
+    try:
+        data = STATS_SAMPLE_SIGNER.unsign_object(token, max_age=USAGE_MAX_WINDOW)
+    except signing.BadSignature:
+        return None
+    if data.get("vm") != [instance.id, instance.compute_id, instance.uuid]:
+        return None
+    return data["sample"]
+
+
 def stats(request, pk):
     instance = get_instance(request.user, pk)
-    json_blk = []
-    json_net = []
-
-    # TODO: stats are inaccurate
-    cpu_usage, blk_usage, net_usage = instance.proxy.usage()
+    usage = instance.proxy.usage(_stats_sample(instance, request.GET.get("sample")))
     mem_usage = instance.proxy.mem_usage()
 
-    current_time = time.strftime("%H:%M:%S")
-    for blk in blk_usage:
-        json_blk.append(
-            {
-                "dev": blk["dev"],
-                "data": [int(blk["rd"]) / 1048576, int(blk["wr"]) / 1048576],
-            }
-        )
+    def mega(value):
+        return None if value is None else value / 10**6
 
-    for net in net_usage:
-        json_net.append(
-            {
-                "dev": net["dev"],
-                "data": [int(net["rx"]) / 1048576, int(net["tx"]) / 1048576],
-            }
+    result = {
+        "cpudata": None if usage["cpu"] is None else round(usage["cpu"], 1),
+        "memdata": mem_usage,
+        # MB/s and Mbit/s
+        "blkdata": [{"dev": disk["dev"], "data": [mega(disk["rd"]), mega(disk["wr"])]} for disk in usage["disks"]],
+        "netdata": [{"dev": nic["dev"], "data": [mega(nic["rx"]), mega(nic["tx"])]} for nic in usage["nics"]],
+        "timeline": time.strftime("%H:%M:%S"),
+    }
+    if "sample" in usage:
+        result["window"] = None if usage["window"] is None else round(usage["window"], 1)
+        result["sample"] = STATS_SAMPLE_SIGNER.sign_object(
+            {"vm": [instance.id, instance.compute_id, instance.uuid], "sample": usage["sample"]}, compress=True
         )
-
-    return JsonResponse(
-        {
-            "cpudata": int(cpu_usage["cpu"]),
-            "memdata": mem_usage,
-            "blkdata": json_blk,
-            "netdata": json_net,
-            "timeline": current_time,
-        }
-    )
+    return JsonResponse(result)
 
 
 def osinfo(request, pk):

static/js/poll.js

@@ -5,8 +5,9 @@
 // interval ms after the previous one has finished, so a slow host does not
 // pile up requests, each holding a web worker. A hidden page asks nothing;
 // when it is shown again it asks at once. stop() pauses, start() resumes
-// at once (for example while a tab of the page is shown).
-function pollWhileVisible(interval, request, onData) {
+// at once (for example while a tab of the page is shown). The first request
+// goes out after firstDelay ms, the interval if it is not given.
+function pollWhileVisible(interval, request, onData, firstDelay) {
     var timer = null;
     var running = false;
     var stopped = false;
@@ -34,7 +35,7 @@ function pollWhileVisible(interval, request, onData) {
     }
 
     document.addEventListener("visibilitychange", resume);
-    timer = setTimeout(run, interval);
+    timer = setTimeout(run, firstDelay === undefined ? interval : firstDelay);
 
     return {
         stop: function () {

vrtManager/instance.py

@@ -60,6 +60,16 @@ from vrtManager.storage import wvmStorage, wvmStorages
 # with secrets. Live XML would drop pending changes, non-secure XML the VNC
 # password.
 PERSISTENT_XML = VIR_DOMAIN_XML_INACTIVE | VIR_DOMAIN_XML_SECURE
+# the longest window wvmInstance.usage() averages over
+USAGE_MAX_WINDOW = 30
+
+
+def _rate(before, after, seconds, scale=1):
+    """Growth per second of a counter, or None when a reading is unsupported
+    (-1) or the counter went back."""
+    if before < 0 or after < before:
+        return None
+    return scale * (after - before) / seconds
 
 
 class wvmInstances(wvmConnect):
@@ -1266,68 +1276,91 @@ class wvmInstance(wvmConnect):
             mem_usage["total"] = 0
         return mem_usage
 
-    def usage(self):
+    def usage(self, previous=None):
         """
         CPU (percent of the host), disk read/write bytes and network rx/tx
-        bits of the last second, for the stats view. All counters are read
-        before and after one shared second: the view runs in a web worker,
-        and a second per device kept it busy for 1 + disks + NICs seconds.
+        bits per second, for the stats view. With previous, the "sample" of an
+        earlier answer, they are averages since then and nothing waits; without
+        a usable one, over one second. A sample is usable for the same run of
+        the VM (domain ID) on a host with the same CPUs and devices, at most
+        USAGE_MAX_WINDOW seconds old. A value the host does not report (-1) or
+        whose counter went back is None.
         """
-        devices = []
         tree = ElementTree.fromstring(self._XMLDesc(0))
+        disks = []
         for disk in tree.findall("devices/disk"):
-            if disk.get("device") == "disk":
-                dev_file = None
-                dev_bus = None
-                network_disk = True
-                for elm in disk:
-                    if elm.tag == "source":
-                        if elm.get("protocol"):
-                            dev_file = elm.get("protocol")
-                            network_disk = True
-                        if elm.get("file"):
-                            dev_file = elm.get("file")
-                        if elm.get("dev"):
-                            dev_file = elm.get("dev")
-                    if elm.tag == "target":
-                        dev_bus = elm.get("dev")
-                if (dev_file and dev_bus) is not None:
-                    if network_disk:
-                        dev_file = dev_bus
-                    devices.append([dev_file, dev_bus])
+            source, target = disk.find("source"), disk.find("target")
+            if (
+                disk.get("device") == "disk"
+                and source is not None
+                and any(source.get(name) for name in ("protocol", "file", "dev", "volume"))
+                and target is not None
+                and target.get("dev")
+            ):
+                disks.append(target.get("dev"))  # blockStats takes the target name
+        # all NICs, in the order of the page's charts; one without a target is not measured
         nics = []
-        running = self.get_status() == 1
-        if running:
-            tree = ElementTree.fromstring(self._XMLDesc(0))
-            for target in tree.findall("devices/interface/target"):
-                nics.append(target.get("dev"))
+        for nic in tree.findall("devices/interface"):
+            mac, target = nic.find("mac"), nic.find("target")
+            nics.append([
+                "" if mac is None else mac.get("address", ""),
+                "" if target is None else target.get("dev", ""),
+            ])
 
-        def sample():
-            return (
+        if self.get_status() != 1:
+            return {
+                "cpu": 0,
+                "disks": [{"dev": dev, "rd": 0, "wr": 0} for dev in disks],
+                "nics": [{"dev": i, "rx": 0, "tx": 0} for i in range(len(nics))],
+            }
+
+        key = [self.instance.ID(), self._host_info()[2], disks, nics]
+        unread = (-1,) * 8
+
+        def read():
+            blocks = [self.instance.blockStats(dev) for dev in disks]
+            links = [self.instance.interfaceStats(target) if target else unread for _, target in nics]
+            return [
                 self.instance.info()[4],
-                [self.instance.blockStats(dev[0]) for dev in devices],
-                [self.instance.interfaceStats(dev) for dev in nics],
-            )
+                [[stats[1], stats[3]] for stats in blocks],
+                [[stats[0], stats[4]] for stats in links],
+                time.monotonic(),
+            ]
 
-        cpu_usage = {"cpu": 0}
-        dev_usage = [{"dev": dev[1], "rd": 0, "wr": 0} for dev in devices]
-        if not running:
-            net_usage = [{"dev": i, "rx": 0, "tx": 0} for i, _ in enumerate(self.get_net_devices())]
-            return cpu_usage, dev_usage, net_usage
+        def rates(before, after):
+            seconds = after[3] - before[3]
+            if before[0] > after[0] or not 0 < seconds <= USAGE_MAX_WINDOW:
+                return None
+            return {
+                "cpu": 100 * (after[0] - before[0]) / (seconds * key[1] * 10**9),
+                "disks": [
+                    {"dev": dev, "rd": _rate(old[0], new[0], seconds), "wr": _rate(old[1], new[1], seconds)}
+                    for dev, old, new in zip(disks, before[1], after[1])
+                ],
+                "nics": [
+                    {"dev": i, "rx": _rate(old[0], new[0], seconds, 8), "tx": _rate(old[1], new[1], seconds, 8)}
+                    for i, (old, new) in enumerate(zip(before[2], after[2]))
+                ],
+                "window": seconds,
+            }
 
-        nbcore = self.wvm.getInfo()[2]
-        cpu_ago, blk_ago, net_ago = sample()
-        time.sleep(1)
-        cpu_now, blk_now, net_now = sample()
-        cpu_usage["cpu"] = 100 * (cpu_now - cpu_ago) / (1 * nbcore * 10**9)
-        for usage, ago, now in zip(dev_usage, blk_ago, blk_now):
-            usage["rd"] = now[1] - ago[1]
-            usage["wr"] = now[3] - ago[3]
-        net_usage = [
-            {"dev": i, "rx": (now[0] - ago[0]) * 8, "tx": (now[4] - ago[4]) * 8}
-            for i, (ago, now) in enumerate(zip(net_ago, net_now))
-        ]
-        return cpu_usage, dev_usage, net_usage
+        counters = read()
+        usage = None
+        if previous is not None and previous.get("key") == key:
+            usage = rates(previous["counters"], counters)
+        if usage is None:
+            before = counters
+            time.sleep(1)
+            counters = read()
+            # None everywhere if the VM restarted meanwhile
+            usage = rates(before, counters) or {
+                "cpu": None,
+                "disks": [{"dev": dev, "rd": None, "wr": None} for dev in disks],
+                "nics": [{"dev": i, "rx": None, "tx": None} for i in range(len(nics))],
+                "window": None,
+            }
+        usage["sample"] = {"key": key, "counters": counters}
+        return usage
 
     def get_telnet_port(self):
         telnet_port = None

vrtManager/test_usage.py

@@ -1,5 +1,7 @@
-"""VM statistics sample CPU, disks and NICs over one shared second: the stats
-view runs in a sync worker, polled every 10 s by each open VM page."""
+"""VM statistics: rates are averages since the sample of the previous answer,
+without waiting, when it is from the same run of the VM on the same host and
+devices; otherwise over one second. The stats view runs in a sync worker,
+polled every 5 s by each open VM page."""
 
 import unittest
 from unittest.mock import patch
@@ -9,69 +11,168 @@ from django.conf import settings
 if not settings.configured:
     settings.configure(MAC_OUI="52:54:10")
 
-from vrtManager.instance import wvmInstance
+from vrtManager.instance import USAGE_MAX_WINDOW, wvmInstance
 
 XML = """<domain><devices>
 <disk type='file' device='disk'><source file='/var/lib/libvirt/images/a.qcow2'/><target dev='vda'/></disk>
-<disk type='file' device='disk'><source file='/var/lib/libvirt/images/b.qcow2'/><target dev='vdb'/></disk>
+<disk type='volume' device='disk'><source pool='default' volume='b.qcow2'/><target dev='vdb'/></disk>
 <disk type='file' device='cdrom'><target dev='sda'/></disk>
-<interface type='network'><target dev='vnet0'/></interface>
+<interface type='hostdev'><mac address='52:54:00:00:00:01'/></interface>
+<interface type='network'><mac address='52:54:00:00:00:02'/><target dev='vnet0'/></interface>
 </devices></domain>"""
 
 
 class FakeDomain:
-    """Counters grow by a fixed amount per elapsed second (per sleep)."""
+    """Counters grow by a fixed amount per second of the clock."""
 
-    def __init__(self, state):
+    def __init__(self, state, clock):
         self.state = state
-        self.second = 0
+        self.clock = clock
+        self.domain_id = 7
+        self.unsupported = False
+
+    def now(self):
+        return self.clock[0]
+
+    def ID(self):
+        return self.domain_id
 
     def info(self):
-        return [self.state, 0, 0, 2, self.second * 500_000_000]  # 0.5 s of CPU time per second
+        return [self.state, 0, 0, 2, int(self.now() * 500_000_000)]  # 0.5 s of CPU time per second
 
     def blockStats(self, dev):
         n = {"vda": 1, "vdb": 2}[dev]  # disks are read by target name
-        return (0, self.second * n * 1000, 0, self.second * n * 3000, 0)
+        written = -1 if self.unsupported and dev == "vdb" else int(self.now() * n * 3000)
+        return (0, int(self.now() * n * 1000), 0, written, 0)
 
     def interfaceStats(self, dev):
-        return (self.second * 100, 0, 0, 0, self.second * 50, 0, 0, 0)
+        assert dev == "vnet0"
+        return (int(self.now() * 100), 0, 0, 0, int(self.now() * 50), 0, 0, 0)
 
 
-class FakeConn:
-    def getInfo(self):
-        return ["x86_64", 0, 4]  # 4 host CPUs
-
-
-def instance(state):
+def instance(state, clock):
     proxy = wvmInstance.__new__(wvmInstance)  # no connection: only these calls
-    proxy.instance = FakeDomain(state)
-    proxy.wvm = FakeConn()
+    proxy._read_cache = None
+    proxy.instance = FakeDomain(state, clock)
     proxy._XMLDesc = lambda flags: XML
-    proxy.get_net_devices = lambda: [{"mac": "52:54:00:00:00:01"}]
+    proxy._host_info = lambda: ["x86_64", 0, 4]  # 4 host CPUs
+    proxy.get_status = lambda: state
     return proxy
 
 
 class UsageTestCase(unittest.TestCase):
-    def sample(self, state):
-        proxy = instance(state)
+    def setUp(self):
+        self.clock = [100.0]
+        patcher = patch("vrtManager.instance.time")
+        self.addCleanup(patcher.stop)
+        self.time = patcher.start()
+        self.time.monotonic.side_effect = lambda: self.clock[0]
+        self.time.sleep.side_effect = self.advance
 
-        def sleep(seconds):
-            proxy.instance.second += seconds
+    def advance(self, seconds):
+        self.clock[0] += seconds
 
-        with patch("vrtManager.instance.time.sleep", side_effect=sleep) as slept:
-            result = proxy.usage()
-        return result, slept
+    def usage(self, proxy, previous=None):
+        result = proxy.usage(previous)
+        self.time.sleep.reset_mock()
+        return result
 
-    def test_a_running_vm_is_sampled_over_one_second(self):
-        (cpu, disks, nets), slept = self.sample(1)
-        slept.assert_called_once_with(1)
-        self.assertEqual(cpu, {"cpu": 12.5})  # 0.5 s of 4 host CPUs
-        self.assertEqual(disks, [{"dev": "vda", "rd": 1000, "wr": 3000}, {"dev": "vdb", "rd": 2000, "wr": 6000}])
-        self.assertEqual(nets, [{"dev": 0, "rx": 800, "tx": 400}])  # bits
+    def test_without_a_sample_over_one_second(self):
+        proxy = instance(1, self.clock)
+        result = proxy.usage()
+        self.time.sleep.assert_called_once_with(1)
+        self.assertEqual((result["cpu"], result["window"]), (12.5, 1.0))  # 0.5 s of 4 host CPUs
+        # a pool volume is measured like a file
+        self.assertEqual(
+            result["disks"], [{"dev": "vda", "rd": 1000, "wr": 3000}, {"dev": "vdb", "rd": 2000, "wr": 6000}]
+        )
+        # bits; the NIC without a target is not measured and keeps its place
+        self.assertEqual(result["nics"], [{"dev": 0, "rx": None, "tx": None}, {"dev": 1, "rx": 800, "tx": 400}])
+        self.assertEqual(result["sample"]["counters"][3], 101.0)
 
-    def test_a_vm_that_is_not_running_is_not_sampled(self):
-        (cpu, disks, nets), slept = self.sample(5)
-        slept.assert_not_called()
-        self.assertEqual(cpu, {"cpu": 0})
-        self.assertEqual(disks, [{"dev": "vda", "rd": 0, "wr": 0}, {"dev": "vdb", "rd": 0, "wr": 0}])
-        self.assertEqual(nets, [{"dev": 0, "rx": 0, "tx": 0}])
+    def test_since_the_previous_sample_without_waiting(self):
+        proxy = instance(1, self.clock)
+        sample = self.usage(proxy)["sample"]
+        self.advance(9.5)
+        result = proxy.usage(sample)
+        self.time.sleep.assert_not_called()
+        self.assertEqual((result["cpu"], result["window"]), (12.5, 9.5))
+        self.assertEqual(result["disks"][1], {"dev": "vdb", "rd": 2000, "wr": 6000})  # per second
+        self.assertEqual(result["nics"][1], {"dev": 1, "rx": 800, "tx": 400})
+        self.assertEqual(result["sample"]["counters"][3], 110.5)
+
+    def test_a_sample_of_another_run_host_or_devices_is_not_used(self):
+        def restarted(proxy):
+            proxy.instance.domain_id = 8
+
+        def other_host_cpus(proxy):
+            proxy._host_info = lambda: ["x86_64", 0, 8]
+
+        def nic_renamed(proxy):
+            proxy._XMLDesc = lambda flags: XML.replace("vnet0", "vnet1")
+
+        for change in (restarted, other_host_cpus, nic_renamed):
+            with self.subTest(change.__name__):
+                proxy = instance(1, self.clock)
+                sample = self.usage(proxy)["sample"]
+                change(proxy)
+                self.advance(5)
+                proxy.instance.interfaceStats = lambda dev: (0,) * 8
+                self.assertEqual(proxy.usage(sample)["window"], 1.0)
+                self.time.sleep.assert_called_once_with(1)
+                self.time.sleep.reset_mock()
+
+    def test_a_sample_too_old_from_the_future_or_counting_back_is_not_used(self):
+        for name, seconds in (("too old", USAGE_MAX_WINDOW + 1), ("future", -5), ("same time", 0)):
+            with self.subTest(name):
+                proxy = instance(1, self.clock)
+                sample = self.usage(proxy)["sample"]
+                self.advance(seconds)
+                self.assertEqual(proxy.usage(sample)["window"], 1.0)
+                self.time.sleep.assert_called_once_with(1)
+                self.time.sleep.reset_mock()
+        proxy = instance(1, self.clock)
+        sample = self.usage(proxy)["sample"]
+        sample["counters"][0] += 10**12  # CPU time went back: a restart with the same domain ID
+        self.advance(5)
+        self.assertEqual(proxy.usage(sample)["cpu"], 12.5)
+        self.time.sleep.assert_called_once_with(1)
+
+    def test_a_counter_the_host_does_not_report_has_no_value(self):
+        proxy = instance(1, self.clock)
+        proxy.instance.unsupported = True
+        disks = proxy.usage()["disks"]
+        self.assertEqual(disks[1], {"dev": "vdb", "rd": 2000, "wr": None})
+
+    def test_a_vm_restarted_during_the_second_has_no_values(self):
+        proxy = instance(1, self.clock)
+        domain = proxy.instance
+        real_sleep = self.time.sleep.side_effect
+
+        def restart(seconds):
+            real_sleep(seconds)
+            domain.info = lambda: [1, 0, 0, 2, 0]  # CPU time starts over
+
+        self.time.sleep.side_effect = restart
+        result = proxy.usage()
+        self.assertEqual((result["cpu"], result["window"]), (None, None))
+        self.assertEqual(result["disks"][0], {"dev": "vda", "rd": None, "wr": None})
+        self.assertIn("sample", result)
+
+    def test_a_paused_or_shut_off_vm_is_not_sampled(self):
+        for state in (3, 5):
+            with self.subTest(state=state):
+                result = instance(state, self.clock).usage()
+                self.time.sleep.assert_not_called()
+                self.assertEqual(
+                    result,
+                    {
+                        "cpu": 0,
+                        "disks": [{"dev": "vda", "rd": 0, "wr": 0}, {"dev": "vdb", "rd": 0, "wr": 0}],
+                        "nics": [{"dev": 0, "rx": 0, "tx": 0}, {"dev": 1, "rx": 0, "tx": 0}],
+                    },
+                )
+
+
+if __name__ == "__main__":
+    unittest.main()

webvirtcloud/js_tests/poll.test.js

@@ -92,4 +92,13 @@ assert.strictEqual(answers.length, 8, "each answer reaches onData");
 failNext = true; requests[8].finish();
 assert.ok(requests[8].error, "the error reaches the page");
 advance(10000); assert.strictEqual(requests.length, 10, "polling goes on after an error in onData");
+
+// a first delay of 0: the first request at once, then every interval
+poll.stop(); requests[9].finish();
+const atOnce = pollWhileVisible(5000, request, onData, 0);
+advance(0); assert.strictEqual(requests.length, 11, "first request after firstDelay");
+requests[10].finish();
+advance(4999); assert.strictEqual(requests.length, 11);
+advance(1); assert.strictEqual(requests.length, 12, "then every interval");
+atOnce.stop();
 console.log("poll.js: all checks passed");