fix(instances): the new-VM dialog shows each host's current CPU load

The CPU bar showed the average since the host booted, read for every
compute on every load of the VM list. The dialog now reads each compute's
graph endpoint (load over one second) when it opens, two computes at a
time so the reads hold at most two workers, and ignores answers of an
earlier opening. A host that cannot be read shows a dash. The since-boot
reading had no other caller and is removed.

README.md

@@ -536,6 +536,7 @@ A rollback leaves the clone on a detached HEAD at the restored commit; the updat
 - A disk can be grown while its VM runs or is paused, not only when it is shut off.
 - Enabling vCPU hot plug keeps the VM's current vCPU count (it enabled all of them up to the maximum).
 - A host's CPU graph shows the average since its previous point (normally 5 s) instead of a 1 s sample taken while a worker waits.
+- The CPU usage in the new-VM dialog is the current load (over 1 s), read when the dialog opens; it was the average since the host booted.
 - A VM's stats are polled every 5 s like the host graph, the first point as soon as the tab opens; each point is the average since the previous one, disk rates in MB/s (they were MiB/s labelled Mb/s); a counter the host does not report shows a gap instead of 0.
 - An ISO uploads to an SSH compute about 3.5 times faster (SFTP writes no longer wait for each 32 KiB); a failed write still fails the upload.
 - The VM page shows each NIC's own QoS and each CD-ROM's own pool (they showed the previous device's), and a NIC without a source or model no longer breaks it.

computes/models.py

@@ -78,10 +78,6 @@ class Compute(Model):
     def cpu_count(self):
         return self.proxy.get_node_info()[3]
     
-    @cached_property
-    def cpu_usage(self):
-        return round(self.proxy.get_cpu_usage(diff=False).get('usage'))
-
     @cached_property
     def ram_size(self):
         return self.proxy.get_node_info()[2]

computes/test_cpu_graph.py

@@ -66,10 +66,6 @@ class CpuUsageTestCase(UnitTestCase):
         clock.sleep.assert_called_once_with(1)
         self.assertEqual((usage["usage"], usage["window"]), (80.0, 1.0))
 
-    def test_since_the_host_started(self, clock):
-        self.assertEqual(host((25, 75)).get_cpu_usage(diff=False), {"usage": 75.0})
-        clock.sleep.assert_not_called()
-
 
 class GraphTestCase(TestCase):
     def setUp(self):

instances/templates/allinstances.html

@@ -47,6 +47,49 @@
                 //let compute = $("#compute_select").val();
                 window.location.href = "{% url 'instances:create_instance_select_type' 1 %}".replace(1, compute);
             }
+
+            // The new-VM dialog's CPU bars: each compute's load over one second,
+            // read when the dialog opens, two computes at a time (each read holds
+            // a worker for that second). Answers of an earlier opening are ignored.
+            const addInstance = document.getElementById("AddInstance");
+            let cpuOpening = 0, cpuQueue = [], cpuReading = 0;
+
+            function setCpuUsage(cell, percent, text) {
+                const bar = cell.querySelector(".progress-bar");
+                bar.style.width = (percent || 0) + "%";
+                if (percent === null) {
+                    bar.removeAttribute("aria-valuenow");
+                } else {
+                    bar.setAttribute("aria-valuenow", percent);
+                }
+                cell.querySelector(".cpu-usage-value").textContent = text;
+            }
+
+            function readCpuUsage() {
+                while (cpuReading < 2 && cpuQueue.length) {
+                    const cell = cpuQueue.shift(), opening = cpuOpening;
+                    const show = (percent, text) => { if (opening === cpuOpening) setCpuUsage(cell, percent, text); };
+                    cpuReading++;
+                    $.getJSON(cell.dataset.url)
+                        .done((data) => {
+                            const usage = typeof data.cpudata === "number" ? Math.round(data.cpudata) : null;
+                            show(usage, usage === null ? "—" : usage + "%");
+                        })
+                        .fail(() => show(null, "—"))
+                        .always(() => { cpuReading--; readCpuUsage(); });
+                }
+            }
+
+            addInstance.addEventListener("show.bs.modal", function () {
+                cpuOpening++;
+                cpuQueue = Array.from(addInstance.querySelectorAll(".cpu-usage"));
+                cpuQueue.forEach((cell) => setCpuUsage(cell, null, "…"));
+                readCpuUsage();
+            });
+            addInstance.addEventListener("hide.bs.modal", function () {
+                cpuOpening++;
+                cpuQueue = [];
+            });
         </script>
     {% endif %}
 {% endblock script %}

instances/templates/create_inst_block.html

@@ -32,12 +32,13 @@
                                         <tr style="text-decoration: none">
                                             <td>{{ compute.name }}</td>
                                             <td>{{ compute.cpu_count }}</td>
-                                            <td>
+                                            <td class="cpu-usage" data-url="{% url 'compute_graph' compute.id %}">
                                                 <div class="progress">
-                                                    <div class="progress-bar bg-primary" role="progressbar" style="width: {{ compute.cpu_usage }}%"
-                                                        aria-valuenow="{{ compute.cpu_usage }}" aria-valuemin="0" aria-valuemax="100">{{ compute.cpu_usage }}%
+                                                    <div class="progress-bar bg-primary" role="progressbar" style="width: 0%"
+                                                        aria-valuemin="0" aria-valuemax="100" aria-label="{% trans "CPU Usage" %}">
                                                     </div>
                                                 </div>
+                                                <small class="cpu-usage-value"></small>
                                             </td>
                                             <td>{{ compute.ram_size|filesizeformat }}</td>
                                             <td>

instances/test_create_dialog_cpu.py

@@ -0,0 +1,114 @@
+"""The new-VM dialog's CPU bars: the page carries each compute's statistics
+URL and reads no CPU usage itself; its script fills the bars when the dialog
+opens, two computes at a time, and ignores answers of an earlier opening."""
+
+import json
+import shutil
+import subprocess
+import unittest
+from unittest.mock import PropertyMock, patch
+
+from computes.models import Compute
+from django.contrib.auth import get_user_model
+from django.test import TestCase
+from django.urls import reverse
+from lxml import html
+
+RUN = """
+const vm = require("vm");
+function element() {
+    const attrs = {};
+    return {style: {}, textContent: "", attrs,
+            setAttribute: (k, v) => { attrs[k] = String(v); }, removeAttribute: (k) => { delete attrs[k]; }};
+}
+const cells = URLS.map((url) => {
+    const bar = element(), label = element();
+    return {dataset: {url}, bar, label,
+            querySelector: (sel) => (sel === ".progress-bar" ? bar : label)};
+});
+const handlers = {}, requests = [];
+const dialog = {addEventListener: (name, fn) => { handlers[name] = fn; }, querySelectorAll: () => cells};
+const page = {
+    document: {getElementById: () => dialog},
+    $: {getJSON: (url) => {
+        const r = {url, done: (f) => { r.ok = f; return r; }, fail: (f) => { r.err = f; return r; },
+                   always: (f) => { r.end = f; return r; }};
+        requests.push(r);
+        return r;
+    }},
+    window: {location: {}},
+};
+vm.runInNewContext(SCRIPT, vm.createContext(page));
+const state = () => cells.map((c) => [c.bar.style.width, c.bar.attrs["aria-valuenow"] || null, c.label.textContent]);
+const answer = (i, data) => { const r = requests[i]; data === null ? r.err() : r.ok(data); r.end(); };
+const out = {};
+handlers["show.bs.modal"]();
+out.loading = state();
+out.first = requests.map((r) => r.url);
+handlers["hide.bs.modal"]();
+answer(0, {cpudata: 5});
+answer(1, null);
+out.hidden = [state(), requests.length];
+handlers["show.bs.modal"]();
+answer(2, {cpudata: 0});
+answer(3, {cpudata: null});
+out.filled = state();
+handlers["hide.bs.modal"]();
+handlers["show.bs.modal"]();
+out.reopened = requests.length;
+answer(4, {cpudata: 99});
+answer(5, {cpudata: 41.6});
+answer(6, null);
+out.final = state();
+out.all = requests.map((r) => r.url);
+console.log(JSON.stringify(out));
+"""
+
+
+class CreateDialogCpuTestCase(TestCase):
+    def setUp(self):
+        self.client.force_login(get_user_model().objects.create_superuser("cd-admin", "cd@example.com", "pw"))
+        self.computes = [
+            Compute.objects.create(name=f"cd{i}", hostname=f"192.0.2.{i}", login="u", password="p", type=1)
+            for i in (1, 2, 3)
+        ]
+        patches = [
+            patch.object(Compute, name, new_callable=PropertyMock, return_value=value)
+            for name, value in (("status", True), ("cpu_count", 1), ("ram_size", 1024), ("ram_usage", 0))
+        ]
+        patches += [patch("instances.views.utils.refr"), patch("computes.models.wvmHostDetails")]
+        for p in patches:
+            self.details = p.start()  # the last one: the host connection
+            self.addCleanup(p.stop)
+
+    def page(self):
+        response = self.client.get(reverse("instances:index"))
+        self.assertEqual(response.status_code, 200)
+        return response.content.decode()
+
+    def test_the_page_carries_the_statistics_urls_and_reads_no_cpu_usage(self):
+        page = self.page()
+        urls = html.fromstring(page).xpath("//td[@class='cpu-usage']/@data-url")
+        self.assertEqual(urls, [reverse("compute_graph", args=[c.id]) for c in self.computes])
+        self.details.return_value.get_cpu_usage.assert_not_called()
+
+    @unittest.skipUnless(shutil.which("node"), "node is not installed")
+    def test_the_dialog_fills_the_bars_two_at_a_time(self):
+        page = self.page()
+        script = next(s for s in html.fromstring(page).xpath("//script[not(@src)]/text()") if "show.bs.modal" in s)
+        urls = [reverse("compute_graph", args=[c.id]) for c in self.computes]
+        js = RUN.replace("SCRIPT", json.dumps(script)).replace("URLS", json.dumps(urls))
+        out = subprocess.run(["node", "-e", js], capture_output=True, text=True, timeout=30)
+        self.assertEqual(out.returncode, 0, out.stderr)
+        result = json.loads(out.stdout)
+        self.assertEqual(result["loading"], [["0%", None, "…"]] * 3)
+        self.assertEqual(result["first"], urls[:2])
+        # answers that come after the dialog closed change nothing and read no more
+        self.assertEqual(result["hidden"], [[["0%", None, "…"]] * 3, 2])
+        # 0 is a value, no value is a dash
+        self.assertEqual(result["filled"], [["0%", "0", "0%"], ["0%", None, "—"], ["0%", None, "…"]])
+        # reopened while the third read runs: one more read, two in all
+        self.assertEqual(result["reopened"], 6)
+        # the earlier opening's answer (99) is ignored; a failed read is a dash
+        self.assertEqual(result["final"], [["42%", "42", "42%"], ["0%", None, "—"], ["0%", None, "…"]])
+        self.assertEqual(result["all"], urls[:2] + urls + urls)

instances/test_index.py

@@ -36,8 +36,6 @@ class InstancesIndexTests(TestCase):
             Compute, "ram_size", new_callable=PropertyMock, return_value=1024
         ), patch.object(
             Compute, "ram_usage", new_callable=PropertyMock, return_value=0
-        ), patch.object(
-            Compute, "cpu_usage", new_callable=PropertyMock, return_value=0
         ):
             response = self.client.get(reverse("instances:index"))
 

instances/test_list_rpc.py

@@ -50,7 +50,7 @@ class ListInfoCallTests(TestCase):
         )
 
     def get(self, url):
-        compute_patches = {"status": True, "cpu_count": 1, "ram_size": 1024, "ram_usage": 0, "cpu_usage": 0}
+        compute_patches = {"status": True, "cpu_count": 1, "ram_size": 1024, "ram_usage": 0}
         with patch("instances.views.utils.refr"), patch("computes.views.utils.refresh_instance_database"), patch(
             "instances.models.wvmInstance", side_effect=self.proxy
         ):

instances/test_list_unreachable.py

@@ -44,7 +44,7 @@ class UnreachableListTests(TestCase):
 
         props = {"status": up, "connection_error": None if up else ERROR}
         patches = [patch.object(Compute, k, new_callable=PropertyMock, return_value=v) for k, v in props.items()]
-        for name in ("cpu_count", "ram_size", "ram_usage", "cpu_usage"):
+        for name in ("cpu_count", "ram_size", "ram_usage"):
             if up:
                 patches.append(patch.object(Compute, name, new_callable=PropertyMock, return_value=1))
             else:

instances/test_states.py

@@ -59,7 +59,7 @@ class StateTests(TestCase):
             return SimpleNamespace(instance=Domain(by_uuid[uuid]), get_title=lambda: "", get_uuid=lambda: uuid)
 
         self.client.force_login(user)
-        props = {"status": True, "connection_error": None, "cpu_count": 1, "ram_size": 1, "ram_usage": 0, "cpu_usage": 0}
+        props = {"status": True, "connection_error": None, "cpu_count": 1, "ram_size": 1, "ram_usage": 0}
         patches = [patch.object(Compute, k, new_callable=PropertyMock, return_value=v) for k, v in props.items()]
         patches += [patch("instances.views.utils.refr"), patch("computes.views.utils.refresh_instance_database"),
                     patch("instances.models.wvmInstance", side_effect=wvm)]

vrtManager/hostdetails.py

@@ -49,17 +49,14 @@ class wvmHostDetails(wvmConnect):
         stats = self.wvm.getCPUStats(VIR_NODE_CPU_STATS_ALL_CPUS, 0)
         return stats["idle"], sum(stats.get(name, 0) for name in CPU_TIMES)
 
-    def get_cpu_usage(self, diff=True, previous=None):
+    def get_cpu_usage(self, previous=None):
         """
-        Busy share of all host CPUs in percent. diff=False: since the host
-        started. Otherwise since previous, an (idle, total, time) sample this
-        method returned, or over one second when there is none or it does not
-        lead to this one (the host restarted). Also returns the closing sample
-        and the seconds the usage covers.
+        Busy share of all host CPUs in percent since previous, an (idle, total,
+        time) sample this method returned, or over one second when there is
+        none or it does not lead to this one (the host restarted). Also returns
+        the closing sample and the seconds the usage covers.
         """
         times = self._cpu_times()
-        if not diff:
-            return {"usage": cpu_percent((0, 0), times)}
         now = time.time()
         usage = None
         if previous is not None: