From eb6e6ca6265d5377986715cfde1b1874ee9b4afc Mon Sep 17 00:00:00 2001 From: catborise Date: Sun, 1 Nov 2026 19:37:33 +0300 Subject: [PATCH] feat(instances): the VM lists follow changes by themselves A VM list showed what it read when it was loaded: a VM stopped from inside its guest or with virsh, or made or removed elsewhere, showed only after a reload, unlike the VM page, which reloads when its VM's state changes. The lists of all VMs and of one host's VMs now poll the states of their VMs, at once and then every 5 s as the VM page does, and reload when an answer differs from the one before. The states come from one libvirt call per host; a user who does not see hosts gets only their own VMs. The page in use (a dialog, a field, an unsaved XML edit, a job outcome shown) holds the reload until it is free; that check is shared with the job panel. --- computes/templates/computes/instances.html | 1 + computes/views.py | 3 +- instances/templates/allinstances.html | 2 + .../templates/instances/vm_states_poll.html | 5 ++ instances/tests/test_vm_states.py | 90 +++++++++++++++++++ instances/urls.py | 1 + instances/utils.py | 19 ++++ instances/views.py | 25 +++++- operations/static/operations/job_panel.js | 11 +-- static/js/poll.js | 9 ++ static/js/vm_states.js | 28 ++++++ vrtManager/connection.py | 5 ++ webvirtcloud/tests/js/vm_states.test.js | 76 ++++++++++++++++ .../tests/test_authorization_matrix.py | 1 + webvirtcloud/tests/test_poll_js.py | 11 +++ 15 files changed, 276 insertions(+), 11 deletions(-) create mode 100644 instances/templates/instances/vm_states_poll.html create mode 100644 instances/tests/test_vm_states.py create mode 100644 static/js/vm_states.js create mode 100644 webvirtcloud/tests/js/vm_states.test.js diff --git a/computes/templates/computes/instances.html b/computes/templates/computes/instances.html index dfceb80b..e56c32af 100644 --- a/computes/templates/computes/instances.html +++ b/computes/templates/computes/instances.html @@ -88,5 +88,6 @@ {% block script %} {% job_panel_script job_resource %} +{% include "instances/vm_states_poll.html" %} {% endblock %} \ No newline at end of file diff --git a/computes/views.py b/computes/views.py index 3706456e..2e175035 100644 --- a/computes/views.py +++ b/computes/views.py @@ -138,7 +138,8 @@ def instances(request, compute_id): return render( request, "computes/instances.html", - {"compute": compute, "instances": instances, "job_resource": claims.compute_key(compute.id)}, + {"compute": compute, "instances": instances, "job_resource": claims.compute_key(compute.id), + "states_url": reverse("instances:states") + f"?compute={compute.id}"}, ) diff --git a/instances/templates/allinstances.html b/instances/templates/allinstances.html index 4841db37..a3fe344b 100644 --- a/instances/templates/allinstances.html +++ b/instances/templates/allinstances.html @@ -116,6 +116,8 @@ {% endblock content %} {% block script %} + + {% include "instances/vm_states_poll.html" %} {% if request.user.is_superuser %} diff --git a/instances/templates/instances/vm_states_poll.html b/instances/templates/instances/vm_states_poll.html new file mode 100644 index 00000000..68c1b1a6 --- /dev/null +++ b/instances/templates/instances/vm_states_poll.html @@ -0,0 +1,5 @@ +{% load static %} +{# the VM list reloads when its VMs change: vm_states.js #} +{{ states_url|json_script:"vm-states-url" }} + + diff --git a/instances/tests/test_vm_states.py b/instances/tests/test_vm_states.py new file mode 100644 index 00000000..6f54e35f --- /dev/null +++ b/instances/tests/test_vm_states.py @@ -0,0 +1,90 @@ +"""The VM lists follow libvirt: instances:states gives the states of the +listed VMs, read in one call per host; vm_states.js polls it and reloads the +list when an answer differs from the one before. A user who does not see +hosts gets only the hosts and VMs of their own.""" + +import json +from unittest.mock import MagicMock, PropertyMock, patch + +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.urls import reverse +from libvirt import libvirtError +from lxml import html + +from accounts.models import UserInstance +from computes.models import Compute +from instances.models import Instance + +UP = {"aaaaaaaa-0000-4000-8000-000000000001": ["web", 1], "aaaaaaaa-0000-4000-8000-000000000002": ["db", 5]} + + +class VmStatesTestCase(TestCase): + def setUp(self): + self.up = Compute.objects.create(name="up", hostname="192.0.2.70", login="u", password="p", type=1) + self.down = Compute.objects.create(name="down", hostname="192.0.2.71", login="u", password="p", type=1) + self.web = Instance.objects.create(compute=self.up, name="web", uuid="aaaaaaaa-0000-4000-8000-000000000001") + self.admin = get_user_model().objects.create_superuser("st-admin", "st@example.com", "pw") + + def get(self, user, query="", read=lambda: dict(UP)): + self.client.force_login(user) + proxy = MagicMock() + proxy.get_domain_states.side_effect = read + # self.up answers, self.down cannot be reached + with patch.object(Compute, "status", property(lambda c: c.pk == self.up.pk)), \ + patch.object(Compute, "proxy", new_callable=PropertyMock, return_value=proxy): + return self.client.get(reverse("instances:states") + query) + + def test_an_administrator_gets_every_host_and_a_host_that_cannot_be_read(self): + response = self.get(self.admin) + self.assertEqual(response["Cache-Control"], "no-store") + self.assertEqual(response.json()["computes"], {str(self.up.pk): {"ok": True, "vms": UP}, + str(self.down.pk): {"ok": False}}) + + def test_a_host_whose_listing_fails_cannot_be_read(self): + def fails(): + raise libvirtError("Cannot recv data") + self.assertEqual(self.get(self.admin, read=fails).json()["computes"][str(self.up.pk)], {"ok": False}) + + def test_a_user_gets_the_hosts_and_vms_of_their_own(self): + owner = get_user_model().objects.create_user("st-owner", password="pw") + UserInstance.objects.create(user=owner, instance=self.web) + self.assertEqual(self.get(owner).json()["computes"], + {str(self.up.pk): {"ok": True, "vms": {self.web.uuid: ["web", 1]}}}) + stranger = get_user_model().objects.create_user("st-stranger", password="pw") + self.assertEqual(self.get(stranger).json()["computes"], {}) + + def test_one_host_s_list_is_for_administrators(self): + self.assertEqual(self.get(self.admin, f"?compute={self.up.pk}").json()["computes"], + {str(self.up.pk): {"ok": True, "vms": UP}}) + owner = get_user_model().objects.create_user("st-owner2", password="pw") + self.assertEqual(self.get(owner, f"?compute={self.up.pk}").status_code, 403) + + def test_a_user_s_uuid_on_another_host_is_not_theirs(self): + # one UUID may be defined on two hosts: a VM is its host and UUID + other = Compute.objects.create(name="other", hostname="192.0.2.72", login="u", password="p", type=1) + Instance.objects.create(compute=other, name="mine-too", uuid="aaaaaaaa-0000-4000-8000-000000000009") + owner = get_user_model().objects.create_user("st-owner3", password="pw") + UserInstance.objects.create(user=owner, instance=self.web) + UserInstance.objects.create(user=owner, instance=Instance.objects.get(name="mine-too")) + self.client.force_login(owner) + proxy = MagicMock() + proxy.get_domain_states.return_value = dict(UP) # both hosts list the same UUIDs + with patch.object(Compute, "status", True), \ + patch.object(Compute, "proxy", new_callable=PropertyMock, return_value=proxy): + computes = self.client.get(reverse("instances:states")).json()["computes"] + self.assertEqual(computes, {str(self.up.pk): {"ok": True, "vms": {self.web.uuid: ["web", 1]}}, + str(other.pk): {"ok": True, "vms": {}}}) + + def test_the_lists_poll_it(self): + self.client.force_login(self.admin) + with patch.object(Compute, "status", property(lambda c: c.pk == self.up.pk)), \ + patch.object(Compute, "proxy", new_callable=PropertyMock, return_value=MagicMock()), \ + patch("instances.views.utils.refr"), patch("computes.views.utils.refresh_instance_database"), \ + patch.object(Instance, "proxy", new_callable=PropertyMock, return_value=MagicMock()): + for url, states_url in ((reverse("instances:index"), reverse("instances:states")), + (reverse("instances", args=[self.up.pk]), + reverse("instances:states") + f"?compute={self.up.pk}")): + with self.subTest(url=url): + page = html.fromstring(self.client.get(url).content) + self.assertEqual(json.loads(page.get_element_by_id("vm-states-url").text), states_url) diff --git a/instances/urls.py b/instances/urls.py index c12ba49d..f07fdf72 100755 --- a/instances/urls.py +++ b/instances/urls.py @@ -6,6 +6,7 @@ app_name = "instances" urlpatterns = [ path("", views.index, name="index"), + path("states/", views.states, name="states"), path("find_address/", views.find_address, name="find_address"), path("report.xlsx", views.vm_report, name="vm_report"), path("flavor/create/", views.flavor_create, name="flavor_create"), diff --git a/instances/utils.py b/instances/utils.py index e27334c2..31800dd4 100644 --- a/instances/utils.py +++ b/instances/utils.py @@ -338,6 +338,25 @@ def record_migration(instance, new_compute): instance.save() +def vm_states(user, computes): + """{compute pk: {"ok": True, "vms": {uuid: [name, state]}}, or {"ok": False} + for a host that cannot be read}: what the VM lists poll, read from libvirt; + for a user who does not see hosts only their own VMs (a VM is its host and + UUID: one UUID may be defined on two hosts).""" + show_hosts = user.is_superuser or user.has_perm("instances.view_instances") + own = None if show_hosts else set( + Instance.objects.filter(userinstance__user=user).values_list("compute_id", "uuid")) + result = {} + for compute in computes: + try: + vms = compute.proxy.get_domain_states() if compute.status is True else None + except libvirtError: + vms = None + result[str(compute.pk)] = {"ok": False} if vms is None else { + "ok": True, "vms": {u: s for u, s in vms.items() if own is None or (compute.pk, u) in own}} + return result + + def migration_target(compute): """The destination for the log: its host name, and the migration address when one is set and differs (else the migration goes to the host name).""" diff --git a/instances/views.py b/instances/views.py index 50fbaceb..6ab46fca 100755 --- a/instances/views.py +++ b/instances/views.py @@ -103,10 +103,33 @@ def index(request): {"computes": computes, "instances": instances, "show_hosts": show_hosts, "own_hosts_down": own_hosts_down, "reachable_computes": [c for c in computes if c.status is True], "unreachable_computes": [c for c in computes if c.status is not True], - "states": [label for label, _color in STATES.values()] + [UNKNOWN[0]]}, + "states": [label for label, _color in STATES.values()] + [UNKNOWN[0]], + "states_url": reverse("instances:states")}, ) +def _listed_computes(user, computes): + """The hosts of the VM list: every host for a user who sees hosts, else + only those of the user's own VMs.""" + if user.is_superuser or user.has_perm("instances.view_instances"): + return computes + own = set(Instance.objects.filter(userinstance__user=user).values_list("compute_id", flat=True)) + return [c for c in computes if c.pk in own] + + +def states(request): + """The VM lists' poll: utils.vm_states of the list's hosts; ?compute=id + for one host's list (administrators).""" + computes = list(Compute.objects.all()) + if request.GET.get("compute"): + if not request.user.is_superuser: + raise PermissionDenied + computes = [get_object_or_404(Compute, pk=request.GET["compute"])] + response = JsonResponse({"computes": utils.vm_states(request.user, _listed_computes(request.user, computes))}) + response["Cache-Control"] = "no-store" + return response + + def _address_scope(user): """What the address search and the VM report read for the user: whether the user sees hosts, the registered VMs by (compute id, UUID), and the diff --git a/operations/static/operations/job_panel.js b/operations/static/operations/job_panel.js index c7157a4f..9a13e0b1 100644 --- a/operations/static/operations/job_panel.js +++ b/operations/static/operations/job_panel.js @@ -120,14 +120,6 @@ function jobPanel(element, urls, csrfToken, initialIds) { return job.state === "succeeded" && !result.warning && !(result.left_behind || []).length; } - // a dialog open, a field in use or an XML edit not saved: its outcome waits in the panel, with Refresh - function interacting() { - var active = document.activeElement; - return !!document.querySelector(".modal.show") - || !!(active && active.matches && active.matches("input, select, textarea")) - || !!(window.xmlEditUnsaved && window.xmlEditUnsaved()); - } - function fetchOutcome(id) { if (inflight[id]) { return; } inflight[id] = true; @@ -140,7 +132,8 @@ function jobPanel(element, urls, csrfToken, initialIds) { delete pending[id]; // only when no other job runs, is shown or awaited: a reload would lose its outcome var others = Object.keys(outcomes).length + Object.keys(pending).length + Object.keys(shown).length; - if (quiet(job) && !interacting() && !others) { + // the page in use (poll.js): its outcome waits in the panel, with Refresh + if (quiet(job) && !pageInUse() && !others) { window.location.reload(); // the job is not remembered again: no loop return; } diff --git a/static/js/poll.js b/static/js/poll.js index 113ee05c..04c6dd29 100644 --- a/static/js/poll.js +++ b/static/js/poll.js @@ -49,3 +49,12 @@ function pollWhileVisible(interval, request, onData, firstDelay) { }, }; } + +// The user is in the page: a dialog open, a field in use or an XML edit not +// saved. A poll that would reload the page waits for its next answer. +function pageInUse() { + var active = document.activeElement; + return !!document.querySelector(".modal.show") + || !!(active && active.matches && active.matches("input, select, textarea")) + || !!(window.xmlEditUnsaved && window.xmlEditUnsaved()); +} diff --git a/static/js/vm_states.js b/static/js/vm_states.js new file mode 100644 index 00000000..1cf17548 --- /dev/null +++ b/static/js/vm_states.js @@ -0,0 +1,28 @@ +// A VM list follows libvirt: it polls the states of its VMs (instances:states) +// at once, then every 5 s, as the VM page, and reloads when an answer differs +// from the one before: a VM started or stopped, made, removed or renamed +// outside the page included. The page in use or the job panel showing +// something: it waits for the next answer. poll.js gives the polling. +function vmStatesKey(states) { + return Object.keys(states).sort().map(function (id) { + var host = states[id]; + return id + ":" + (host.ok ? Object.keys(host.vms).sort().map(function (uuid) { + return uuid + "=" + host.vms[uuid].join("/"); + }).join(",") : "unreadable"); + }).join(";"); +} + +function followVmStates(url) { + var last = null; + // the first answer, at once, is what the page was rendered with + return pollWhileVisible(5000, function () { return $.getJSON(url); }, function (data) { + var key = vmStatesKey(data.computes); + if (last !== null && key !== last) { + if (!pageInUse() && !window.backgroundJobRunning) { + window.location.reload(); + } + return; // in use: the change stays against the answer before + } + last = key; + }, 0); +} diff --git a/vrtManager/connection.py b/vrtManager/connection.py index 57bb7feb..d3a7c6b5 100644 --- a/vrtManager/connection.py +++ b/vrtManager/connection.py @@ -1011,6 +1011,11 @@ class wvmConnect(object): instances.append(name) return instances + def get_domain_states(self): + """{uuid: [name, state]} of every domain, in one call.""" + return {dom.UUIDString(): [dom.name(), stats["state.state"]] + for dom, stats in self.wvm.getAllDomainStats(libvirt.VIR_DOMAIN_STATS_STATE)} + def domain_trees(self, running_only=False): """Each domain's name with its definitions, parsed: the live one, and for a persistent domain the one saved for its next boot too, read diff --git a/webvirtcloud/tests/js/vm_states.test.js b/webvirtcloud/tests/js/vm_states.test.js new file mode 100644 index 00000000..5bc6fa2a --- /dev/null +++ b/webvirtcloud/tests/js/vm_states.test.js @@ -0,0 +1,76 @@ +// Behaviour of static/js/vm_states.js (with static/js/poll.js) under a fake +// clock, document and jQuery. Run by webvirtcloud/tests/test_poll_js.py: +// node vm_states.test.js static/js/poll.js static/js/vm_states.js +const fs = require("fs"); +const assert = require("assert"); + +let now = 0, timers = [], nextId = 1; +global.setTimeout = (fn, ms) => { const id = nextId++; timers.push({ id, at: now + ms, fn }); return id; }; +global.clearTimeout = (id) => { timers = timers.filter((t) => t.id !== id); }; +function advance(ms) { + const end = now + ms; + for (;;) { + timers.sort((a, b) => a.at - b.at); + const t = timers[0]; + if (!t || t.at > end) break; + timers.shift(); now = t.at; t.fn(); + } + now = end; +} +global.document = { hidden: false, addEventListener() {}, activeElement: null, querySelector: () => null }; +global.window = { backgroundJobRunning: false, location: { reload() { window.reloads += 1; } }, reloads: 0 }; + +// each poll answers with what the test puts in `answer` +let answer = null, asked = []; +global.$ = { getJSON(url) { + asked.push(url); + const value = answer; + const r = { always(f) { f(); return r; }, done(f) { f(value); return r; } }; + return r; +} }; +eval(fs.readFileSync(process.argv[2], "utf8")); +eval(fs.readFileSync(process.argv[3], "utf8")); + +const base = { "1": { ok: true, vms: { "u-a": ["web", 1], "u-b": ["db", 5] } }, "2": { ok: false } }; +followVmStates("/instances/states/?compute=1"); + +// the first answer, asked at once, is where it starts; the same VMs in another order change nothing +answer = { computes: base }; +advance(0); +assert.strictEqual(asked.length, 1, "asks at once"); +answer = { computes: { "2": { ok: false }, "1": { ok: true, vms: { "u-b": ["db", 5], "u-a": ["web", 1] } } } }; +advance(5000); +assert.deepStrictEqual(asked, ["/instances/states/?compute=1", "/instances/states/?compute=1"], "asks every 5 s"); +assert.strictEqual(window.reloads, 0, "nothing changed"); + +// a VM stopped, a new VM, a removed one, a renamed one, a host read again: each reloads +for (const host1 of [{ ok: true, vms: { "u-a": ["web", 5], "u-b": ["db", 5] } }, + { ok: true, vms: { "u-a": ["web", 1], "u-b": ["db", 5], "u-c": ["new", 5] } }, + { ok: true, vms: { "u-a": ["web", 1] } }, { ok: true, vms: { "u-a": ["web2", 1], "u-b": ["db", 5] } }]) { + window.reloads = 0; + answer = { computes: { "1": host1, "2": { ok: false } } }; + advance(5000); + assert.strictEqual(window.reloads, 1, JSON.stringify(host1)); + answer = { computes: base }; // the reloaded page starts again from here + advance(5000); + window.reloads = 0; +} + +// not while the page is in use or the job panel shows something: the next answer tells again +for (const [what, set, unset] of [ + ["a dialog", () => { document.querySelector = (s) => s === ".modal.show" ? {} : null; }, () => { document.querySelector = () => null; }], + ["the filter", () => { document.activeElement = { matches: () => true }; }, () => { document.activeElement = null; }], + ["a job outcome", () => { window.backgroundJobRunning = true; }, () => { window.backgroundJobRunning = false; }], +]) { + window.reloads = 0; + answer = { computes: { "1": { ok: true, vms: { "u-a": ["web", 5], "u-b": ["db", 5] } }, "2": { ok: false } } }; + set(); + advance(5000); + assert.strictEqual(window.reloads, 0, "waits for " + what); + unset(); + advance(5000); + assert.strictEqual(window.reloads, 1, "then reloads: " + what); + answer = { computes: base }; + advance(5000); +} +console.log("ok"); diff --git a/webvirtcloud/tests/test_authorization_matrix.py b/webvirtcloud/tests/test_authorization_matrix.py index 2c132ab4..f9aa1f60 100644 --- a/webvirtcloud/tests/test_authorization_matrix.py +++ b/webvirtcloud/tests/test_authorization_matrix.py @@ -32,6 +32,7 @@ ALLOWED_FOR_ANY_USER = { "index": "landing page", "instances:index": "lists only the user's own instances", "instances:find_address": "searches only the user's own instances", + "instances:states": "the VM list's poll: only the user's own instances", "accounts:login": "login page", "accounts:email_otp": "part of the login flow (its own issues are tracked separately)", "accounts:logout": "logout", diff --git a/webvirtcloud/tests/test_poll_js.py b/webvirtcloud/tests/test_poll_js.py index 2678c42a..c6791033 100644 --- a/webvirtcloud/tests/test_poll_js.py +++ b/webvirtcloud/tests/test_poll_js.py @@ -25,6 +25,17 @@ class PollTestCase(unittest.TestCase): ) self.assertEqual(result.returncode, 0, result.stderr) + @unittest.skipUnless(shutil.which("node"), "node is not installed") + def test_vm_states_js(self): + result = subprocess.run( + ["node", str(BASE / "webvirtcloud/tests/js/vm_states.test.js"), str(BASE / "static/js/poll.js"), + str(BASE / "static/js/vm_states.js")], + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stderr) + def test_pages_poll_through_poll_js(self): for template in POLLING_TEMPLATES: with self.subTest(template=template):