feat(instances): the VM lists follow changes by themselves

A VM list showed what it read when it was loaded: a VM stopped from
inside its guest or with virsh, or made or removed elsewhere, showed only
after a reload, unlike the VM page, which reloads when its VM's state
changes. The lists of all VMs and of one host's VMs now poll the states
of their VMs, at once and then every 5 s as the VM page does, and reload
when an answer differs from the one before. The states come from one
libvirt call per host; a user who does not see hosts gets only their own
VMs. The page in use (a dialog, a field, an unsaved XML edit, a job
outcome shown) holds the reload until it is free; that check is shared
with the job panel.

instances/templates/allinstances.html

@@ -116,6 +116,8 @@
     </div>
 {% endblock content %}
 {% block script %}
+    <script src="{% static 'js/poll.js' %}"></script>
+    {% include "instances/vm_states_poll.html" %}
     <script src="{% static 'js/table_filter.js' %}"></script>
     <script src="{% static 'js/find_address.js' %}"></script>
     {% if request.user.is_superuser %}

instances/templates/instances/vm_states_poll.html

@@ -0,0 +1,5 @@
+{% load static %}
+{# the VM list reloads when its VMs change: vm_states.js #}
+{{ states_url|json_script:"vm-states-url" }}
+<script src="{% static 'js/vm_states.js' %}"></script>
+<script>followVmStates(JSON.parse(document.getElementById("vm-states-url").textContent));</script>

instances/tests/test_vm_states.py

@@ -0,0 +1,90 @@
+"""The VM lists follow libvirt: instances:states gives the states of the
+listed VMs, read in one call per host; vm_states.js polls it and reloads the
+list when an answer differs from the one before. A user who does not see
+hosts gets only the hosts and VMs of their own."""
+
+import json
+from unittest.mock import MagicMock, PropertyMock, patch
+
+from django.contrib.auth import get_user_model
+from django.test import TestCase
+from django.urls import reverse
+from libvirt import libvirtError
+from lxml import html
+
+from accounts.models import UserInstance
+from computes.models import Compute
+from instances.models import Instance
+
+UP = {"aaaaaaaa-0000-4000-8000-000000000001": ["web", 1], "aaaaaaaa-0000-4000-8000-000000000002": ["db", 5]}
+
+
+class VmStatesTestCase(TestCase):
+    def setUp(self):
+        self.up = Compute.objects.create(name="up", hostname="192.0.2.70", login="u", password="p", type=1)
+        self.down = Compute.objects.create(name="down", hostname="192.0.2.71", login="u", password="p", type=1)
+        self.web = Instance.objects.create(compute=self.up, name="web", uuid="aaaaaaaa-0000-4000-8000-000000000001")
+        self.admin = get_user_model().objects.create_superuser("st-admin", "st@example.com", "pw")
+
+    def get(self, user, query="", read=lambda: dict(UP)):
+        self.client.force_login(user)
+        proxy = MagicMock()
+        proxy.get_domain_states.side_effect = read
+        # self.up answers, self.down cannot be reached
+        with patch.object(Compute, "status", property(lambda c: c.pk == self.up.pk)), \
+                patch.object(Compute, "proxy", new_callable=PropertyMock, return_value=proxy):
+            return self.client.get(reverse("instances:states") + query)
+
+    def test_an_administrator_gets_every_host_and_a_host_that_cannot_be_read(self):
+        response = self.get(self.admin)
+        self.assertEqual(response["Cache-Control"], "no-store")
+        self.assertEqual(response.json()["computes"], {str(self.up.pk): {"ok": True, "vms": UP},
+                                                       str(self.down.pk): {"ok": False}})
+
+    def test_a_host_whose_listing_fails_cannot_be_read(self):
+        def fails():
+            raise libvirtError("Cannot recv data")
+        self.assertEqual(self.get(self.admin, read=fails).json()["computes"][str(self.up.pk)], {"ok": False})
+
+    def test_a_user_gets_the_hosts_and_vms_of_their_own(self):
+        owner = get_user_model().objects.create_user("st-owner", password="pw")
+        UserInstance.objects.create(user=owner, instance=self.web)
+        self.assertEqual(self.get(owner).json()["computes"],
+                         {str(self.up.pk): {"ok": True, "vms": {self.web.uuid: ["web", 1]}}})
+        stranger = get_user_model().objects.create_user("st-stranger", password="pw")
+        self.assertEqual(self.get(stranger).json()["computes"], {})
+
+    def test_one_host_s_list_is_for_administrators(self):
+        self.assertEqual(self.get(self.admin, f"?compute={self.up.pk}").json()["computes"],
+                         {str(self.up.pk): {"ok": True, "vms": UP}})
+        owner = get_user_model().objects.create_user("st-owner2", password="pw")
+        self.assertEqual(self.get(owner, f"?compute={self.up.pk}").status_code, 403)
+
+    def test_a_user_s_uuid_on_another_host_is_not_theirs(self):
+        # one UUID may be defined on two hosts: a VM is its host and UUID
+        other = Compute.objects.create(name="other", hostname="192.0.2.72", login="u", password="p", type=1)
+        Instance.objects.create(compute=other, name="mine-too", uuid="aaaaaaaa-0000-4000-8000-000000000009")
+        owner = get_user_model().objects.create_user("st-owner3", password="pw")
+        UserInstance.objects.create(user=owner, instance=self.web)
+        UserInstance.objects.create(user=owner, instance=Instance.objects.get(name="mine-too"))
+        self.client.force_login(owner)
+        proxy = MagicMock()
+        proxy.get_domain_states.return_value = dict(UP)  # both hosts list the same UUIDs
+        with patch.object(Compute, "status", True), \
+                patch.object(Compute, "proxy", new_callable=PropertyMock, return_value=proxy):
+            computes = self.client.get(reverse("instances:states")).json()["computes"]
+        self.assertEqual(computes, {str(self.up.pk): {"ok": True, "vms": {self.web.uuid: ["web", 1]}},
+                                    str(other.pk): {"ok": True, "vms": {}}})
+
+    def test_the_lists_poll_it(self):
+        self.client.force_login(self.admin)
+        with patch.object(Compute, "status", property(lambda c: c.pk == self.up.pk)), \
+                patch.object(Compute, "proxy", new_callable=PropertyMock, return_value=MagicMock()), \
+                patch("instances.views.utils.refr"), patch("computes.views.utils.refresh_instance_database"), \
+                patch.object(Instance, "proxy", new_callable=PropertyMock, return_value=MagicMock()):
+            for url, states_url in ((reverse("instances:index"), reverse("instances:states")),
+                                    (reverse("instances", args=[self.up.pk]),
+                                     reverse("instances:states") + f"?compute={self.up.pk}")):
+                with self.subTest(url=url):
+                    page = html.fromstring(self.client.get(url).content)
+                    self.assertEqual(json.loads(page.get_element_by_id("vm-states-url").text), states_url)

instances/urls.py

@@ -6,6 +6,7 @@ app_name = "instances"
 
 urlpatterns = [
     path("", views.index, name="index"),
+    path("states/", views.states, name="states"),
     path("find_address/", views.find_address, name="find_address"),
     path("report.xlsx", views.vm_report, name="vm_report"),
     path("flavor/create/", views.flavor_create, name="flavor_create"),

instances/utils.py

@@ -338,6 +338,25 @@ def record_migration(instance, new_compute):
             instance.save()
 
 
+def vm_states(user, computes):
+    """{compute pk: {"ok": True, "vms": {uuid: [name, state]}}, or {"ok": False}
+    for a host that cannot be read}: what the VM lists poll, read from libvirt;
+    for a user who does not see hosts only their own VMs (a VM is its host and
+    UUID: one UUID may be defined on two hosts)."""
+    show_hosts = user.is_superuser or user.has_perm("instances.view_instances")
+    own = None if show_hosts else set(
+        Instance.objects.filter(userinstance__user=user).values_list("compute_id", "uuid"))
+    result = {}
+    for compute in computes:
+        try:
+            vms = compute.proxy.get_domain_states() if compute.status is True else None
+        except libvirtError:
+            vms = None
+        result[str(compute.pk)] = {"ok": False} if vms is None else {
+            "ok": True, "vms": {u: s for u, s in vms.items() if own is None or (compute.pk, u) in own}}
+    return result
+
+
 def migration_target(compute):
     """The destination for the log: its host name, and the migration address
     when one is set and differs (else the migration goes to the host name)."""

instances/views.py

@@ -103,10 +103,33 @@ def index(request):
         {"computes": computes, "instances": instances, "show_hosts": show_hosts, "own_hosts_down": own_hosts_down,
          "reachable_computes": [c for c in computes if c.status is True],
          "unreachable_computes": [c for c in computes if c.status is not True],
-         "states": [label for label, _color in STATES.values()] + [UNKNOWN[0]]},
+         "states": [label for label, _color in STATES.values()] + [UNKNOWN[0]],
+         "states_url": reverse("instances:states")},
     )
 
 
+def _listed_computes(user, computes):
+    """The hosts of the VM list: every host for a user who sees hosts, else
+    only those of the user's own VMs."""
+    if user.is_superuser or user.has_perm("instances.view_instances"):
+        return computes
+    own = set(Instance.objects.filter(userinstance__user=user).values_list("compute_id", flat=True))
+    return [c for c in computes if c.pk in own]
+
+
+def states(request):
+    """The VM lists' poll: utils.vm_states of the list's hosts; ?compute=id
+    for one host's list (administrators)."""
+    computes = list(Compute.objects.all())
+    if request.GET.get("compute"):
+        if not request.user.is_superuser:
+            raise PermissionDenied
+        computes = [get_object_or_404(Compute, pk=request.GET["compute"])]
+    response = JsonResponse({"computes": utils.vm_states(request.user, _listed_computes(request.user, computes))})
+    response["Cache-Control"] = "no-store"
+    return response
+
+
 def _address_scope(user):
     """What the address search and the VM report read for the user: whether
     the user sees hosts, the registered VMs by (compute id, UUID), and the