feat(instances): a report of every VM with its addresses and OS, as an Excel file

The Administration menu gets "VM report (.xlsx)", a workbook for
superusers. The VMs sheet has one row per VM on every compute, shut-off
ones and those the panel has not registered too: name,
state, host, OS and OS family, vCPUs, memory, IP addresses and where they
came from, MAC addresses, owners, and whether the panel has registered
it. A Coverage sheet says when it was made and what could not be read.

The addresses come from the same live reading as the address search,
now shared by both. The OS is the guest agent's (libvirt's guest info)
or, without one, the libosinfo id a tool such as virt-install wrote into
the definition. Link-local IPv6 addresses are left out of the report.
Text cells stay text: a VM name starting with "=" is not a formula.

openpyxl 3.1.5 writes the workbook.

conf/requirements.txt

@@ -14,6 +14,7 @@ drf-spectacular[sidecar]==0.30.0
 gunicorn==26.2.0
 libvirt-python>=9.0.0
 lxml>=6.1
+openpyxl==3.1.5
 paramiko==5.0.0
 psycopg[binary]==3.3.6
 python-engineio==4.14.0

instances/addresses.py

@@ -1,5 +1,6 @@
 """
-The VMs that use an IP address or an address in a network, read live from
+The VMs that use an IP address or an address in a network, and an inventory
+of every VM with its addresses and operating system, read live from
 libvirt: DHCP leases of libvirt's networks, the host's ARP table, and the
 guest agent. An address counts only on one of the VM's own NICs (by MAC), so
 interfaces inside the guest such as docker0 never match.
@@ -10,9 +11,11 @@ import time
 from concurrent import futures
 from dataclasses import dataclass
 from ipaddress import ip_address, ip_network
+from urllib.parse import urlparse
 
 from libvirt import (
     VIR_CONNECT_LIST_DOMAINS_ACTIVE,
+    VIR_DOMAIN_GUEST_INFO_OS,
     VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT,
     VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_ARP,
     VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE,
@@ -88,47 +91,87 @@ def _counted(func, *args):
             _running -= 1
 
 
+def os_family(name):
+    """Windows, FreeBSD or Linux, from a guest agent's OS id or a libosinfo
+    URL's host; "" for none."""
+    if not name:
+        return ""
+    if "microsoft" in name or name == "mswindows":
+        return "Windows"
+    if "freebsd" in name:
+        return "FreeBSD"
+    return "Linux"
+
+
+def defined_os(xml):
+    """(name, family) of the libosinfo id a tool such as virt-install wrote
+    into the definition (http://almalinux.org/almalinux/9: almalinux 9)."""
+    ids = etree.fromstring(xml).xpath("/domain/metadata/*[local-name()='libosinfo']/*[local-name()='os']/@id")
+    if not ids:
+        return "", ""
+    url = urlparse(ids[0])
+    return " ".join(url.path.strip("/").split("/")), os_family(url.netloc)
+
+
 def _matching(dom, vm, source, macs, network):
-    """The source's addresses in the network on the domain's own NICs, as
-    (vm, MAC, address, source) observations; vm is (compute id, UUID, name).
-    None if the source could not be read."""
+    """The source's addresses on the domain's own NICs (in the network, if
+    one is given), as (vm, MAC, address, source) observations; vm is
+    (compute id, UUID, name). None if the source could not be read."""
     pairs = read(dom, source)
     if pairs is None:
         return None
-    return [(vm, mac, address, source) for mac, address in pairs if mac in macs and address in network]
+    return [(vm, mac, address, source) for mac, address in pairs
+            if mac in macs and (network is None or address in network)]
 
 
-def _read_host(target, network):
-    """The cheap first phase on one compute: leases and ARP of each running VM
-    in scope. Returns the observations, the (domain, vm, MACs) whose agent is
-    to be asked, the number of VMs read and of those without a connected
-    agent; None when the compute cannot be reached (connecting may take the
-    connect timeout, so this too runs on the compute's thread). A VM that
-    stops or leaves meanwhile is skipped."""
+def _read_os(dom):
+    """The guest agent's OS as (name, family); None if it gives none."""
+    try:
+        info = dom.guestInfo(VIR_DOMAIN_GUEST_INFO_OS)
+    except libvirtError:
+        return None
+    if not info.get("os.id"):
+        return None
+    return info.get("os.pretty-name") or info.get("os.name") or info["os.id"], os_family(info["os.id"])
+
+
+def _read_host(target, network, everything):
+    """The cheap first phase on one compute: leases and ARP of each running
+    VM in scope (with everything, the shut-off ones are listed too). Returns
+    the VMs' records, the observations, the (domain, vm, MACs) whose agent is
+    to be asked, the number of running VMs read and of those without a
+    connected agent; None when the compute cannot be reached (connecting may
+    take the connect timeout, so this too runs on the compute's thread). A
+    VM that stops or leaves meanwhile is skipped."""
     if target.compute.status is not True:
         return None
-    found, agents, checked, no_agent = [], [], 0, 0
-    for dom in target.compute.connection.listAllDomains(VIR_CONNECT_LIST_DOMAINS_ACTIVE):
+    records, found, agents, checked, no_agent = [], [], [], 0, 0
+    listing = 0 if everything else VIR_CONNECT_LIST_DOMAINS_ACTIVE
+    for dom in target.compute.connection.listAllDomains(listing):
         try:
             uuid = dom.UUIDString()
             if target.uuids is not None and uuid not in target.uuids:
                 continue
             vm = (target.compute.id, uuid, dom.name())
             xml = dom.XMLDesc(0)
-            running = dom.state()[0] == VIR_DOMAIN_RUNNING  # a paused guest cannot answer
+            state, _, memory, vcpus, _ = dom.info()
+            active = dom.isActive()
         except libvirtError:
             continue
         macs = nic_macs(xml)
+        os, family = defined_os(xml)
+        records.append({"vm": vm, "state": state, "vcpus": vcpus, "memory": memory // 1024,
+                        "macs": sorted(macs), "os": os, "os_family": family})
+        if not active:
+            continue
         checked += 1
         for source in (VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE, VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_ARP):
             found += _matching(dom, vm, source, macs, network) or []
         if not agent_connected(xml):
             no_agent += 1
-        elif running:
+        elif state == VIR_DOMAIN_RUNNING:  # a paused guest cannot answer
             agents.append((dom, vm, macs))
-    return found, agents, checked, no_agent
-
-
+    return records, found, agents, checked, no_agent
 def _rows(observations):
     """One row per VM, MAC and address with its sources; sorted by address.
     A VM is its compute and UUID: one UUID may be defined on two hosts."""
@@ -146,22 +189,24 @@ def _rows(observations):
     return rows
 
 
-def find(targets, network, deadline=5.0):
+def _survey(targets, network, deadline, everything):
     """
-    The running VMs of the targets with an address in the network, read by
-    the deadline (seconds). First each compute's leases and ARP, then each
-    VM's guest agent on its own thread, so a hung agent holds back only its
-    VM. What was not read in time is counted, not waited for: hosts not
-    reached, hosts not finished, and agents not read (late or failed).
+    Read the targets by the deadline (seconds): first each compute's VMs,
+    leases and ARP, then each VM's guest agent on its own thread, so a hung
+    agent holds back only its VM. What was not read in time is counted, not
+    waited for: hosts not reached, hosts not finished, and agents not read
+    (late or failed). Returns the VMs' records, the observations, the agents'
+    OS by vm, and the counts.
     """
     if _running > MAX_RUNNING - MAX_WORKERS:
         raise Busy
     end = time.monotonic() + deadline
-    observations, hosts_down, agent_reads = [], [], []
+    records, observations, hosts_down, agent_reads, os_reads, systems = [], [], [], [], {}, {}
     checked = no_agent = unread = 0
     pool = futures.ThreadPoolExecutor(max_workers=MAX_WORKERS)
     try:
-        hosts = {pool.submit(_counted, _read_host, target, network): target.compute for target in targets}
+        hosts = {pool.submit(_counted, _read_host, target, network, everything): target.compute
+                 for target in targets}
         waiting = set(hosts)
         try:
             for done in futures.as_completed(hosts, timeout=max(0, end - time.monotonic())):
@@ -173,24 +218,56 @@ def find(targets, network, deadline=5.0):
                 if read_host is None:
                     hosts_down.append(hosts[done].name)
                     continue
-                found, agents, host_checked, host_no_agent = read_host
+                host_records, found, agents, host_checked, host_no_agent = read_host
+                records += host_records
                 observations += found
                 checked += host_checked
                 no_agent += host_no_agent
-                agent_reads += [pool.submit(_counted, _matching, dom, vm,
-                                            VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT, macs, network)
-                                for dom, vm, macs in agents]
+                # the OS on its own thread: a hung OS read keeps the addresses read
+                for dom, vm, macs in agents:
+                    agent_reads.append(pool.submit(_counted, _matching, dom, vm,
+                                                   VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT, macs, network))
+                    if everything:
+                        os_reads[pool.submit(_counted, _read_os, dom)] = vm
         except futures.TimeoutError:
             pass
-        finished, late = futures.wait(agent_reads, timeout=max(0, end - time.monotonic()))
-        unread = len(late)
-        for done in finished:
-            found = done.result()
+        futures.wait(agent_reads + list(os_reads), timeout=max(0, end - time.monotonic()))
+        for done in agent_reads:
+            found = done.result() if done.done() and not done.cancelled() else None
             if found is None:
                 unread += 1
             else:
                 observations += found
+        for done, vm in os_reads.items():
+            if done.done() and not done.cancelled() and done.result():
+                systems[vm] = done.result()
     finally:
         pool.shutdown(wait=False, cancel_futures=True)
-    return {"matches": _rows(observations), "checked": checked, "no_agent": no_agent, "unread": unread,
-            "hosts_down": sorted(hosts_down), "hosts_unfinished": sorted(hosts[f].name for f in waiting)}
+    return records, observations, systems, {
+        "checked": checked, "no_agent": no_agent, "unread": unread, "hosts_down": sorted(hosts_down),
+        "hosts_unfinished": sorted(hosts[future].name for future in waiting)}
+
+
+def find(targets, network, deadline=5.0):
+    """The running VMs of the targets with an address in the network, one
+    row per VM, MAC and address, with what could not be read."""
+    _, observations, _, counts = _survey(targets, network, deadline, everything=False)
+    return {"matches": _rows(observations), **counts}
+
+
+def inventory(targets, deadline=20.0):
+    """Every VM of the targets, shut-off ones too, with its addresses and OS
+    (the agent's, else the definition's libosinfo id), sorted by name."""
+    records, observations, systems, counts = _survey(targets, None, deadline, everything=True)
+    addresses = {}
+    for row in _rows(observations):
+        vm = (row["compute_id"], row["uuid"], row["name"])
+        addresses.setdefault(vm, []).append((row["address"], row["sources"]))
+    vms = []
+    for record in records:
+        vm = record.pop("vm")
+        os, family = systems.get(vm, (record["os"], record["os_family"]))
+        vms.append({"compute_id": vm[0], "uuid": vm[1], "name": vm[2], **record, "os": os, "os_family": family,
+                    "addresses": addresses.get(vm, [])})
+    vms.sort(key=lambda row: (row["name"], row["compute_id"]))
+    return {"vms": vms, **counts}

instances/report.py

@@ -0,0 +1,38 @@
+"""An Excel workbook: rows under a bold, filterable header, and a sheet of notes."""
+
+from io import BytesIO
+
+from openpyxl import Workbook
+from openpyxl.styles import Font
+from openpyxl.utils import get_column_letter
+
+MAX_WIDTH = 60
+
+
+def workbook(title, header, rows, notes_title, notes):
+    """The .xlsx file's bytes. Every text cell stays text: openpyxl would
+    write one starting with "=" as a formula, and the text comes from VM
+    names and guests."""
+    book = Workbook()
+    sheet = book.active
+    sheet.title = title
+    for values in [header, *rows]:
+        sheet.append(values)
+    for row in sheet.iter_rows():
+        for cell in row:
+            if isinstance(cell.value, str):
+                cell.data_type = "s"
+    for cell in sheet[1]:
+        cell.font = Font(bold=True)
+    sheet.freeze_panes = "A2"
+    sheet.auto_filter.ref = sheet.dimensions
+    for index, column in enumerate(sheet.columns, start=1):
+        width = max(len(str(cell.value or "")) for cell in column) + 2
+        sheet.column_dimensions[get_column_letter(index)].width = min(width, MAX_WIDTH)
+    notes_sheet = book.create_sheet(notes_title)
+    for line in notes:
+        notes_sheet.append([line])
+    notes_sheet.column_dimensions["A"].width = MAX_WIDTH * 2
+    out = BytesIO()
+    book.save(out)
+    return out.getvalue()

instances/tests/test_addresses.py

@@ -9,6 +9,7 @@ from types import SimpleNamespace
 
 from instances import addresses
 from libvirt import (
+    VIR_CONNECT_LIST_DOMAINS_ACTIVE,
     VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT,
     VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_ARP,
     VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE,
@@ -27,18 +28,20 @@ def iface(mac, *addrs):
     return {"hwaddr": mac, "addrs": [{"type": t, "addr": a, "prefix": p} for t, a, p in addrs]}
 
 
-def domain_xml(*macs, agent=None):
+def domain_xml(*macs, agent=None, osinfo=None):
     nics = "".join(f"<interface type='bridge'><mac address='{mac}'/></interface>" for mac in macs)
     channel = ("<channel type='unix'><target type='virtio' name='org.qemu.guest_agent.0' "
                f"state='{agent}'/></channel>") if agent else ""
-    return f"<domain><devices>{nics}{channel}</devices></domain>"
+    metadata = ("<metadata><libosinfo:libosinfo xmlns:libosinfo='http://libosinfo.org/xmlns/libvirt/domain/1.0'>"
+                f"<libosinfo:os id='{osinfo}'/></libosinfo:libosinfo></metadata>") if osinfo else ""
+    return f"<domain>{metadata}<devices>{nics}{channel}</devices></domain>"
 
 
 class FakeDomain:
     """answers: source -> dict, an exception to raise, or an Event to wait on (a hung agent)."""
 
-    def __init__(self, answers, uuid="uuid-a", name="vm-a", xml=None, state=1):
-        self.answers, self.uuid, self.vname, self.state_ = answers, uuid, name, state
+    def __init__(self, answers, uuid="uuid-a", name="vm-a", xml=None, state=1, os=None):
+        self.answers, self.uuid, self.vname, self.state_, self.os = answers, uuid, name, state, os
         self.xml = xml or domain_xml(MAC1, agent="connected")
 
     def interfaceAddresses(self, source):
@@ -59,12 +62,29 @@ class FakeDomain:
     def state(self):
         return [self.state_, 0]
 
+    def isActive(self):
+        return self.state_ != 5
+
+    def info(self):
+        return [self.state_, 2097152, 1048576, 2, 0]  # max memory, memory (KiB), vCPUs
+
+    def guestInfo(self, types=0, flags=0):
+        if isinstance(self.os, threading.Event):
+            self.os.wait()
+            return {}
+        if self.os is None:
+            raise libvirtError("no guest agent")
+        return self.os
+
     def XMLDesc(self, flags=0):
         return self.xml
 
 
 def compute(name, *domains, pk=1, down=False):
-    connection = SimpleNamespace(listAllDomains=lambda flags=0: list(domains))
+    def listing(flags=0):
+        return [d for d in domains if flags != VIR_CONNECT_LIST_DOMAINS_ACTIVE or d.state_ != 5]
+
+    connection = SimpleNamespace(listAllDomains=listing)
     return SimpleNamespace(id=pk, name=name, status=not down or Exception("unreachable"), connection=connection)
 
 
@@ -233,3 +253,54 @@ class FindTests(unittest.TestCase):
         result = self.find(addresses.Target(compute("c1", vm("same", MAC1, "192.0.2.10")), None),
                            addresses.Target(compute("c2", vm("same", MAC2, "192.0.2.10"), pk=2), None))
         self.assertEqual([(m["compute_id"], m["conflict"]) for m in result["matches"]], [(1, True), (2, True)])
+
+
+class InventoryTests(unittest.TestCase):
+    """Every VM of the targets, one row each, for the report."""
+
+    def test_every_vm_comes_with_its_addresses_and_os(self):
+        running = FakeDomain({ARP: {"e": iface(MAC1, (0, "192.0.2.10", 24))},
+                              AGENT: {"eth0": iface(MAC1, (0, "192.0.2.10", 24), (1, "2001:db8::10", 64))}},
+                             uuid="a", name="web", os={"os.id": "almalinux", "os.pretty-name": "AlmaLinux 10.0"})
+        off = FakeDomain({}, uuid="b", name="win", state=5,
+                         xml=domain_xml(MAC2, osinfo="http://microsoft.com/win/2k22"))
+        result = addresses.inventory([addresses.Target(compute("c1", running, off), None)])
+        web, win = result["vms"]
+        self.assertEqual((web["name"], web["state"], web["vcpus"], web["memory"]), ("web", 1, 2, 1024))
+        self.assertEqual((web["os"], web["os_family"]), ("AlmaLinux 10.0", "Linux"))
+        self.assertEqual(web["macs"], [MAC1])
+        self.assertEqual(web["addresses"], [("192.0.2.10", ["ARP", "agent"]), ("2001:db8::10", ["agent"])])
+        self.assertEqual((win["name"], win["state"], win["addresses"]), ("win", 5, []))
+        self.assertEqual((win["os"], win["os_family"]), ("win 2k22", "Windows"))
+
+    def test_an_os_without_an_agent_or_metadata_is_unknown(self):
+        dom = vm("a", MAC1, "192.0.2.10", agent=None)
+        (row,) = addresses.inventory([addresses.Target(compute("c1", dom), None)])["vms"]
+        self.assertEqual((row["os"], row["os_family"]), ("", ""))
+
+    def test_an_agent_that_reports_no_os_falls_back_to_the_metadata(self):
+        dom = FakeDomain({}, xml=domain_xml(MAC1, agent="connected", osinfo="http://almalinux.org/almalinux/9"))
+        (row,) = addresses.inventory([addresses.Target(compute("c1", dom), None)])["vms"]
+        self.assertEqual((row["os"], row["os_family"]), ("almalinux 9", "Linux"))
+
+    def test_a_down_host_and_a_late_agent_are_counted(self):
+        hung = threading.Event()
+        try:
+            stuck = FakeDomain({AGENT: hung}, uuid="hung", xml=domain_xml(MAC2, agent="connected"))
+            result = addresses.inventory([addresses.Target(compute("c1", stuck), None),
+                                          addresses.Target(compute("c2", pk=2, down=True), None)], deadline=0.3)
+            self.assertEqual((result["hosts_down"], result["unread"]), (["c2"], 1))
+            self.assertEqual([row["name"] for row in result["vms"]], ["vm-a"])
+        finally:
+            hung.set()
+
+    def test_an_os_read_that_hangs_keeps_the_addresses_read(self):
+        hung = threading.Event()
+        try:
+            dom = FakeDomain({AGENT: {"eth0": iface(MAC1, (0, "192.0.2.10", 24))}}, os=hung)
+            result = addresses.inventory([addresses.Target(compute("c1", dom), None)], deadline=0.3)
+            (row,) = result["vms"]
+            self.assertEqual(row["addresses"], [("192.0.2.10", ["agent"])])
+            self.assertEqual(result["unread"], 0)
+        finally:
+            hung.set()

instances/tests/test_vm_report.py

@@ -0,0 +1,100 @@
+"""The VM report (instances.views.vm_report): every VM the user may see, with
+its addresses and OS, as an Excel workbook."""
+
+from io import BytesIO
+from unittest.mock import patch
+
+from accounts.models import UserInstance
+from computes.models import Compute
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import Permission
+from django.contrib.messages import get_messages
+from django.test import TestCase
+from django.urls import reverse
+from instances import addresses
+from instances.models import Instance
+from lxml import html
+from openpyxl import load_workbook
+
+UUID_A = "11111111-2222-3333-4444-00000000000a"
+UUID_B = "11111111-2222-3333-4444-00000000000b"
+UNKNOWN = "11111111-2222-3333-4444-00000000000c"
+COUNTS = {"checked": 2, "no_agent": 1, "unread": 0, "hosts_down": ["host-three"], "hosts_unfinished": []}
+
+
+def vm(uuid, name, compute_id, state=1, addresses_=(), os="AlmaLinux 10.0", family="Linux"):
+    return {"compute_id": compute_id, "uuid": uuid, "name": name, "state": state, "vcpus": 2, "memory": 2048,
+            "macs": ["52:54:00:00:00:01"], "os": os, "os_family": family, "addresses": list(addresses_)}
+
+
+class VmReportTests(TestCase):
+    def setUp(self):
+        User = get_user_model()
+        self.admin = User.objects.create_superuser("vr-admin", "vr@example.com", "pw")
+        self.owner = User.objects.create_user("vr-owner", password="pw")
+        self.c1 = Compute.objects.create(name="host-one", hostname="192.0.2.1", login="u", password="p", type=1)
+        self.c2 = Compute.objects.create(name="host-two", hostname="192.0.2.2", login="u", password="p", type=1)
+        self.a = Instance.objects.create(compute=self.c1, name="vm-a", uuid=UUID_A)
+        Instance.objects.create(compute=self.c2, name="vm-b", uuid=UUID_B)
+        UserInstance.objects.create(user=self.owner, instance=self.a)
+
+    def report(self, user, vms=(), side_effect=None):
+        self.client.force_login(user)
+        with patch("instances.addresses.inventory", return_value={"vms": list(vms), **COUNTS},
+                   side_effect=side_effect) as inventory:
+            response = self.client.get(reverse("instances:vm_report"))
+        return response, inventory
+
+    def sheets(self, response):
+        book = load_workbook(BytesIO(response.content))
+        return ([list(row) for row in book["VMs"].iter_rows(values_only=True)],
+                [row[0] for row in book["Coverage"].iter_rows(values_only=True)])
+
+    def test_an_administrator_gets_every_vm_with_hosts_and_owners(self):
+        response, inventory = self.report(self.admin, [
+            vm(UUID_A, "vm-a", self.c1.id, addresses_=[("192.0.2.10", ["ARP", "agent"]), ("2001:db8::10", ["agent"]),
+                                                       ("fe80::1", ["agent"])]),  # link-local: not an address to report
+            vm(UNKNOWN, "made-with-virsh", self.c2.id, state=5, os="", family=""),
+        ])
+        self.assertEqual({(t.compute.id, t.uuids) for t in inventory.call_args.args[0]},
+                         {(self.c1.id, None), (self.c2.id, None)})
+        self.assertEqual(response["Content-Type"],
+                         "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
+        self.assertRegex(response["Content-Disposition"], r'attachment; filename="vms-\d{8}-\d{4}\.xlsx"')
+        rows, coverage = self.sheets(response)
+        self.assertEqual(rows[0], ["VM", "State", "Host", "OS", "OS family", "vCPUs", "Memory (MiB)", "IP addresses",
+                                   "Address sources", "MAC addresses", "Owners", "In the panel"])
+        self.assertEqual(rows[1], ["vm-a", "Active", "host-one", "AlmaLinux 10.0", "Linux", 2, 2048,
+                                   "192.0.2.10, 2001:db8::10", "ARP, agent", "52:54:00:00:00:01", "vr-owner", "yes"])
+        self.assertEqual(rows[2][:3] + rows[2][-1:], ["made-with-virsh", "Off", "host-two", "no"])
+        self.assertIn("Hosts not reached: host-three", coverage)
+
+    def test_only_superusers_get_it(self):
+        viewer = get_user_model().objects.create_user("vr-viewer", password="pw")
+        viewer.user_permissions.add(Permission.objects.get(codename="view_instances"))
+        for user in (self.owner, viewer):
+            with self.subTest(user=user.username):
+                response, inventory = self.report(user)
+                self.assertEqual(response.status_code, 403)
+                inventory.assert_not_called()
+
+    def test_the_administration_menu_offers_it_to_superusers_only(self):
+        for user, offered in ((self.admin, True), (self.owner, False)):
+            with self.subTest(user=user.username):
+                self.client.force_login(user)
+                page = html.fromstring(self.client.get(reverse("accounts:profile")).content)
+                links = page.xpath("//div[@aria-labelledby='administration']//a/@href")
+                self.assertEqual(reverse("instances:vm_report") in links, offered)
+
+    def test_too_many_searches_return_to_the_list(self):
+        response, _ = self.report(self.admin, side_effect=addresses.Busy)
+        self.assertRedirects(response, reverse("instances:index"), fetch_redirect_response=False)
+        self.assertIn("try again shortly", " ".join(str(m) for m in get_messages(response.wsgi_request)))
+
+    def test_text_that_looks_like_a_formula_stays_text(self):
+        # names come from VM owners and guests: Excel must not run them
+        response, _ = self.report(self.admin, [vm(UUID_A, '=HYPERLINK("http://example.com","x")', self.c1.id,
+                                                  os="+cmd|' /C calc'!A0")])
+        cells = load_workbook(BytesIO(response.content))["VMs"]["A2":"D2"][0]
+        self.assertEqual([c.data_type for c in cells], ["s", "s", "s", "s"])
+        self.assertEqual(cells[0].value, '=HYPERLINK("http://example.com","x")')

instances/urls.py

@@ -7,6 +7,7 @@ app_name = "instances"
 urlpatterns = [
     path("", views.index, name="index"),
     path("find_address/", views.find_address, name="find_address"),
+    path("report.xlsx", views.vm_report, name="vm_report"),
     path("flavor/create/", views.flavor_create, name="flavor_create"),
     path("flavor/<int:pk>/update/", views.flavor_update, name="flavor_update"),
     path("flavor/<int:pk>/delete/", views.flavor_delete, name="flavor_delete"),

instances/views.py

@@ -11,6 +11,7 @@ import socket
 import subprocess
 import time
 from bisect import insort
+from ipaddress import ip_address
 
 from accounts.models import UserInstance, UserSSHKey
 from admin.decorators import superuser_only
@@ -52,10 +53,10 @@ from vrtManager.util import randomPasswd
 
 from instances.models import Instance
 
-from . import addresses, utils
+from . import addresses, report, utils
 from .forms import ConsoleForm, FlavorForm, NewVMForm
 from .models import Flavor
-from .states import STATES, UNKNOWN
+from .states import STATES, UNKNOWN, state_label
 
 # the VM page waits this long (seconds) for the compute's drbdadm status
 DRBD_STATUS_TIMEOUT = 10
@@ -93,6 +94,24 @@ def index(request):
     )
 
 
+def _address_scope(user):
+    """What the address search and the VM report read for the user: whether
+    the user sees hosts, the registered VMs by (compute id, UUID), and the
+    targets. Administrators read every compute, VMs the panel has not
+    registered included; other users only their own VMs. A VM is its compute
+    and UUID: one UUID may be defined on two hosts."""
+    show_hosts = user.is_superuser or user.has_perm("instances.view_instances")
+    if show_hosts:
+        known = {(inst.compute_id, inst.uuid): inst for inst in Instance.objects.all()}
+        return show_hosts, known, [addresses.Target(compute, None) for compute in Compute.objects.all()]
+    known = {(inst.compute_id, inst.uuid): inst for inst in
+             Instance.objects.filter(userinstance__user=user).select_related("compute")}
+    scopes = {}
+    for inst in known.values():
+        scopes.setdefault(inst.compute_id, addresses.Target(inst.compute, set())).uuids.add(inst.uuid)
+    return show_hosts, known, list(scopes.values())
+
+
 def find_address(request):
     """
     The running VMs using an IP address or an address in a network
@@ -105,18 +124,7 @@ def find_address(request):
         network = addresses.parse_query(request.GET.get("q", ""))
     except ValueError:
         return JsonResponse({"error": _("Enter an IP address or a network such as 192.0.2.0/24.")}, status=400)
-    show_hosts = request.user.is_superuser or request.user.has_perm("instances.view_instances")
-    # a VM is its compute and UUID: one UUID may be defined on two hosts
-    if show_hosts:
-        known = {(inst.compute_id, inst.uuid): inst for inst in Instance.objects.all()}
-        targets = [addresses.Target(compute, None) for compute in Compute.objects.all()]
-    else:
-        known = {(inst.compute_id, inst.uuid): inst for inst in
-                 Instance.objects.filter(userinstance__user=request.user).select_related("compute")}
-        scopes = {}
-        for inst in known.values():
-            scopes.setdefault(inst.compute_id, addresses.Target(inst.compute, set())).uuids.add(inst.uuid)
-        targets = list(scopes.values())
+    show_hosts, known, targets = _address_scope(request.user)
     try:
         result = addresses.find(targets, network)
     except addresses.Busy:
@@ -142,6 +150,57 @@ def find_address(request):
     return JsonResponse({"matches": matches, "coverage": coverage, "read_at": timezone.now().isoformat()})
 
 
+@superuser_only
+def vm_report(request):
+    """
+    Every VM on every compute (shut-off ones and those the panel has not
+    registered too), with its state, host, OS, size, addresses and owners,
+    as an Excel workbook; a second sheet says what could not be read
+    (instances/addresses.py inventory). Linked from the Administration menu.
+    """
+    known, targets = _address_scope(request.user)[1:]
+    try:
+        result = addresses.inventory(targets)
+    except addresses.Busy:
+        messages.error(request, _("Too many searches are still running; try again shortly."))
+        return redirect("instances:index")
+
+    hosts = {target.compute.id: target.compute.name for target in targets}
+    owners = {}
+    for owner in UserInstance.objects.select_related("user", "instance"):
+        owners.setdefault((owner.instance.compute_id, owner.instance.uuid), []).append(owner.user.username)
+    header = ["VM", "State", "Host", "OS", "OS family", "vCPUs", "Memory (MiB)", "IP addresses",
+              "Address sources", "MAC addresses", "Owners", "In the panel"]
+    rows = []
+    for vm in result["vms"]:
+        key = (vm["compute_id"], vm["uuid"])
+        # a link-local address (fe80::) is not one to report: every NIC has its own
+        reported = [(address, sources) for address, sources in vm["addresses"]
+                    if not ip_address(address).is_link_local]
+        found = {source for _address, sources in reported for source in sources}
+        row = {"VM": vm["name"], "State": str(state_label(vm["state"])[0]), "Host": hosts[vm["compute_id"]],
+               "OS": vm["os"], "OS family": vm["os_family"], "vCPUs": vm["vcpus"], "Memory (MiB)": vm["memory"],
+               "IP addresses": ", ".join(address for address, _sources in reported),
+               "Address sources": ", ".join(s for s in addresses.SOURCES.values() if s in found),
+               "MAC addresses": ", ".join(vm["macs"]), "Owners": ", ".join(sorted(owners.get(key, []))),
+               "In the panel": "yes" if key in known else "no"}
+        rows.append(list(row.values()))
+
+    notes = [f"Created: {timezone.localtime():%Y-%m-%d %H:%M}",
+             f"VMs listed: {len(rows)}",
+             f"Running VMs read: {result['checked']}",
+             f"Without a connected guest agent (a static address on them cannot be seen): {result['no_agent']}",
+             f"Guest agents not read (no answer in time, or an error): {result['unread']}"]
+    if result["hosts_down"]:
+        notes.append("Hosts not reached: " + ", ".join(result["hosts_down"]))
+    if result["hosts_unfinished"]:
+        notes.append("Hosts not finished in time: " + ", ".join(result["hosts_unfinished"]))
+    response = HttpResponse(report.workbook("VMs", header, rows, "Coverage", notes),
+                            content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
+    response["Content-Disposition"] = f'attachment; filename="vms-{timezone.localtime():%Y%m%d-%H%M}.xlsx"'
+    return response
+
+
 def instance(request, pk):
     instance: Instance = get_instance(request.user, pk)
     compute: Compute = instance.compute

templates/navbar.html

@@ -33,6 +33,7 @@
             <a class="dropdown-item {% view_active request 'admin:group_list' %}" href="{% url 'admin:group_list' %}">{% bs_icon 'people-fill' %} {% trans "Groups" %}</a>
             <a class="dropdown-item {% view_active request 'admin:logs' %}" href="{% url 'admin:logs' %}">{% bs_icon 'card-list' %} {% trans "Logs" %}</a>
             <a class="dropdown-item {% view_active request 'admin:tombstone_list' %}" href="{% url 'admin:tombstone_list' %}">{% bs_icon 'trash' %} {% trans "Removed VMs" %}</a>
+            <a class="dropdown-item" href="{% url 'instances:vm_report' %}">{% bs_icon 'download' %} {% trans "VM report (.xlsx)" %}</a>
             <a class="dropdown-item {% view_active request 'appsettings' %}" href="{% url 'appsettings' %}">{% bs_icon 'gear-wide-connected' %} {% trans "Settings" %}</a>
           </div>
         </li>