diff --git a/computes/capacity.py b/computes/capacity.py new file mode 100644 index 00000000..d5944b22 --- /dev/null +++ b/computes/capacity.py @@ -0,0 +1,82 @@ +""" +What each compute has given its VMs: vCPUs and memory of all VMs (a +shut-off VM takes its share again when it starts) and of the running ones, +against the host's CPUs and memory, and the free space of its active +storage pools. Read live from libvirt for the Computes page. +""" + +import threading +import time +from concurrent import futures + +from libvirt import VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE, VIR_DOMAIN_SHUTOFF, libvirtError + +MAX_WORKERS = 8 + +# A libvirt call may hang: its thread outlives the page. Until it ends, its +# compute is not read again, so the threads stay at one per compute. +_reading = set() +_reading_lock = threading.Lock() + + +def read(compute): + """The compute's capacity, or None when it cannot be reached. A VM that + goes while it is read is skipped.""" + if compute.status is not True: + return None + conn = compute.connection + _arch, memory_mib, cpus, *_ = conn.getInfo() + vcpus = {"all": 0, "running": 0} + assigned = {"all": 0, "running": 0} + for dom in conn.listAllDomains(0): + try: + state, _max_memory, memory_kib, vcpu_count, _cpu_time = dom.info() + except libvirtError: + continue + for share in ("all", "running") if state != VIR_DOMAIN_SHUTOFF else ("all",): + vcpus[share] += vcpu_count + assigned[share] += memory_kib * 1024 + pools = [] + for pool in conn.listAllStoragePools(VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE): + try: + _state, size, _allocation, available = pool.info() + except libvirtError: + continue + pools.append((pool.name(), size, available)) + return {"cpus": cpus, "memory": memory_mib * 1024 * 1024, "vcpus": vcpus, "assigned": assigned, + "pools": sorted(pools)} + + +def _read_once(compute): + try: + return read(compute) + finally: + with _reading_lock: + _reading.discard(compute.id) + + +def read_all(computes, deadline=10.0): + """{compute id: capacity} of the computes read by the deadline (seconds), + in parallel; a compute down, failing, late, or still being read for an + earlier page is left out.""" + with _reading_lock: + computes = [compute for compute in computes if compute.id not in _reading] + _reading.update(compute.id for compute in computes) + pool = futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) + end = time.monotonic() + deadline + try: + reads = {pool.submit(_read_once, compute): compute.id for compute in computes} + done, _late = futures.wait(reads, timeout=max(0, end - time.monotonic())) + found = {} + for future in done: + try: + result = future.result() + except libvirtError: + continue + if result is not None: + found[reads[future]] = result + return found + finally: + pool.shutdown(wait=False, cancel_futures=True) + with _reading_lock: # a read cancelled before it started ends nothing itself + _reading.difference_update(cid for future, cid in reads.items() if future.cancelled()) diff --git a/computes/templates/computes/list.html b/computes/templates/computes/list.html index a8e4fb9e..4b418a4a 100644 --- a/computes/templates/computes/list.html +++ b/computes/templates/computes/list.html @@ -21,32 +21,70 @@ {% else %}
-
- + {# position-relative: the hidden status texts are positioned within it, not the page #} +
+
- - - - + + + + + {% for compute in computes %} + {% with given=compute.capacity %} - - - - + + {% else %} + + {% endif %} + + {% endwith %} {% endfor %}
{% trans "Name" %}{% trans "Status" %}{% trans "Details" %}{% trans "Actions" %}{% trans "Host" %}{% trans "vCPUs" %}{% trans "Memory" %}{% trans "Storage" %}
- {{ compute.name }} - - {% if compute.status is True %}{% trans "Connected" %}{% else %}{% trans "Not Connected" %} -
{{ compute.connection_error }} + 		+
+ {% if compute.status is True %} + + + {% else %} + + + {% endif %} + {{ compute.name }} +
+
{{ compute.hostname }}{% if compute.details %}
{{ compute.details }}{% endif %}
+ {% if compute.status is not True %} +
{{ compute.connection_error }}
{% endif %} 		- {{ compute.details|default:"" }} + {% if given %} + +
{{ given.ratio|floatformat:1 }}:1
+
{% blocktrans with all=given.vcpus.all cpus=given.cpus %}{{ all }} vCPUs on {{ cpus }} CPUs{% endblocktrans %}
+
{% blocktrans with running=given.vcpus.running %}{{ running }} running{% endblocktrans %}
 -
+
+
{% blocktrans with all=given.assigned.all|filesizeformat memory=given.memory|filesizeformat %}{{ all }} given of {{ memory }}{% endblocktrans %}
+ +
{% blocktrans with running=given.assigned.running|filesizeformat %}{{ running }} of it running{% endblocktrans %}
+ 		+ {% if given.pool %}{% include "computes/pool_bar.html" with pool=given.pool %}{% endif %} + {% if given.more_pools %} +
+ {% blocktrans count counter=given.more_pools|length %}{{ counter }} more pool{% plural %}{{ counter }} more pools{% endblocktrans %} + {% for pool in given.more_pools %} +
{% include "computes/pool_bar.html" %}
+ {% endfor %} +
+ {% endif %} + 		{% if compute.status is True %}{% trans "The capacity could not be read in time." %}{% else %}{% trans "No capacity while the host cannot be reached." %}{% endif %} +
{% if compute.status is True %} {% bs_icon 'eye-fill' %} {% else %} @@ -57,6 +95,7 @@
diff --git a/computes/templates/computes/pool_bar.html b/computes/templates/computes/pool_bar.html new file mode 100644 index 00000000..108b4710 --- /dev/null +++ b/computes/templates/computes/pool_bar.html @@ -0,0 +1,6 @@ +{% load i18n %} +
{{ pool.name }}
+ +
{% blocktrans with free=pool.free|filesizeformat size=pool.size|filesizeformat %}{{ free }} free of {{ size }}{% endblocktrans %}
diff --git a/computes/tests/test_capacity.py b/computes/tests/test_capacity.py new file mode 100644 index 00000000..f8b07c28 --- /dev/null +++ b/computes/tests/test_capacity.py @@ -0,0 +1,144 @@ +"""What each compute has given its VMs (computes/capacity.py) and the +Computes page that shows it.""" + +import threading +import time +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from computes import capacity +from computes.models import Compute +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.urls import reverse +from libvirt import libvirtError +from lxml import html + +GIB = 1024 ** 3 + + +def domain(state, memory_mib, vcpus): + info = [state, memory_mib * 1024, memory_mib * 1024, vcpus, 0] + return SimpleNamespace(info=lambda: info) + + +def pool(name, capacity_gib, available_gib): + return SimpleNamespace(name=lambda: name, info=lambda: [2, capacity_gib * GIB, 0, available_gib * GIB]) + + +def compute(name, domains=(), pools=(), cpus=16, memory_mib=65536, down=False, pk=1): + connection = SimpleNamespace( + getInfo=lambda: ["x86_64", memory_mib, cpus, 2400, 1, 1, 8, 2], + listAllDomains=lambda flags=0: list(domains), + listAllStoragePools=lambda flags=0: list(pools), + ) + return SimpleNamespace(id=pk, name=name, status=Exception("down") if down else True, connection=connection) + + +class ReadTests(unittest.TestCase): + def setUp(self): + capacity._reading.clear() # a stuck read of an earlier test may not have ended yet + + def test_a_host_gives_its_vms_cpus_and_memory(self): + c = compute("c1", [domain(1, 4096, 4), domain(1, 2048, 2), domain(5, 8192, 8)], + [pool("default", 500, 120), pool("iso", 100, 90)]) + result = capacity.read(c) + self.assertEqual(result["cpus"], 16) + self.assertEqual(result["memory"], 64 * GIB) + self.assertEqual(result["vcpus"], {"all": 14, "running": 6}) + self.assertEqual(result["assigned"], {"all": 14 * GIB, "running": 6 * GIB}) + self.assertEqual(result["pools"], [("default", 500 * GIB, 120 * GIB), ("iso", 100 * GIB, 90 * GIB)]) + + def test_a_vm_that_goes_meanwhile_is_skipped(self): + def gone(): + raise libvirtError("Domain not found") + + c = compute("c1", [SimpleNamespace(info=gone), domain(1, 1024, 1)]) + self.assertEqual(capacity.read(c)["vcpus"], {"all": 1, "running": 1}) + + def test_a_down_host_has_no_capacity(self): + self.assertIsNone(capacity.read(compute("c1", down=True))) + + def test_hosts_are_read_together_by_the_deadline(self): + slow = threading.Event() + try: + stuck = compute("c2", pk=2) + stuck.connection.getInfo = lambda: slow.wait() and None + started = time.monotonic() + found = capacity.read_all([compute("c1"), stuck], deadline=0.3) + self.assertLess(time.monotonic() - started, 1) + self.assertEqual(set(found), {1}) + finally: + slow.set() + + + def test_a_host_still_being_read_is_not_read_again(self): + # a stuck read keeps its thread; a second page load must not add another + slow, calls = threading.Event(), [] + try: + stuck = compute("c2", pk=2) + stuck.connection.getInfo = lambda: calls.append(1) or slow.wait() and None + for _ in range(2): + self.assertEqual(set(capacity.read_all([compute("c1"), stuck], deadline=0.2)), {1}) + self.assertEqual(len(calls), 1) + finally: + slow.set() + + + def test_a_host_whose_read_never_started_is_read_next_time(self): + # more hosts than threads: a read still queued at the deadline is cancelled + slow = threading.Event() + try: + stuck = compute("c1") + stuck.connection.getInfo = lambda: slow.wait() and None + with patch.object(capacity, "MAX_WORKERS", 1): + self.assertEqual(capacity.read_all([stuck, compute("c2", pk=2)], deadline=0.2), {}) + self.assertEqual(set(capacity.read_all([compute("c2", pk=2)], deadline=1)), {2}) + finally: + slow.set() + + +class ComputesPageTests(TestCase): + def test_the_list_shows_what_each_compute_has_given(self): + admin = get_user_model().objects.create_superuser("cap-admin", "cap@example.com", "pw") + self.client.force_login(admin) + c1 = Compute.objects.create(name="host-one", hostname="192.0.2.1", login="u", password="p", type=1) + Compute.objects.create(name="host-two", hostname="192.0.2.2", login="u", password="p", type=1) + found = {c1.id: {"cpus": 16, "memory": 64 * GIB, "vcpus": {"all": 40, "running": 24}, + "assigned": {"all": 48 * GIB, "running": 32 * GIB}, + "pools": [("default", 500 * GIB, 120 * GIB)]}} + with patch("computes.views.capacity.read_all", return_value=found), \ + patch.object(Compute, "status", new=True): + page = html.fromstring(self.client.get(reverse("computes")).content) + headers = [" ".join(th.text_content().split()) for th in page.xpath("//thead//th")] + self.assertEqual(headers, ["Host", "vCPUs", "Memory", "Storage", "Actions"]) + one, two = page.xpath("//tbody/tr") + cells = [" ".join(td.text_content().split()) for td in one.xpath("td")] + self.assertIn("host-one", cells[0]) + self.assertIn("2.5:1", cells[1]) + self.assertIn("40 vCPUs on 16 CPUs 24 running", cells[1]) + self.assertIn("48.0 GB given of 64.0 GB", cells[2]) + self.assertIn("32.0 GB of it running", cells[2]) + self.assertIn("120.0 GB free of 500.0 GB", cells[3]) + # host-two answered nothing in time: one cell says so + self.assertIn("could not be read in time", " ".join(two.text_content().split())) + + def test_the_default_pool_shows_and_the_others_open_on_a_click(self): + admin = get_user_model().objects.create_superuser("cap-pools", "pools@example.com", "pw") + self.client.force_login(admin) + c1 = Compute.objects.create(name="host-one", hostname="192.0.2.1", login="u", password="p", type=1) + pools = [("backup", 900 * GIB, 800 * GIB), ("default", 500 * GIB, 120 * GIB), + ("iso", 100 * GIB, 90 * GIB), ("nfs", 2000 * GIB, 1000 * GIB)] + found = {c1.id: {"cpus": 16, "memory": 64 * GIB, "vcpus": {"all": 8, "running": 8}, + "assigned": {"all": 8 * GIB, "running": 8 * GIB}, "pools": pools}} + with patch("computes.views.capacity.read_all", return_value=found), \ + patch.object(Compute, "status", new=True): + page = html.fromstring(self.client.get(reverse("computes")).content) + (storage,) = page.xpath("//tbody/tr/td[4]") + shown = " ".join(" ".join(storage.xpath("./div//text()")).split()) + self.assertIn("default", shown) + self.assertNotIn("backup", shown) + (more,) = storage.xpath(".//details") + self.assertEqual(" ".join(more.xpath("summary")[0].text_content().split()), "3 more pools") + self.assertEqual(more.xpath(".//div[@class='small']/text()"), ["backup", "iso", "nfs"]) diff --git a/computes/views.py b/computes/views.py index 223cd35c..f51e13dc 100644 --- a/computes/views.py +++ b/computes/views.py @@ -23,7 +23,7 @@ from computes.forms import ( ) from computes.models import Compute -from . import utils +from . import capacity, utils # The host CPU graph sends back the CPU sample of its previous answer; usage # is taken since then when it is this recent, otherwise over one second. @@ -33,13 +33,27 @@ CPU_SAMPLE_SIGNER = signing.TimestampSigner(salt="computes.cpu-sample") @superuser_only def computes(request): - """ - :param request: - :return: - """ - - computes = Compute.objects.filter().order_by("name") - + """The computes, with what each has given its VMs (computes/capacity.py).""" + computes = list(Compute.objects.order_by("name")) + found = capacity.read_all(computes) + for compute in computes: + compute.capacity = given = found.get(compute.id) + if given: + given["ratio"] = given["vcpus"]["all"] / given["cpus"] if given["cpus"] else 0 + # the memory bar: running and shut-off VMs' memory, and a mark at the + # host's memory, on a scale that holds whichever is larger + assigned, memory = given["assigned"], given["memory"] + scale = max(assigned["all"], memory) or 1 + given["bar"] = {"running": assigned["running"] * 100 / scale, + "stopped": (assigned["all"] - assigned["running"]) * 100 / scale, + "host": memory * 100 / scale, "over": assigned["all"] > memory} + given["memory_percent"] = round(assigned["all"] * 100 / memory) if memory else 0 + pools = [{"name": name, "size": size, "free": free, + "used_percent": (size - free) * 100 / size if size else 0} + for name, size, free in given["pools"]] + # the default pool shows; the others (a host may have dozens) open on a click + given["pool"] = next((pool for pool in pools if pool["name"] == "default"), pools[0] if pools else None) + given["more_pools"] = [pool for pool in pools if pool is not given["pool"]] return render(request, "computes/list.html", {"computes": computes}) diff --git a/static/css/webvirtcloud.css b/static/css/webvirtcloud.css index 59766dca..e3627f96 100644 --- a/static/css/webvirtcloud.css +++ b/static/css/webvirtcloud.css @@ -222,4 +222,65 @@ a { font-size: 12px; line-height: 1.5; /* If Placeholder of the input is moved up, rem/modify this. */ border-radius: 2px; - } \ No newline at end of file + } + +/* the Computes list: what each host has given its VMs */ +.compute-list td { + padding-top: 1rem; + padding-bottom: 1rem; +} + +.compute-dot { + display: inline-block; + width: 0.6rem; + height: 0.6rem; + border-radius: 50%; + flex-shrink: 0; +} + +.compute-ratio { + font-size: 1.5rem; + line-height: 1.2; + font-variant-numeric: tabular-nums; +} + +.compute-memory { + min-width: 11rem; +} + +.compute-storage { + min-width: 9rem; +} + +/* a bar of given capacity: running VMs solid, shut-off VMs faint, and a + mark where the host's own capacity ends */ +.cap-bar { + position: relative; + display: flex; + height: 0.6rem; + border-radius: 0.3rem; + background: var(--bs-secondary-bg); +} + +.cap-bar.cap-thin { + height: 0.3rem; +} + +.cap-running { + background: var(--bs-primary); + border-radius: 0.3rem 0 0 0.3rem; +} + +.cap-stopped { + background: var(--bs-primary); + opacity: 0.35; +} + +.cap-host { + position: absolute; + top: -0.2rem; + bottom: -0.2rem; + width: 2px; + margin-left: -1px; + background: var(--bs-body-color); +}