diff --git a/instances/tests/test_detail_unreachable.py b/instances/tests/test_detail_unreachable.py index cc082f13..f3077d4e 100644 --- a/instances/tests/test_detail_unreachable.py +++ b/instances/tests/test_detail_unreachable.py @@ -36,6 +36,7 @@ class DetailOfUnreachableHostTests(TestCase): self.assertIn("far-vm", messages[0]) self.assertNotIn("far-host", messages[0]) self.assertNotIn("10.0.0.9", messages[0]) + self.assertIn("Jobs", messages[0]) def test_superuser_sees_the_host_and_its_error(self): admin = get_user_model().objects.create_superuser("far-admin", password="pw", email="far@example.com") diff --git a/instances/views.py b/instances/views.py index 7e178724..67cc8e91 100755 --- a/instances/views.py +++ b/instances/views.py @@ -102,7 +102,7 @@ def instance(request, pk): } else: msg = _("The host of '%(name)s' cannot be reached right now.") % {"name": instance.name} - messages.error(request, msg) + messages.error(request, msg + _(" Jobs on it can be followed under Jobs in the menu.")) return redirect(reverse("instances:index")) # The page only reads the VM, and its getters read the same XML some 30 # times: libvirt is asked once per kind of XML. diff --git a/operations/templates/operations/job.html b/operations/templates/operations/job.html new file mode 100644 index 00000000..564e923d --- /dev/null +++ b/operations/templates/operations/job.html @@ -0,0 +1,99 @@ +{% extends "base.html" %} +{% load i18n %} +{% load bootstrap_icons %} +{% load operations %} + +{% block title %}{{ job.label }}{% endblock %} + +{% block page_heading %}{{ job.label }}{% endblock page_heading %} + +{% block page_heading_extra %} +
+ {% if job.cancellable %} +
+ {% csrf_token %} + +
+ {% endif %} + {% trans "Jobs" %} +
+{% endblock page_heading_extra %} + +{% block content %} +
+
+ {% if op.error %} + + {% endif %} +
+
{% trans "Resource" %}
{{ job.resource_label }}
+
{% trans "User" %}
{{ job.username }}
+
{% trans "State" %}
+
{% include "operations/state_badge.html" with state=op.state %}
+ {% if op.phase %}
{% trans "Phase" %}
{% translate op.phase %}
{% endif %} + {% if op.progress_total %} + {% widthratio op.progress_done|default:0 op.progress_total 100 as percent %} +
{% trans "Progress" %}
+
+
+
{{ percent }}%
+
+ + {% if progress_bytes %}{{ op.progress_done|default:0|filesizeformat }} / {{ op.progress_total|filesizeformat }}{% else %}{{ op.progress_done|default:0 }} / {{ op.progress_total }}{% endif %} + +
+ {% endif %} +
{% trans "Created" %}
{{ op.created_at|date:"M d H:i:s" }}
+ {% if op.started_at %}
{% trans "Started" %}
{{ op.started_at|date:"M d H:i:s" }}
{% endif %} + {% if op.finished_at %}
{% trans "Finished" %}
{{ op.finished_at|date:"M d H:i:s" }}
{% endif %} + {% if job.result.left_behind %} +
{% trans "Left behind" %}
+
+
    + {% for item in job.result.left_behind %}
  • {{ item }}
    • {% endfor %} +
+
+ {% endif %} + {% if job.result.links %} +
{% trans "Results" %}
+
+ +
+ {% endif %} + {% if job.resolution %} +
{% trans "Resolution" %}
+
{{ job.resolution|state_label }}: {{ job.resolution_note }}
+ {% endif %} +
+ + {% if request.user.is_superuser and op.state == "uncertain" %} +
{% trans "Resolve" %}
+
+ {% csrf_token %} +
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+
+ {% endif %} +
+
+{% endblock %} diff --git a/operations/templates/operations/jobs.html b/operations/templates/operations/jobs.html new file mode 100644 index 00000000..c8b158bf --- /dev/null +++ b/operations/templates/operations/jobs.html @@ -0,0 +1,94 @@ +{% extends "base.html" %} +{% load i18n %} +{% load django_bootstrap5 %} +{% load bootstrap_icons %} +{% load operations %} + +{% block title %}{% trans "Jobs" %}{% endblock %} + +{% block page_heading %}{% trans "Jobs" %}{% endblock page_heading %} + +{% block content %} +
+
+ {% if worker_unavailable %} +
+ {% bs_icon 'exclamation-triangle' %} {% trans "Worker unavailable, last seen" %} + {% if worker_last_seen %}{{ worker_last_seen|date:"M d H:i:s" }}{% else %}{% trans "never" %}{% endif %} +
+ {% endif %} + {% if request.user.is_superuser %} + + {% endif %} + {% if not rows %} +
+ {% bs_icon 'exclamation-triangle' %} {% trans "No jobs" %} +
+ {% else %} +
+ + + + + + + + {% if request.user.is_superuser %}{% endif %} + + + + {% for op, row in rows %} + + + + + + {% if request.user.is_superuser %}{% endif %} + + {% endfor %} + +
{% trans "Created" %}{% trans "Job" %}{% trans "State" %}{% trans "Resource" %}{% trans "User" %}
{{ op.created_at|date:"M d H:i:s" }} + {{ row.label }} + {% if op.state in active_states and op.phase %} +
{% translate op.phase %}
+ {% elif op.state == "failed" or op.state == "uncertain" %} +
{{ op.error|truncatechars:80 }}
+ {% endif %} + {# below md the hidden columns show here #} +
{{ row.resource_label|wrap_at_slash }}
{{ op.created_at|date:"M d H:i:s" }}{% if request.user.is_superuser %} · {{ row.username }}{% endif %}
+ 		{% include "operations/state_badge.html" with state=op.state %}{{ row.resource_label|wrap_at_slash }}{{ row.username }}
+
+ {% if page.has_other_pages %} + {% bootstrap_pagination page extra=page_query %} + {% endif %} + {% endif %} +
+
+{% endblock %} diff --git a/operations/templates/operations/state_badge.html b/operations/templates/operations/state_badge.html new file mode 100644 index 00000000..553ffebb --- /dev/null +++ b/operations/templates/operations/state_badge.html @@ -0,0 +1 @@ +{% load operations %}{% with color=state|state_color %}{{ state|state_label }}{% endwith %} diff --git a/operations/templatetags/operations.py b/operations/templatetags/operations.py index 476b3cee..65f4d048 100644 --- a/operations/templatetags/operations.py +++ b/operations/templatetags/operations.py @@ -1,12 +1,55 @@ import uuid from django import template -from django.utils.html import format_html +from django.utils.html import format_html, format_html_join +from django.utils.safestring import mark_safe +from django.utils.translation import pgettext_lazy + +from operations.models import Operation register = template.Library() +# The Bootstrap colour of each job state, for badges and alerts; job_panel.js has the same map +STATE_COLORS = { + Operation.QUEUED: "secondary", + Operation.RUNNING: "primary", + Operation.CANCELLING: "info", + Operation.SUCCEEDED: "success", + Operation.FAILED: "danger", + Operation.CANCELLED: "secondary", + Operation.UNCERTAIN: "warning", +} + + +# What a job's state is called on the pages; the stored state stays the code +STATE_LABELS = { + Operation.QUEUED: pgettext_lazy("job state", "queued"), + Operation.RUNNING: pgettext_lazy("job state", "running"), + Operation.CANCELLING: pgettext_lazy("job state", "cancelling"), + Operation.SUCCEEDED: pgettext_lazy("job state", "succeeded"), + Operation.FAILED: pgettext_lazy("job state", "failed"), + Operation.CANCELLED: pgettext_lazy("job state", "cancelled"), + Operation.UNCERTAIN: pgettext_lazy("job state", "uncertain"), +} + @register.simple_tag def submission_key(): """A new key per rendered form: a resubmitted form returns its job.""" return format_html('', uuid.uuid4().hex) + + +@register.filter +def state_color(state): + return STATE_COLORS[state] + + +@register.filter +def state_label(state): + return STATE_LABELS[state] + + +@register.filter +def wrap_at_slash(label): + """A "/" label that wraps after its slash only, never inside a name.""" + return format_html_join(mark_safe("/"), '{}', ((part,) for part in label.split("/"))) diff --git a/operations/tests/test_views.py b/operations/tests/test_views.py new file mode 100644 index 00000000..482aa3f5 --- /dev/null +++ b/operations/tests/test_views.py @@ -0,0 +1,212 @@ +import re +from datetime import datetime, timedelta, timezone +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Permission +from django.test import SimpleTestCase, TransactionTestCase, override_settings +from django.urls import reverse +from django.utils import timezone as django_timezone +from django.utils import translation + +from operations import jobs +from operations.jobs import Outcome +from operations.models import Claim, Operation, WorkerLease +from operations.templatetags.operations import STATE_COLORS, STATE_LABELS, state_color, state_label +from operations.tests import fakes + + +class StateColorsTestCase(SimpleTestCase): + def test_every_state_has_its_colour(self): + # job_panel.test.js checks the same map in job_panel.js + self.assertEqual(STATE_COLORS, { + "queued": "secondary", "running": "primary", "cancelling": "info", "succeeded": "success", + "failed": "danger", "cancelled": "secondary", "uncertain": "warning", + }) + self.assertEqual(set(STATE_COLORS), {state for state, _label in Operation.STATES}) + self.assertEqual(state_color("failed"), "danger") + + def test_every_state_has_its_label(self): + self.assertEqual({state: str(label) for state, label in STATE_LABELS.items()}, + {state: state for state, _label in Operation.STATES}) + self.assertEqual(state_label("uncertain"), "uncertain") + + +class JobsPageTestCase(TransactionTestCase): + def setUp(self): + fakes.reset() + U = get_user_model() + self.user = U.objects.create_user("page_user", "p@example.com", "pw") + self.admin = U.objects.create_superuser("page_admin", "a@example.com", "pw") + self.op = jobs.submit("test.noop", self.user, {}, "key-00000001") + + def test_a_user_sees_own_jobs_only(self): + other = get_user_model().objects.create_user("page_other", "o@example.com", "pw") + self.client.force_login(other) + self.assertNotContains(self.client.get(reverse("operations:jobs")), "No-op") + self.assertEqual(self.client.get(reverse("operations:job", args=[self.op.pk])).status_code, 404) + self.client.force_login(self.user) + self.assertContains(self.client.get(reverse("operations:jobs")), "No-op") + + def test_superusers_filter_by_compute_and_state(self): + self.client.force_login(self.admin) + link = reverse("operations:job", args=[self.op.pk]) + self.assertContains(self.client.get(reverse("operations:jobs"), {"compute": "c"}), link) + self.assertNotContains(self.client.get(reverse("operations:jobs"), {"state": "failed"}), link) + + def test_resolve_needs_a_superuser_an_outcome_and_a_note(self): + jobs.finish(self.op, Outcome(Operation.UNCERTAIN, error="check")) + self.client.force_login(self.user) + self.assertEqual(self.client.post(reverse("operations:resolve", args=[self.op.pk]), + {"outcome": "failed", "note": "x"}).status_code, 403) + self.client.force_login(self.admin) + self.client.post(reverse("operations:resolve", args=[self.op.pk]), {"outcome": "failed", "note": ""}) + self.assertTrue(Claim.objects.filter(operation=self.op).exists()) + self.client.post(reverse("operations:resolve", args=[self.op.pk]), {"outcome": "failed", "note": "checked"}) + self.assertFalse(Claim.objects.filter(operation=self.op).exists()) + + def test_the_detail_shows_what_recovery_saw_escaped(self): + Operation.objects.filter(pk=self.op.pk).update(error="", phase="Waiting for recovery", + state=Operation.UNCERTAIN) + self.client.force_login(self.user) + response = self.client.get(reverse("operations:job", args=[self.op.pk])) + self.assertNotContains(response, "") + self.assertContains(response, "Waiting for recovery") + + def test_pagination_keeps_the_filters(self): + for i in range(51): + Operation.objects.create(kind="test.noop", user=self.user, username="page_user", + submission_key=f"page-{i:08d}", resource="r", resource_label="c/vm") + self.client.force_login(self.admin) + response = self.client.get(reverse("operations:jobs"), {"compute": "c"}) + self.assertContains(response, "compute=c") + self.assertContains(response, "page=2") + + def _pages_seen_by(self, user): + Operation.objects.filter(pk=self.op.pk).update(resource_label="wvc-test-host/wvc-test-vm") + self.client.force_login(user) + return [self.client.get(reverse("operations:jobs")), self.client.get(reverse("operations:job", args=[self.op.pk]))] + + def test_a_plain_owner_sees_only_the_vm_name(self): + for response in self._pages_seen_by(self.user): + self.assertContains(response, "wvc-test-vm") + self.assertNotContains(response, "wvc-test-host") + + def test_users_who_may_see_hosts_see_the_compute_name(self): + self.user.user_permissions.add(Permission.objects.get(codename="view_instances", + content_type__app_label="instances")) + for response in self._pages_seen_by(self.user) + self._pages_seen_by(self.admin): + self.assertContains(response, "wvc-test-host") + self.assertContains(response, "wvc-test-vm") + + def test_the_list_wraps_a_resource_after_its_slash_only(self): + Operation.objects.filter(pk=self.op.pk).update(resource_label="wvc-test-host/wvc-test-") + self.client.force_login(self.admin) + self.assertContains(self.client.get(reverse("operations:jobs")), + 'wvc-test-host/' + 'wvc-test-<vm>') + + @override_settings(TIME_ZONE="Europe/Istanbul") + def test_dates_show_in_the_app_format_and_time_zone(self): + at = datetime(2026, 10, 13, 20, 4, 0, 801375, tzinfo=timezone.utc) + Operation.objects.filter(pk=self.op.pk).update(created_at=at, started_at=at, finished_at=at, + state=Operation.SUCCEEDED) + self.client.force_login(self.admin) + for response in (self.client.get(reverse("operations:jobs")), + self.client.get(reverse("operations:job", args=[self.op.pk]))): + self.assertContains(response, "Oct 13 23:04:00") + self.assertNotContains(response, "2026-10-13T20:04") + + def test_list_and_detail_show_a_state_badge(self): + Operation.objects.filter(pk=self.op.pk).update(state=Operation.FAILED, error="disk busy") + self.client.force_login(self.admin) + for response in (self.client.get(reverse("operations:jobs")), + self.client.get(reverse("operations:job", args=[self.op.pk]))): + self.assertContains(response, 'failed') + detail = self.client.get(reverse("operations:job", args=[self.op.pk])) + self.assertContains(detail, 'class="alert alert-danger"') + + def test_the_list_shows_the_phase_while_active_and_the_error_after_a_failure(self): + Operation.objects.filter(pk=self.op.pk).update(state=Operation.RUNNING, phase="Copying", error="") + self.client.force_login(self.admin) + self.assertContains(self.client.get(reverse("operations:jobs")), "Copying") + Operation.objects.filter(pk=self.op.pk).update(state=Operation.SUCCEEDED) + self.assertNotContains(self.client.get(reverse("operations:jobs")), "Copying") + Operation.objects.filter(pk=self.op.pk).update(state=Operation.FAILED, error="x" * 200) + response = self.client.get(reverse("operations:jobs")) + self.assertNotContains(response, "Copying") + self.assertContains(response, "x" * 79 + "\u2026") + + def test_the_filters_offer_states_and_kinds_and_a_reset(self): + self.client.force_login(self.admin) + response = self.client.get(reverse("operations:jobs"), {"state": "queued", "kind": "test.noop"}) + self.assertContains(response, '') + self.assertContains(response, '') + self.assertContains(response, f'href="{reverse("operations:jobs")}"') + self.assertNotContains(response, 'class="pagination') + + def test_the_worker_warning_shows_when_a_visible_job_waits_for_a_stale_worker(self): + # the panel uses the same rule (job_panel.test.js) + stale = django_timezone.now() - timedelta(minutes=2) + WorkerLease.objects.create(name="worker", holder="h:1:a", host="h", heartbeat_at=stale) + self.client.force_login(self.user) + for state, warned in ((Operation.QUEUED, True), (Operation.RUNNING, True), (Operation.CANCELLING, True), + (Operation.UNCERTAIN, False), (Operation.SUCCEEDED, False)): + Operation.objects.filter(pk=self.op.pk).update(state=state) + response = self.client.get(reverse("operations:jobs")) + (self.assertContains if warned else self.assertNotContains)(response, "Worker unavailable", msg_prefix=state) + Operation.objects.filter(pk=self.op.pk).update(state=Operation.RUNNING) + WorkerLease.objects.update(heartbeat_at=django_timezone.now()) + self.assertNotContains(self.client.get(reverse("operations:jobs")), "Worker unavailable") + # another user's running job is not this user's concern + other = get_user_model().objects.create_user("page_other2", "o2@example.com", "pw") + WorkerLease.objects.update(heartbeat_at=stale) + self.client.force_login(other) + self.assertNotContains(self.client.get(reverse("operations:jobs")), "Worker unavailable") + + def test_an_uncertain_badge_has_dark_text(self): + Operation.objects.filter(pk=self.op.pk).update(state=Operation.UNCERTAIN) + self.client.force_login(self.admin) + # the app's theme draws white text on bg-warning + self.assertContains(self.client.get(reverse("operations:jobs")), + 'uncertain') + + def test_a_narrow_list_shows_resource_date_and_user_under_the_job(self): + Operation.objects.filter(pk=self.op.pk).update(resource_label="wvc-test-host/wvc-test-vm") + self.client.force_login(self.admin) + html = self.client.get(reverse("operations:jobs")).content.decode() + # below md only Job and State are columns + for header in ("Created", "Resource", "User"): + self.assertIn(f'{header}', html) + narrow = re.search(r'
(.*?)
', html, re.S).group(1) + for text in ('wvc-test-vm', "page_user"): + self.assertIn(text, narrow) + self.assertRegex(narrow, r"[A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2}") + # a plain user's narrow line has no user + self.client.force_login(self.user) + html = self.client.get(reverse("operations:jobs")).content.decode() + narrow = re.search(r'
(.*?)
', html, re.S).group(1) + self.assertNotIn("page_user", narrow) + + def test_states_and_phases_show_translated(self): + Operation.objects.filter(pk=self.op.pk).update(state=Operation.RUNNING, phase="Merging the snapshot") + self.client.force_login(self.admin) + fake = lambda *args: f"T[{args[-1]}]" # noqa: E731 + with patch.object(translation._trans, "gettext", side_effect=fake), \ + patch.object(translation._trans, "pgettext", side_effect=fake): + listed = self.client.get(reverse("operations:jobs")) + detail = self.client.get(reverse("operations:job", args=[self.op.pk])) + for response in (listed, detail): + self.assertContains(response, 'T[running]') + self.assertContains(response, "T[Merging the snapshot]") + self.assertContains(listed, '') + + def test_the_detail_shows_progress_in_bytes_for_kinds_that_count_bytes(self): + Operation.objects.filter(pk=self.op.pk).update(progress_done=3 * 1024 ** 3, progress_total=6 * 1024 ** 3) + self.client.force_login(self.admin) + url = reverse("operations:job", args=[self.op.pk]) + response = self.client.get(url) + self.assertContains(response, 'aria-valuenow="50"') + self.assertContains(response, str(3 * 1024 ** 3)) + with patch.object(fakes.NoopKind, "progress_bytes", True): + self.assertContains(self.client.get(url), "3.0\xa0GB / 6.0\xa0GB") diff --git a/operations/urls.py b/operations/urls.py new file mode 100644 index 00000000..13f2be32 --- /dev/null +++ b/operations/urls.py @@ -0,0 +1,11 @@ +from django.urls import path + +from . import views + +app_name = "operations" +urlpatterns = [ + path("", views.jobs, name="jobs"), + path("/", views.job, name="job"), + path("/cancel/", views.cancel, name="cancel"), + path("/resolve/", views.resolve, name="resolve"), +] diff --git a/operations/views.py b/operations/views.py new file mode 100644 index 00000000..6f07bca8 --- /dev/null +++ b/operations/views.py @@ -0,0 +1,80 @@ +from datetime import timedelta +from urllib.parse import urlencode + +from django.contrib import messages +from django.core.exceptions import PermissionDenied +from django.core.paginator import Paginator +from django.http import Http404 +from django.shortcuts import redirect, render +from django.utils import timezone +from django.utils.translation import gettext as _ +from django.views.decorators.http import require_POST +from vrtManager.util import OperationError + +from . import jobs as jobs_module +from .api.serializers import OperationSerializer +from .models import Operation, last_seen + + +def _visible(user): + return Operation.objects.all() if user.is_superuser else Operation.objects.filter(user=user) + + +def _get(user, pk): + try: + return _visible(user).get(pk=pk) + except Operation.DoesNotExist: + raise Http404 from None + + +def jobs(request): + qs = _visible(request.user) + if request.user.is_superuser: + if compute := request.GET.get("compute"): + qs = qs.filter(resource_label__startswith=f"{compute}/") + for field in ("state", "kind", "username"): + if value := request.GET.get(field): + qs = qs.filter(**{field: value}) + page = Paginator(qs, 50).get_page(request.GET.get("page")) + # the model for its datetimes, the serialized job for its labels + rows = [(op, OperationSerializer(op, context={"request": request}).data) for op in page.object_list] + seen = last_seen() + # a job the worker has to start, run or stop (job_panel.js uses the same rule) + unavailable = ((seen is None or timezone.now() - seen > timedelta(seconds=60)) + and _visible(request.user).filter(state__in=Operation.ACTIVE).exists()) + return render(request, "operations/jobs.html", + {"page": page, "rows": rows, "worker_last_seen": seen, "worker_unavailable": unavailable, + "filters": request.GET, "active_states": Operation.ACTIVE, + "states": Operation.ACTIVE + Operation.TERMINAL, + "kinds": sorted((name, _(kind.label)) for name, kind in jobs_module.KINDS.items()), + "page_query": urlencode({k: v for k, v in request.GET.items() if k != "page"})}) + + +def job(request, pk): + op = _get(request.user, pk) + kind = jobs_module.KINDS.get(op.kind) + return render(request, "operations/job.html", + {"job": OperationSerializer(op, context={"request": request}).data, "op": op, + "progress_bytes": kind is not None and kind.progress_bytes}) + + +@require_POST +def cancel(request, pk): + op = _get(request.user, pk) + try: + jobs_module.cancel(op.pk, request.user) + except OperationError as err: + messages.error(request, str(err)) + return redirect("operations:job", pk=op.pk) + + +@require_POST +def resolve(request, pk): + if not request.user.is_superuser: + raise PermissionDenied + op = _get(request.user, pk) + try: + jobs_module.resolve(op.pk, request.user, request.POST.get("outcome", ""), request.POST.get("note", "")) + except OperationError as err: + messages.error(request, str(err)) + return redirect("operations:job", pk=op.pk) diff --git a/static/icon_cache/list-task_None_None_None.svg b/static/icon_cache/list-task_None_None_None.svg new file mode 100644 index 00000000..acce647a --- /dev/null +++ b/static/icon_cache/list-task_None_None_None.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/templates/navbar.html b/templates/navbar.html index 8ac5c375..26f85f02 100644 --- a/templates/navbar.html +++ b/templates/navbar.html @@ -20,6 +20,9 @@ {% endif %}