feat(operations): job panel on the VM page

instances/templates/instance.html

@@ -64,6 +64,7 @@
         <hr>
     </div>
 
+    <div id="job-panel" aria-live="polite"></div>
     <div id="mainTabMenu">
         <!-- Nav tabs -->
         <ul class="nav nav-pills nav-fill" id="navbtn" role="tablist" aria-label="Instance actions">
@@ -681,6 +682,14 @@
         return true;
     });
 </script>
+<script src="{% static 'operations/job_panel.js' %}"></script>
+<script>
+    jobPanel(document.getElementById("job-panel"), {
+        resource: "{% url 'operation-resource' %}?key=instance%3A{{ instance.uuid|urlencode }}",
+        job: function (id) { return "{% url 'operation-list' %}" + id + "/"; },
+        cancel: function (id) { return "{% url 'operation-list' %}" + id + "/cancel/"; },
+    }, "{{ csrf_token }}", {{ remembered_jobs|safe }});
+</script>
 <script src="{% static 'js/instance_tabs.js' %}"></script>
 <script>
     // Open the tab whose control targets "#<pane id>"

instances/templates/instances/snapshots_tab.html

@@ -25,9 +25,7 @@
     <div class="tab-content">
         {% if busy %}
             <div class="alert alert-warning mt-3" role="status">
-                {% trans "A snapshot job is running:" %}
-                {% for job in busy %}{{ job.dev }} {% widthratio job.cur job.end 100 %}%{% if not forloop.last %}, {% endif %}{% endfor %}.
-                {% trans "Reload the page to see its progress." %}
+                {% trans "A disk job is running on this VM; snapshot actions are unavailable until it ends." %}
             </div>
         {% endif %}
         <div role="tabpanel" class="tab-pane tab-pane-bordered active" id="takesnapshot">

instances/tests/test_page_controls.py

@@ -130,6 +130,21 @@ class PageControlsTestCase(PageControls, TestCase):
             | {"clone", "snapshots", "console settings", "destroy", "host links"},
         )
 
+    def test_the_job_panel_gets_the_jobs_just_submitted_once(self):
+        superuser = get_user_model().objects.create_superuser("ctl_jobs", "ctl_jobs@example.com", "x")
+        self.client.force_login(superuser)
+        session = self.client.session
+        other = {"resource": "instance:aaaaaaaa-bbbb-cccc-0000-111111111111", "id": "4343"}
+        session["operations_shown"] = [{"resource": f"instance:{self.instance.uuid}", "id": "4242"}, other]
+        session.save()
+        self.controls(superuser)
+        self.assertIn('["4242"]', self.last_page)
+        self.assertNotIn('"4343"', self.last_page)
+        # a job remembered for another VM waits for that VM's page
+        self.assertEqual(self.client.session["operations_shown"], [other])
+        self.controls(superuser)
+        self.assertNotIn('"4242"', self.last_page)
+
     def test_global_viewers_see_nothing_to_act_on(self):
         perms = ("view_instances", "clone_instances", "snapshot_instances")
         for user in (self.user("ctl_viewer", perms=perms), self.user("ctl_staff", staff=True, perms=perms)):

instances/tests/test_snapshot_views.py

@@ -191,9 +191,13 @@ class SnapshotTabTestCase(TestCase):
         self.assertFalse(self.page(status=1).xpath("//input[@name='memory']")[0].get("disabled"))
         self.assertIsNotNone(self.page(status=5).xpath("//input[@name='memory']")[0].get("disabled"))
 
-    def test_a_running_job_shows_progress_and_disables_actions(self):
+    def test_a_running_job_disables_actions(self):
+        # its progress shows in the job panel, not in the tab; the tab says why the actions are off
+        line = "A disk job is running on this VM; snapshot actions are unavailable until it ends."
         doc = self.page(external=[self.snap("e")], jobs=[{"dev": "vda", "cur": 1, "end": 4}])
-        self.assertIn("vda 25%", doc.text_content())
+        self.assertIn(line, doc.text_content())
+        self.assertNotIn("vda 25%", doc.text_content())
+        self.assertNotIn(line, self.page(external=[self.snap("e")]).text_content())
         self.assertTrue(all(b.get("disabled") is not None
                             for b in doc.xpath("//div[@id='snapshots']//button[@type='submit']")))
 

instances/views.py

@@ -205,6 +205,7 @@ def _instance_page(request, pk, instance, compute):
         instance.drbd = drbd_status(request, pk)
         instance.save()
 
+    remembered_jobs = json.dumps(jobs.take_remembered(request, claims.instance_key(instance.uuid)))
     return render(request, "instance.html", locals(),)
 
 

operations/jobs.py

@@ -252,9 +252,14 @@ def resolve(op_id, user, outcome_state, note):
 
 def remember_job(request, op):
     """The resource page's panel shows this job even if it ends before the first poll."""
-    ids = request.session.get("operations_shown", [])
-    request.session["operations_shown"] = [*ids, str(op.pk)][-20:]
+    entries = request.session.get("operations_shown", [])
+    request.session["operations_shown"] = [*entries, {"resource": op.resource, "id": str(op.pk)}][-20:]
 
 
-def take_remembered(request):
-    return request.session.pop("operations_shown", [])
+def take_remembered(request, resource):
+    """The ids remembered for resource; the other resources' entries stay."""
+    entries = request.session.get("operations_shown", [])
+    taken = [e["id"] for e in entries if e["resource"] == resource]
+    if taken:  # untouched otherwise: no session write on every page view
+        request.session["operations_shown"] = [e for e in entries if e["resource"] != resource]
+    return taken

operations/static/operations/job_panel.js

@@ -0,0 +1,173 @@
+// The job panel of a resource page (operations framework). It polls the
+// resource's jobs, shows progress and outcomes, and keeps the page's own
+// status poll from reloading (backgroundJobRunning) while it shows anything.
+
+// The Bootstrap colour of each job state, for badges and alerts; the jobs
+// pages use the same map (operations/templatetags/operations.py)
+var JOB_STATE_COLORS = {
+    queued: "secondary", running: "primary", cancelling: "info", succeeded: "success",
+    failed: "danger", cancelled: "secondary", uncertain: "warning",
+};
+
+function jobPanel(element, urls, csrfToken, initialIds) {
+    var shown = {};     // id -> job in the last active list
+    var pending = {};   // id -> true: left the list (or just submitted), outcome not fetched yet
+    var inflight = {};  // id -> true: its outcome is being fetched
+    var dismissed = {}; // id -> true: never shown again
+    var outcomes = {};  // id -> finished job, until dismissed
+    var cancelling = {}; // id -> true: its cancel request runs
+    var notice = "";    // why the last cancel failed, until dismissed
+    (initialIds || []).forEach(function (id) { pending[id] = true; });
+    var last = { jobs: [], others: [], short_operation: false, worker_last_seen: null };
+    var rendered = null; // the panel's last html: an unchanged one is not rebuilt (focus stays)
+    var holding = false; // the panel set backgroundJobRunning (the please-wait dialog sets it too)
+    if (initialIds && initialIds.length) {
+        // a remembered job must not be lost to a status reload before the first poll answers
+        window.backgroundJobRunning = true;
+        holding = true;
+    }
+
+    function esc(text) {
+        return String(text === null || text === undefined ? "" : text)
+            .replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+    }
+
+    function bar(job) {
+        if (!job.progress_total) { return ""; }
+        var pct = Math.floor(100 * job.progress_done / job.progress_total);
+        // libvirt takes a while to finish the last percent: a full moving bar, not a stuck number
+        var finishing = job.state === "running" && pct >= 99;
+        return '<div class="progress mt-2" role="progressbar" aria-valuenow="' + (finishing ? 100 : pct)
+            + '" aria-valuemin="0" aria-valuemax="100"><div class="progress-bar'
+            + (finishing ? " progress-bar-striped progress-bar-animated" : "") + '" style="width:'
+            + (finishing ? 100 : pct) + '%">' + (finishing ? "Finishing\u2026" : pct + "%") + "</div></div>";
+    }
+
+    // label, state badge and muted phase on one line, the buttons on its right
+    function header(label, state, phase, buttons) {
+        return '<div class="d-flex flex-wrap align-items-center gap-2"><strong>' + esc(label) + "</strong>"
+            + '<span class="badge bg-' + JOB_STATE_COLORS[state]
+            // the theme draws white text on a warning badge
+            + (JOB_STATE_COLORS[state] === "warning" ? " text-dark" : "") + '">' + esc(state) + "</span>"
+            + (phase ? '<span class="small text-body-secondary">' + esc(phase) + "</span>" : "")
+            + (buttons ? '<div class="ms-auto d-flex gap-2">' + buttons + "</div>" : "") + "</div>";
+    }
+
+    function alertHtml(state, body) {
+        return '<div class="alert alert-' + JOB_STATE_COLORS[state] + ' mb-3" role="status">' + body + "</div>";
+    }
+
+    function errorHtml(job) {
+        return job.error ? '<div class="small mt-2">' + esc(job.error) + "</div>" : "";
+    }
+
+    function jobHtml(job) {
+        var cancel = job.cancellable ? '<button type="button" class="btn btn-sm btn-outline-danger" data-cancel="'
+            + esc(job.id) + '"' + (cancelling[job.id] ? " disabled" : "") + ">Cancel</button>" : "";
+        // an uncertain job keeps its claims, so it stays in the list with its error
+        return alertHtml(job.state, header(job.label, job.state, job.phase, cancel) + errorHtml(job) + bar(job));
+    }
+
+    function outcomeHtml(job) {
+        var left = (job.result && job.result.left_behind) || [];
+        var warn = left.length ? '<div class="small mt-2">Files not removed:</div><ul class="list-unstyled small mb-0">'
+            + left.map(function (path) { return '<li class="text-break"><code>' + esc(path) + "</code></li>"; }).join("") + "</ul>" : "";
+        var buttons = '<button type="button" class="btn btn-sm btn-primary" data-refresh="1">Refresh</button>'
+            + '<button type="button" class="btn btn-sm btn-outline-secondary" data-dismiss="' + esc(job.id)
+            + '">Dismiss</button>';
+        return alertHtml(job.state, header(job.label, job.state, "", buttons) + errorHtml(job) + warn);
+    }
+
+    function render() {
+        var html = notice ? alertHtml("failed", '<div class="d-flex flex-wrap align-items-center gap-2"><span>'
+            + esc(notice) + '</span><div class="ms-auto"><button type="button" class="btn btn-sm btn-outline-secondary"'
+            + ' data-dismiss-error="1">Dismiss</button></div></div>') : "";
+        last.jobs.forEach(function (job) { html += jobHtml(job); });
+        Object.keys(outcomes).forEach(function (id) { html += outcomeHtml(outcomes[id]); });
+        last.others.forEach(function (o) {
+            html += '<div class="alert alert-secondary mb-3" role="status">' + header(o.label, o.state, "", "") + "</div>";
+        });
+        if (last.short_operation) {
+            html += '<div class="alert alert-secondary mb-3" role="status">A short operation is in progress.</div>';
+        }
+        // a job the worker has to start, run or stop (the Jobs page uses the same rule)
+        var waiting = last.jobs.some(function (j) { return ["queued", "running", "cancelling"].indexOf(j.state) >= 0; });
+        var seen = last.worker_last_seen ? new Date(last.worker_last_seen) : null;
+        if (waiting && (!seen || Date.now() - seen.getTime() > 60000)) {
+            html += '<div class="alert alert-warning mb-3" role="status">Worker unavailable, last seen '
+                + esc(seen ? seen.toLocaleString() : "never") + ".</div>";
+        }
+        if (html !== rendered) {
+            element.innerHTML = html;
+            rendered = html;
+        }
+        var busy = html !== "" || Object.keys(pending).length > 0;
+        if (busy) {
+            window.backgroundJobRunning = true;
+        } else if (holding) {
+            window.backgroundJobRunning = false;
+        }
+        holding = busy;
+    }
+
+    function fetchOutcome(id) {
+        if (inflight[id]) { return; }
+        inflight[id] = true;
+        $.getJSON(urls.job(id))
+            .done(function (job) {
+                delete pending[id];
+                if (dismissed[id]) { return; }
+                // a just-submitted job that is still active shows in the list instead
+                if (["queued", "running", "cancelling"].indexOf(job.state) < 0) { outcomes[id] = job; }
+                render();
+            })
+            .fail(function (xhr) {
+                // not allowed or gone: asking again cannot help; a network error or 5xx can pass
+                if (xhr.status === 403 || xhr.status === 404) {
+                    delete pending[id];
+                    render();
+                }
+            })
+            .always(function () { delete inflight[id]; });
+    }
+
+    function onData(data) {
+        var now = {};
+        data.jobs.forEach(function (job) { now[job.id] = job; });
+        Object.keys(shown).forEach(function (id) { if (!now[id]) { pending[id] = true; } });
+        Object.keys(now).forEach(function (id) { delete pending[id]; });
+        shown = now;
+        last = data;
+        Object.keys(pending).forEach(fetchOutcome);
+        render();
+    }
+
+    element.addEventListener("click", function (ev) {
+        var t = ev.target;
+        if (t.dataset.refresh) { window.location.reload(); }
+        if (t.dataset.dismiss) { dismissed[t.dataset.dismiss] = true; delete outcomes[t.dataset.dismiss]; render(); }
+        if (t.dataset.dismissError) { notice = ""; render(); }
+        if (t.dataset.cancel && !cancelling[t.dataset.cancel]) { cancel(t.dataset.cancel); }
+    });
+
+    function cancel(id) {
+        cancelling[id] = true;
+        notice = "";
+        render();
+        $.ajax({ url: urls.cancel(id), type: "POST", headers: { "X-CSRFToken": csrfToken } })
+            .fail(function (xhr) {
+                // e.g. a queued job started meanwhile and cannot be stopped any more
+                notice = (xhr.responseJSON && xhr.responseJSON.detail)
+                    || (xhr.status ? "Cancel failed (HTTP " + xhr.status + ")." : "Cancel failed: the server could not be reached.");
+            })
+            .always(function () {
+                delete cancelling[id];
+                render();
+                poller.stop(); // ask for the list now
+                poller.start();
+            });
+    }
+
+    var poller = pollWhileVisible(3000, function () { return $.getJSON(urls.resource); }, onData, 0);
+    return poller;
+}

operations/tests/js/job_panel.test.js

@@ -0,0 +1,224 @@
+// operations/tests/js/job_panel.test.js
+// node job_panel.test.js static/js/poll.js operations/static/operations/job_panel.js
+const fs = require("fs");
+const assert = require("assert");
+
+let now = 0, timers = [], nextId = 1;
+global.setTimeout = (fn, ms) => { const id = nextId++; timers.push({ id, at: now + (ms || 0), fn }); return id; };
+global.clearTimeout = (id) => { timers = timers.filter((t) => t.id !== id); };
+function advance(ms) {
+    const end = now + ms;
+    for (;;) {
+        timers.sort((a, b) => a.at - b.at);
+        const t = timers[0];
+        if (!t || t.at > end) break;
+        timers.shift(); now = t.at; t.fn();
+    }
+    now = end;
+}
+const realNow = Date.now;
+Date.now = () => 1700000000000 + now;
+global.document = { hidden: false, addEventListener() {} };
+global.window = { backgroundJobRunning: false, location: { reload() { window.reloaded = true; } } };
+
+// requests settle only when the test says so; callbacks run in registration order
+let open = [];
+function req(url) {
+    const cbs = [];
+    const r = { url, settled: false,
+        always(f) { cbs.push(["always", f]); return r; },
+        done(f) { cbs.push(["done", f]); return r; },
+        fail(f) { cbs.push(["fail", f]); return r; },
+        settle(ok, value) {
+            r.settled = true;
+            cbs.forEach(([t, f]) => { if (t === "always" || t === (ok ? "done" : "fail")) f(value); });
+        } };
+    open.push(r);
+    return r;
+}
+// a failed request passes a jqXHR-like {status: N}
+function answer(url, value, ok = true) {
+    const r = open.find((x) => x.url === url && !x.settled);
+    assert.ok(r, "no open request to " + url);
+    r.settle(ok, value);
+}
+global.$ = { getJSON: req, ajax: (o) => req(o.url) };
+let listeners = {};
+const el = { innerHTML: "", addEventListener(ev, f) { listeners[ev] = f; } };
+eval(fs.readFileSync(process.argv[2], "utf8"));
+eval(fs.readFileSync(process.argv[3], "utf8"));
+
+const urls = { resource: "/r", job: (id) => "/j/" + id, cancel: (id) => "/c/" + id };
+const seen = new Date(1700000000000).toISOString();
+const running = { id: "a", label: "Delete snapshot", state: "running", phase: "Merging", progress_done: 1,
+                  progress_total: 4, cancellable: false, result: {}, error: "" };
+const empty = { jobs: [], others: [], short_operation: false, worker_last_seen: seen };
+
+jobPanel(el, urls, "tok");
+advance(0);
+answer("/r", { ...empty, jobs: [running] });
+assert.ok(el.innerHTML.includes("Delete snapshot") && el.innerHTML.includes("25%"), "shows the job and real progress");
+
+// one colour per state (operations/tests/test_views.py checks the same map in the template filter)
+assert.deepStrictEqual(JOB_STATE_COLORS, { queued: "secondary", running: "primary", cancelling: "info",
+    succeeded: "success", failed: "danger", cancelled: "secondary", uncertain: "warning" });
+assert.ok(el.innerHTML.includes('class="alert alert-primary') && el.innerHTML.includes('<span class="badge bg-primary">running</span>')
+    && el.innerHTML.includes('<span class="small text-body-secondary">Merging</span>'), "state badge and muted phase");
+assert.strictEqual(window.backgroundJobRunning, true);
+
+assert.ok(!el.innerHTML.includes("Finishing"), "a number below 99 %");
+
+// from 99 % the bar is full, striped, animated and says Finishing
+advance(3000);
+answer("/r", { ...empty, jobs: [{ ...running, progress_done: 99, progress_total: 100 }] });
+assert.ok(el.innerHTML.includes("progress-bar-striped progress-bar-animated"), "striped and animated");
+assert.ok(el.innerHTML.includes("Finishing\u2026") && el.innerHTML.includes('aria-valuenow="100"')
+    && el.innerHTML.includes("width:100%") && !el.innerHTML.includes("99%"), "full width, no number");
+advance(3000);
+answer("/r", { ...empty, jobs: [{ ...running, state: "cancelling", progress_done: 99, progress_total: 100 }] });
+assert.ok(!el.innerHTML.includes("Finishing") && el.innerHTML.includes("99%"), "only a running job is finishing");
+advance(3000);
+answer("/r", { ...empty, jobs: [running] });
+assert.ok(!el.innerHTML.includes("Finishing"), "back to a number");
+
+// the next poll goes out 3 s after the answer
+advance(3000);
+answer("/r", { ...empty, jobs: [running] });
+
+// the job finishes between polls: the panel asks for it by id, keeps the flag meanwhile
+advance(3000);
+answer("/r", empty);
+assert.strictEqual(window.backgroundJobRunning, true, "still true while the outcome is fetched");
+answer("/j/a", { status: 0 }, false); // a network error: asked again on the next poll
+advance(3000);
+answer("/r", empty);
+answer("/j/a", { ...running, state: "succeeded", result: { left_behind: ["/pool/a.one"] } });
+assert.ok(el.innerHTML.includes('<li class="text-break"><code>/pool/a.one</code></li>'), "shows the warning from the result");
+assert.ok(el.innerHTML.includes('class="alert alert-success') && el.innerHTML.includes('<span class="badge bg-success">succeeded</span>')
+    && el.innerHTML.includes('<div class="ms-auto d-flex gap-2"><button type="button" class="btn btn-sm btn-primary" data-refresh'),
+    "the outcome has the state colour and its buttons on the right");
+assert.strictEqual(window.backgroundJobRunning, true, "keeps backgroundJobRunning until dismissed");
+
+listeners.click({ target: { dataset: { dismiss: "a" } } });
+assert.strictEqual(window.backgroundJobRunning, false);
+assert.ok(!window.reloaded);
+
+// a job that ended before the first poll: shown from the submitted ids
+open = []; timers = [];
+const el2 = { innerHTML: "", addEventListener() {} };
+jobPanel(el2, urls, "tok", ["b"]);
+advance(0);
+answer("/r", empty);
+answer("/j/b", { ...running, id: "b", state: "failed", error: "refused" });
+assert.ok(el2.innerHTML.includes("refused") && el2.innerHTML.includes('class="alert alert-danger'),
+    "shows a job that finished before the first poll");
+
+// a late answer for a dismissed job does not bring it back
+const el3 = { innerHTML: "", addEventListener(ev, f) { listeners3 = f; } };
+let listeners3;
+jobPanel(el3, urls, "tok", ["c"]);
+advance(0);
+answer("/r", empty);
+listeners3({ target: { dataset: { dismiss: "c" } } });
+answer("/j/c", { ...running, id: "c", state: "succeeded" });
+assert.ok(!el3.innerHTML.includes("Delete snapshot"), "a dismissed outcome stays dismissed");
+
+// an uncertain job stays in the list (it keeps its claims): its error shows
+open = []; timers = [];
+let writes = 0, html4 = "";
+const el4 = { addEventListener() {},
+              get innerHTML() { return html4; }, set innerHTML(v) { writes++; html4 = v; } };
+jobPanel(el4, urls, "tok");
+advance(0);
+answer("/r", { ...empty, jobs: [{ ...running, state: "uncertain", error: "check the disks" }] });
+assert.ok(el4.innerHTML.includes('<span class="badge bg-warning text-dark">uncertain</span>'), "dark text on the yellow badge");
+assert.ok(el4.innerHTML.includes("check the disks") && el4.innerHTML.includes('class="alert alert-warning'),
+    "an uncertain job in the list shows its error");
+// an unchanged answer does not rebuild the panel (a focused button keeps its focus)
+advance(3000);
+answer("/r", { ...empty, jobs: [{ ...running, state: "uncertain", error: "check the disks" }] });
+assert.strictEqual(writes, 1, "an unchanged answer leaves the panel alone");
+
+// the panel showing nothing leaves a flag the please-wait dialog set alone
+open = []; timers = [];
+jobPanel({ innerHTML: "", addEventListener() {} }, urls, "tok");
+window.backgroundJobRunning = true;
+advance(0);
+answer("/r", empty);
+assert.strictEqual(window.backgroundJobRunning, true, "a foreground operation keeps its guard");
+
+// a job the user may not see or that is gone (403/404) is not asked for again
+open = []; timers = [];
+const el6 = { innerHTML: "", addEventListener() {} };
+window.backgroundJobRunning = false;
+jobPanel(el6, urls, "tok");
+advance(0);
+answer("/r", { ...empty, jobs: [{ ...running, id: "d" }] });
+assert.strictEqual(window.backgroundJobRunning, true);
+advance(3000);
+answer("/r", empty);
+answer("/j/d", { status: 404 }, false);
+assert.strictEqual(el6.innerHTML, "", "a missing job leaves nothing behind");
+assert.strictEqual(window.backgroundJobRunning, false, "the guard goes back once the panel holds nothing");
+advance(3000);
+answer("/r", empty);
+assert.ok(!open.some((r) => r.url === "/j/d" && !r.settled), "a missing job is not asked for again");
+// remembered ids hold the page's own reload at once, before the first poll answers
+open = []; timers = [];
+window.backgroundJobRunning = false;
+const el7 = { innerHTML: "", addEventListener() {} };
+jobPanel(el7, urls, "tok", ["e"]);
+assert.strictEqual(window.backgroundJobRunning, true, "set at init, not after the first poll");
+advance(0);
+answer("/r", empty, false); // a failed first poll keeps it
+assert.strictEqual(window.backgroundJobRunning, true, "kept after a failed first poll");
+advance(3000);
+answer("/r", empty);
+answer("/j/e", { status: 404 }, false);
+assert.strictEqual(window.backgroundJobRunning, false, "released once the remembered job is settled");
+
+// a running (not only a queued) job with a stale lease shows the worker warning
+open = []; timers = [];
+const el8 = { innerHTML: "", addEventListener() {} };
+const stale = new Date(Date.now() - 120000).toISOString();
+jobPanel(el8, urls, "tok");
+advance(0);
+answer("/r", { ...empty, worker_last_seen: stale, jobs: [running] });
+assert.ok(el8.innerHTML.includes("Worker unavailable"), "a running job with a stale lease warns");
+advance(3000);
+answer("/r", { ...empty, worker_last_seen: stale, jobs: [{ ...running, state: "cancelling" }] });
+assert.ok(el8.innerHTML.includes("Worker unavailable"), "a cancelling job with a stale lease warns");
+advance(3000);
+answer("/r", { ...empty, worker_last_seen: new Date(Date.now()).toISOString(), jobs: [running] });
+assert.ok(!el8.innerHTML.includes("Worker unavailable"), "a fresh lease does not warn");
+advance(3000);
+answer("/r", { ...empty, worker_last_seen: stale, jobs: [{ ...running, state: "uncertain" }] });
+assert.ok(!el8.innerHTML.includes("Worker unavailable"), "an uncertain job waits for an administrator, not the worker");
+
+// cancel: the button is disabled while the request runs; a refusal or a network error shows; the list is asked again
+open = []; timers = [];
+let click9;
+const el9 = { innerHTML: "", addEventListener(ev, f) { click9 = f; } };
+const queued = { ...running, id: "q", state: "queued", cancellable: true, progress_total: null };
+jobPanel(el9, urls, "tok");
+advance(0);
+answer("/r", { ...empty, jobs: [queued] });
+assert.ok(el9.innerHTML.includes('data-cancel="q">Cancel</button>'), "an enabled cancel button");
+click9({ target: { dataset: { cancel: "q" } } });
+assert.ok(el9.innerHTML.includes('data-cancel="q" disabled>'), "disabled while the request runs");
+answer("/c/q", { status: 400, responseJSON: { detail: "This job cannot be cancelled once it has started" } }, false);
+assert.ok(el9.innerHTML.includes("This job cannot be cancelled once it has started")
+    && el9.innerHTML.includes('class="alert alert-danger'), "the server's reason shows");
+assert.ok(open.some((r) => r.url === "/r" && !r.settled), "the list is asked again at once");
+answer("/r", { ...empty, jobs: [{ ...queued, state: "running", cancellable: false }] });
+assert.ok(el9.innerHTML.includes("cannot be cancelled") && !el9.innerHTML.includes("data-cancel"),
+    "the reason stays after the refresh");
+click9({ target: { dataset: { dismissError: "1" } } });
+assert.ok(!el9.innerHTML.includes("cannot be cancelled"), "the reason can be dismissed");
+advance(3000);
+answer("/r", { ...empty, jobs: [queued] });
+click9({ target: { dataset: { cancel: "q" } } });
+answer("/c/q", { status: 0 }, false);
+assert.ok(el9.innerHTML.includes("could not be reached"), "a network error shows");
+Date.now = realNow;
+console.log("ok");

operations/tests/test_panel_js.py

@@ -0,0 +1,19 @@
+import shutil
+import subprocess
+import unittest
+from pathlib import Path
+
+from django.conf import settings
+
+BASE = Path(settings.BASE_DIR)
+
+
+class JobPanelJsTestCase(unittest.TestCase):
+    @unittest.skipUnless(shutil.which("node"), "node is not installed")
+    def test_job_panel(self):
+        result = subprocess.run(
+            ["node", str(BASE / "operations/tests/js/job_panel.test.js"),
+             str(BASE / "static/js/poll.js"), str(BASE / "operations/static/operations/job_panel.js")],
+            capture_output=True, text=True, timeout=60,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr + result.stdout)

operations/tests/test_service.py

@@ -198,7 +198,15 @@ class RememberJobTestCase(SimpleTestCase):
     def test_the_last_twenty_jobs_are_remembered_until_taken(self):
         request = SimpleNamespace(session={})
         for n in range(25):
-            jobs.remember_job(request, SimpleNamespace(pk=n))
-        self.assertEqual(request.session["operations_shown"], [str(n) for n in range(5, 25)])
-        self.assertEqual(jobs.take_remembered(request), [str(n) for n in range(5, 25)])
-        self.assertEqual(jobs.take_remembered(request), [])
+            jobs.remember_job(request, SimpleNamespace(pk=n, resource="instance:a"))
+        self.assertEqual(len(request.session["operations_shown"]), 20)
+        self.assertEqual(jobs.take_remembered(request, "instance:a"), [str(n) for n in range(5, 25)])
+        self.assertEqual(jobs.take_remembered(request, "instance:a"), [])
+
+    def test_a_resource_takes_only_its_own_jobs(self):
+        request = SimpleNamespace(session={})
+        jobs.remember_job(request, SimpleNamespace(pk=1, resource="instance:a"))
+        jobs.remember_job(request, SimpleNamespace(pk=2, resource="instance:b"))
+        self.assertEqual(jobs.take_remembered(request, "instance:a"), ["1"])
+        self.assertEqual(request.session["operations_shown"], [{"resource": "instance:b", "id": "2"}])
+        self.assertEqual(jobs.take_remembered(request, "instance:b"), ["2"])