fix(logs): failures name their job, VM and user; the new VM form shows a list it could not load

- A failed cancel hook logged "worker task cancel failed": it now names
  the job, its kind, resource and user, with the traceback for a fault
  of the panel.
- novncd logged "Console connection failed" without the VM or the user.
- django-axes wrote four INFO lines a login ("Reset 0 access
  attempts"...); logins are on the Logs page, so it logs warnings only.
- The new VM form's machine types and disk buses answered 200 with an
  empty object when libvirt failed, and nothing was logged: the list
  stayed empty, the form could not go on and said nothing. They answer
  400 with the error, as the pool's volume list does, the warning is
  logged with the compute, and the form's lists show an error they get
  with the page's messages.

operations/tests/test_worker.py

@@ -21,7 +21,7 @@ from operations.utils import words
 from operations import jobs
 from operations.models import Claim, Operation, WorkerLease
 from operations.tests import fakes
-from operations.worker import Worker, data_dir, host_identity, housekeeping, traceback_of
+from operations.worker import Worker, _guarded, data_dir, host_identity, housekeeping, traceback_of
 from vrtManager.util import ConnectionFailed
 
 
@@ -396,3 +396,28 @@ class HostIdentityTestCase(SimpleTestCase):
         with patch("operations.worker.socket.gethostname", return_value="wvc-test-b"):
             self.assertEqual(host_identity(self.dir), first)
 
+
+
+class CancelHookLogTests(SimpleTestCase):
+    def test_a_failed_cancel_names_its_job(self):
+        # every kind's hook is "cancel": the line alone said nothing of which job
+        op = Operation(kind="instance.clone", resource_label="c1/vm-alpha", username="alice")
+
+        def cancel(op):
+            raise ConnectionFailed("Connection Failed: %(host)s did not answer within 5 s", host="192.0.2.10")
+        with self.assertLogs("operations.worker", "ERROR") as logs:
+            _guarded(cancel, op)
+        self.assertEqual(logs.output, [f"ERROR:operations.worker:cancel of job {op.pk} (instance.clone, c1/vm-alpha, "
+                                       "by alice) failed: Connection Failed: 192.0.2.10 did not answer within 5 s"])
+
+    def test_housekeeping_runs_as_a_control_task_and_its_failure_is_logged(self):
+        # the worker submits it without a job
+        ran = []
+        _guarded(lambda: ran.append(True))
+        self.assertEqual(ran, [True])
+
+        def housekeeping():
+            raise RuntimeError("disk full")
+        with self.assertLogs("operations.worker", "ERROR") as logs:
+            _guarded(housekeeping)
+        self.assertTrue(logs.output[0].startswith("ERROR:operations.worker:housekeeping failed: disk full\nTraceback"))

operations/worker.py

@@ -332,11 +332,16 @@ def hosts_of(op_ids):
     return {op_id: frozenset(found) for op_id, found in hosts.items()}
 
 
-def _guarded(fn, *args):
+def _guarded(fn, op=None):
+    """A control task: housekeeping, or a job's cancel hook, which names its job when it fails."""
     try:
-        fn(*args)
+        if op is None:
+            fn()
+        else:
+            fn(op)
     except Exception as err:
-        logger.error("worker task %s failed: %s", getattr(fn, "__name__", fn), utils.safe_text(err))
+        job = f" of job {op.pk} ({op.kind}, {op.resource_label}, by {op.username})" if op is not None else ""
+        logger.error("%s%s failed: %s%s", getattr(fn, "__name__", fn), job, utils.safe_text(err), traceback_of(err))
     finally:
         close_old_connections()