feat(webvirtcloud): a request waiting on a host holds one thread, not a worker

A host that is up but does not answer (a hung NFS mount, for one) keeps
a libvirt call waiting: libvirt's keepalive does not end it, since the
daemon still answers keepalives. Each such request held a whole sync
worker, and once they held every worker the panel stopped answering,
pages without libvirt too; F5 added another.

gunicorn now runs threaded workers, 4 threads each, and a worker takes
no more connections than it has threads, so one whose threads are all
busy leaves new requests to the others. A threaded worker keeps
reporting in while a thread waits, so gunicorn's timeout no longer ends
the request: the worker now exits itself when a request runs past the
timeout (600 s), and gunicorn starts a new one. Short claims expire on
that bound.

The address search runs one at a time in a process, as it did with one
request per worker.

webvirtcloud/gunicorn_worker.py

@@ -0,0 +1,38 @@
+"""gunicorn's threaded worker with the sync worker's request timeout."""
+
+import os
+import threading
+import time
+
+from gunicorn.workers.gthread import ThreadWorker
+
+
+class Worker(ThreadWorker):
+    """gunicorn kills a sync worker whose request runs past its timeout; a
+    threaded worker keeps reporting in while one of its threads waits on a
+    host that does not answer. This one exits when a request runs past the
+    timeout, and gunicorn starts a new one: a request ends by then, as the
+    short claims of operations.claims expect."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.started = {}  # thread: when its request started
+        self.started_lock = threading.Lock()
+
+    def handle(self, conn):
+        thread = threading.get_ident()
+        with self.started_lock:
+            self.started[thread] = time.monotonic()
+        try:
+            return super().handle(conn)
+        finally:
+            with self.started_lock:
+                del self.started[thread]
+
+    def notify(self):
+        with self.started_lock:
+            oldest = min(self.started.values(), default=None)
+        if oldest is not None and time.monotonic() - oldest > self.cfg.timeout:
+            self.log.critical("A request ran past %s s: the worker exits", self.cfg.timeout)
+            os._exit(1)  # its threads cannot be stopped, and exit() would wait for them
+        super().notify()

webvirtcloud/tests/test_gunicorn_conf.py

@@ -24,3 +24,9 @@ class GunicornConfTestCase(unittest.TestCase):
 
     def test_unknown_cpu_count_falls_back(self):
         self.assertEqual(load(-1)["workers"], 3)
+
+    def test_workers_run_requests_in_threads(self):
+        conf = load(2)
+        self.assertEqual((conf["worker_class"], conf["threads"]), ("webvirtcloud.gunicorn_worker.Worker", 4))
+        # a worker with every thread busy takes no new connection
+        self.assertEqual((conf["worker_connections"], conf["keepalive"]), (4, 0))

webvirtcloud/tests/test_gunicorn_worker.py

@@ -0,0 +1,56 @@
+"""The threaded worker exits when a request runs past gunicorn's timeout."""
+
+import threading
+import time
+import unittest
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+from gunicorn.workers.gthread import ThreadWorker
+
+from webvirtcloud.gunicorn_worker import Worker
+
+
+def worker():
+    w = Worker.__new__(Worker)
+    w.cfg = SimpleNamespace(timeout=600)
+    w.log = MagicMock()
+    w.started, w.started_lock = {}, threading.Lock()
+    return w
+
+
+class WorkerTestCase(unittest.TestCase):
+    def test_a_request_is_timed_while_it_runs(self):
+        w = worker()
+        seen = []
+        with patch.object(ThreadWorker, "handle", lambda self, conn: seen.append(dict(self.started)) or "done"):
+            self.assertEqual(w.handle("conn"), "done")
+        self.assertEqual(list(seen[0]), [threading.get_ident()])
+        self.assertEqual(w.started, {})
+
+    def test_a_failed_request_is_no_longer_timed(self):
+        w = worker()
+        with patch.object(ThreadWorker, "handle", side_effect=OSError("reset")), self.assertRaises(OSError):
+            w.handle("conn")
+        self.assertEqual(w.started, {})
+
+    def test_the_worker_reports_in_while_its_requests_are_within_the_timeout(self):
+        w = worker()
+        w.started[1] = time.monotonic() - 599
+        with patch.object(ThreadWorker, "notify") as notify, patch("os._exit") as exit_:
+            w.notify()
+        notify.assert_called_once_with()
+        exit_.assert_not_called()
+
+    def test_the_worker_exits_when_a_request_runs_past_the_timeout(self):
+        w = worker()
+        w.started.update({1: time.monotonic(), 2: time.monotonic() - 601})
+        with patch.object(ThreadWorker, "notify") as notify, patch("os._exit", side_effect=SystemExit) as exit_:
+            with self.assertRaises(SystemExit):
+                w.notify()
+        exit_.assert_called_once_with(1)
+        notify.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()