feat(webvirtcloud): a request waiting on a host holds one thread, not a worker

A host that is up but does not answer (a hung NFS mount, for one) keeps
a libvirt call waiting: libvirt's keepalive does not end it, since the
daemon still answers keepalives. Each such request held a whole sync
worker, and once they held every worker the panel stopped answering,
pages without libvirt too; F5 added another.

gunicorn now runs threaded workers, 4 threads each, and a worker takes
no more connections than it has threads, so one whose threads are all
busy leaves new requests to the others. A threaded worker keeps
reporting in while a thread waits, so gunicorn's timeout no longer ends
the request: the worker now exits itself when a request runs past the
timeout (600 s), and gunicorn starts a new one. Short claims expire on
that bound.

The address search runs one at a time in a process, as it did with one
request per worker.

instances/addresses.py

@@ -36,11 +36,12 @@ SOURCES = {
 # A read may outlast the search (a guest agent has AGENT_TIMEOUT seconds, a
 # host its connection's timeouts): its thread goes on after the search
 # answers, so a search starts only while its threads fit in MAX_RUNNING
-# search threads in this process.
+# search threads in this process, and one search at a time.
 MAX_WORKERS = 8
 MAX_RUNNING = 16
 _running = 0
 _running_lock = threading.Lock()
+_searching = threading.Lock()
 
 
 class Busy(Exception):
@@ -221,7 +222,7 @@ def _survey(targets, network, deadline, everything, with_os=False, mac=None):
     (late or failed). Returns the VMs' records, the observations, the agents'
     OS by vm, and the counts.
     """
-    if _running > MAX_RUNNING - MAX_WORKERS:
+    if _running > MAX_RUNNING - MAX_WORKERS or not _searching.acquire(blocking=False):
         raise Busy
     end = time.monotonic() + deadline
     records, observations, hosts_down, agent_reads, os_reads, systems = [], [], [], [], {}, {}
@@ -267,6 +268,7 @@ def _survey(targets, network, deadline, everything, with_os=False, mac=None):
                 systems[vm] = done.result()
     finally:
         pool.shutdown(wait=False, cancel_futures=True)
+        _searching.release()
     return records, observations, systems, {
         "checked": checked, "no_agent": no_agent, "unread": unread, "hosts_down": sorted(hosts_down),
         "hosts_unfinished": sorted(hosts[future].name for future in waiting)}

instances/tests/test_addresses.py

@@ -290,6 +290,16 @@ class FindTests(unittest.TestCase):
         finally:
             addresses._running = 0
 
+    def test_a_second_search_waits_for_none(self):
+        # threads of one worker search at once: a second search of the process is refused
+        addresses._searching.acquire()
+        try:
+            with self.assertRaises(addresses.Busy):
+                self.find(addresses.Target(compute("c1"), None))
+        finally:
+            addresses._searching.release()
+        self.find(addresses.Target(compute("c1"), None))  # the search lets the next one in
+
     def test_a_failed_agent_read_is_not_a_read(self):
         dom = FakeDomain({AGENT: libvirtError("guest-network-get-interfaces is disabled")})
         result = self.find(addresses.Target(compute("c1", dom), None))