fix(vrtManager): libvirt connections open within a time limit

libvirt has no timeout for opening a connection. Measured in the test
setup: an unreachable host blocked qemu+tcp for 135 s and qemu+ssh for
133 s; a daemon that accepts and never answers blocks forever. Gunicorn
runs sync workers and the instance list connects to every compute, so
one unreachable compute held a worker for minutes per page load.

- A connection is opened in a helper thread, at most one per
  connection at a time. The request waits LIBVIRT_CONNECT_TIMEOUT
  (default 5 s). An open nobody waits for any more keeps running; until
  it ends, requests get its error at once, and if it succeeds later the
  connection is used. A failed open is not repeated for
  LIBVIRT_RETRY_AFTER (default 30 s). A stuck host holds one thread and
  does not keep other hosts from connecting.
- qemu+ssh runs ssh through vrtManager/libvirt-ssh (the URI's command
  parameter), which adds ConnectTimeout=10 and BatchMode=yes, so an
  abandoned ssh process ends and no prompt can hang. The user's ssh
  configuration (aliases, ProxyJump, keys) still applies.
- The close callback acts only for the connection it was registered
  for: a late callback of a replaced connection dropped its successor.
  The reason is reported as text. A connection that closed before it
  was published is an error.
- Compute.status is a bool (it returned the exception object, so the
  "host is down" check in refresh_instance_database never fired); the
  message is Compute.connection_error.

Unit tests cover the timeout, retry delay, late success, single open,
a stuck host next to a healthy one and the close callback. Live tests
(vrtManager/test_live_connection.py), for TCP and SSH computes of the
test setup: an unreachable address (no ssh left after 10 s), an endpoint
that never answers, a slow open completing later, a host that goes
silent on an established connection (keepalive ends calls within 45 s,
the next request reconnects), an ssh_config alias and a ProxyJump host.

computes/models.py

@@ -17,12 +17,14 @@ class Compute(Model):
 
     @cached_property
     def status(self):
-        # return connection_manager.host_is_up(self.type, self.hostname)
-        # TODO: looks like socket has problems connecting via VPN
-        if isinstance(self.connection, virConnect):
-            return True
-        else:
-            return self.connection
+        """True if WebVirtCloud can connect to the host (within the connect
+        timeout; a host that failed recently is not tried again at once)."""
+        return isinstance(self.connection, virConnect)
+
+    @cached_property
+    def connection_error(self):
+        """Why the host cannot be reached, or None."""
+        return None if self.status else str(self.connection)
 
     @cached_property
     def connection(self):

computes/test_ownership.py

@@ -34,6 +34,7 @@ class OwnershipTestCase(TestCase):
     def compute(self, name):
         compute = Compute.objects.create(name=name, hostname=f"{name}.example", login="", password="", type=1)
         compute.__dict__["proxy"] = MagicMock()
+        compute.__dict__["status"] = True  # reachable
         self.on(compute, [])
         return compute
 

computes/test_reconciliation.py

@@ -40,6 +40,7 @@ class InstanceReconciliationTestCase(TestCase):
         mock_proxy = MagicMock()
         mock_proxy.wvm.listAllDomains.return_value = [dom1, dom2]
         self.compute.__dict__["proxy"] = mock_proxy
+        self.compute.__dict__["status"] = True  # reachable
 
         refresh_instance_database(self.compute)
 
@@ -61,6 +62,7 @@ class InstanceReconciliationTestCase(TestCase):
         mock_proxy = MagicMock()
         mock_proxy.wvm.listAllDomains.return_value = []
         self.compute.__dict__["proxy"] = mock_proxy
+        self.compute.__dict__["status"] = True  # reachable
 
         refresh_instance_database(self.compute)
 
@@ -82,6 +84,7 @@ class InstanceReconciliationTestCase(TestCase):
         mock_proxy = MagicMock()
         mock_proxy.wvm.listAllDomains.return_value = [dom_renamed]
         self.compute.__dict__["proxy"] = mock_proxy
+        self.compute.__dict__["status"] = True  # reachable
 
         refresh_instance_database(self.compute)
 
@@ -105,6 +108,7 @@ class InstanceReconciliationTestCase(TestCase):
         mock_proxy = MagicMock()
         mock_proxy.wvm.listAllDomains.side_effect = Exception("Libvirt socket connection refused")
         self.compute.__dict__["proxy"] = mock_proxy
+        self.compute.__dict__["status"] = True  # reachable
 
         refresh_instance_database(self.compute)
 
@@ -124,6 +128,7 @@ class InstanceReconciliationTestCase(TestCase):
         mock_proxy = MagicMock()
         mock_proxy.wvm.listAllDomains.return_value = [broken_dom]
         self.compute.__dict__["proxy"] = mock_proxy
+        self.compute.__dict__["status"] = True  # reachable
 
         refresh_instance_database(self.compute)
 

computes/tests.py

@@ -214,6 +214,7 @@ class ComputeConcurrencyTestCase(TransactionTestCase):
         mock_proxy = MagicMock()
         mock_proxy.wvm.listAllDomains.return_value = [mock_dom1, mock_dom2]
         compute.proxy = mock_proxy
+        compute.status = True  # reachable
 
         exceptions = []
 
@@ -253,6 +254,7 @@ class ComputeConcurrencyTestCase(TransactionTestCase):
         mock_proxy = MagicMock()
         mock_proxy.wvm.listAllDomains.return_value = []
         compute.proxy = mock_proxy
+        compute.status = True  # reachable
 
         # 1. When lock directory cannot be acquired
         with patch("computes.utils._get_lock_directory", return_value=None):