fix(gstfsd): at most 4 connections at a time; one more is told it is busy

Every connection got a thread before any authentication, and every
signed request a guestfs appliance, with no limit: with remote access
allowed, anyone on that network could make the host run out of threads
or memory. A connection over MAX_CLIENTS (4) now gets an unsigned busy
line and is closed before it gets a thread; the panel shows it as the
gstfsd error.

conf/daemon/gstfsd

@@ -43,6 +43,9 @@ KEY = os.environ.get("GSTFSD_KEY", "").strip()
 PORT = int(os.environ.get("GSTFSD_PORT", 16510))
 ADDRESS = os.environ.get("GSTFSD_BIND_HOST", "127.0.0.1")
 MAX_REQUEST = 64 * 1024
+# connections at a time, each a thread that may run a guestfs appliance; one
+# more is told gstfsd is busy before it gets a thread
+MAX_CLIENTS = 4
 # A request, from accepting the connection to closing the appliance, must
 # finish within this many seconds, or it is stopped: the panel gives up 120 s
 # after connecting and releases the VM, which may then be started.
@@ -344,6 +347,23 @@ def serve():
     class MyTCPServer(socketserver.ThreadingTCPServer):
         allow_reuse_address = True
         address_family = socket.AF_INET6 if ":" in ADDRESS else socket.AF_INET
+        clients = threading.BoundedSemaphore(MAX_CLIENTS)
+
+        def process_request(self, request, client_address):
+            if not self.clients.acquire(blocking=False):
+                try:
+                    request.sendall(b'{"error": "gstfsd is busy with other changes: try again in a moment"}\n')
+                except OSError:
+                    pass
+                self.shutdown_request(request)
+                return
+            super().process_request(request, client_address)
+
+        def process_request_thread(self, request, client_address):
+            try:
+                super().process_request_thread(request, client_address)
+            finally:
+                self.clients.release()
 
     MyTCPServer((ADDRESS, PORT), MyTCPServerHandler).serve_forever()
     return 0

instances/tests/test_gstfsd.py

@@ -292,6 +292,32 @@ class GstfsdTestCase(unittest.TestCase):
                 views.gstfsd_request(compute, {"action": "password", "vname": "vm", "passwd": "$6$other"})
         self.assertIn("root:$6$panel:", json.loads(self.state.read_text())["files"]["/etc/shadow"])
 
+    def test_a_connection_over_the_limit_is_told_gstfsd_is_busy(self):
+        from unittest.mock import patch
+
+        from instances import views
+        from vrtManager.util import OperationError
+
+        port = self._serve()
+        held = []
+        for _ in range(4):  # MAX_CLIENTS connections that send no request
+            s = socket.create_connection(("127.0.0.1", port), timeout=10)
+            self.addCleanup(s.close)
+            challenge_of(s)
+            held.append(s)
+        compute = SimpleNamespace(name="host-a", hostname="127.0.0.1", gstfsd_key=KEY)
+        with patch.object(views, "GSTFSD_PORT", port):
+            with self.assertRaisesRegex(OperationError, "gstfsd is busy with other changes"):
+                views.gstfsd_request(compute, {"action": "password", "vname": "vm", "passwd": "$6$x"})
+            held.pop().close()  # a connection ends: its place is free again
+            for _ in range(50):
+                try:
+                    reply = views.gstfsd_request(compute, {"action": "password", "vname": "vm", "passwd": "$6$x"})
+                    break
+                except OperationError:
+                    time.sleep(0.1)
+        self.assertEqual(reply, {"return": "success"})
+
     def test_root_password_is_set_on_the_guest_root(self):
         reply = self._request("127.0.0.1", self._serve(), {"action": "password", "vname": "vm", "passwd": "$6$new"})
         self.assertEqual(reply, {"return": "success"})

instances/views.py

@@ -1046,7 +1046,10 @@ def gstfsd_request(compute, data):
     try:
         s.settimeout(GSTFSD_TIMEOUT)
         stream = s.makefile("rb")
-        challenge = json.loads(stream.readline(GSTFSD_MAX_FRAME))["challenge"]
+        first = json.loads(stream.readline(GSTFSD_MAX_FRAME))
+        if "error" in first:  # busy: gstfsd took no request
+            raise ValueError(first["error"])
+        challenge = first["challenge"]
         nonce, request = secrets.token_hex(32), json.dumps(data)
         s.sendall(json.dumps({"nonce": nonce, "request": request,
                               "mac": gstfsd_mac(compute.gstfsd_key, "request", challenge, nonce, request)}).encode())