refactor(console): socketiod on gunicorn threads instead of eventlet

eventlet is in maintenance mode. socketiod now runs python-socketio in
threading mode under gunicorn (one gthread worker, which holds the
sessions), with simple-websocket, already a python-engineio dependency.
gunicorn waits 1 s on stop: an open console never finishes, and its
worker ends when the pty closes. The forward loop waits in select
instead of polling, and a stopped worker is reaped by a blocking
waitpid. eventlet leaves the requirements.

conf/requirements.txt

@@ -15,7 +15,6 @@ django-axes==8.3.1
 djangorestframework==3.18.3
 drf-nested-routers==0.95.3
 drf-spectacular[sidecar]==0.30.0
-eventlet==0.41.2
 gunicorn==26.2.0
 libsass==0.23.0
 libvirt-python>=9.0.0

console/socketiod

@@ -24,6 +24,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "webvirtcloud.settings")
 django.setup()
 
 import socketio  # noqa: E402
+from gunicorn.app.base import BaseApplication  # noqa: E402
 from django import db  # noqa: E402
 from django.conf import settings  # noqa: E402
 
@@ -32,7 +33,7 @@ from console.tokens import authorize  # noqa: E402
 CONSOLECALLBACK = os.path.join(ROOT_PATH, "conf", "daemon", "consolecallback")
 MAX_SESSIONS = 20
 
-sio = socketio.Server(async_mode="eventlet")  # the same origin only
+sio = socketio.Server(async_mode="threading")  # the same origin only
 sessions = {}  # sid -> {"pid", "fd", "vm"}
 
 
@@ -62,8 +63,7 @@ def forward(sid, session):
     """The worker's output to its client, until the worker ends."""
     decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")  # a character may be split
     while sessions.get(sid) is session:
-        sio.sleep(0.01)
-        if not select.select([session["fd"]], [], [], 0)[0]:
+        if not select.select([session["fd"]], [], [], 0.1)[0]:
             continue
         try:
             output = os.read(session["fd"], 20480)
@@ -90,17 +90,7 @@ def stop(sid):
         os.kill(session["pid"], signal.SIGKILL)
     except ProcessLookupError:
         pass
-    sio.start_background_task(reap, session["pid"])
-
-
-def reap(pid):
-    while True:
-        try:
-            if os.waitpid(pid, os.WNOHANG)[0]:
-                return
-        except ChildProcessError:
-            return
-        sio.sleep(0.1)
+    sio.start_background_task(os.waitpid, session["pid"], 0)
 
 
 @sio.event
@@ -128,18 +118,27 @@ def disconnect(sid, reason=None):
     stop(sid)
 
 
-def shutdown(signum, frame):
-    for sid in list(sessions):
-        stop(sid)
-    sys.exit(0)
+class Server(BaseApplication):
+    """gunicorn with one worker (the sessions live in it); a thread per connection."""
+
+    def __init__(self, host, port):
+        self.bind = "%s:%d" % (host, port)
+        super().__init__()
+
+    def load_config(self):
+        self.cfg.set("bind", self.bind)
+        self.cfg.set("worker_class", "gthread")
+        self.cfg.set("threads", 2 * MAX_SESSIONS)
+        self.cfg.set("graceful_timeout", 1)  # open consoles never finish; their workers end with the pty
+        self.cfg.set("control_socket_disable", True)
+
+    def load(self):
+        return socketio.WSGIApp(sio)
 
 
 if __name__ == "__main__":
-    import eventlet
-
     parser = argparse.ArgumentParser(description="WebVirtCloud serial console daemon")
     parser.add_argument("-H", "--host", default=settings.SOCKETIO_HOST, help="Listen host")
     parser.add_argument("-p", "--port", type=int, default=settings.SOCKETIO_PORT, help="Listen port")
     options = parser.parse_args()
-    signal.signal(signal.SIGTERM, shutdown)
-    eventlet.wsgi.server(eventlet.listen((options.host, options.port)), socketio.WSGIApp(sio))
+    Server(options.host, options.port).run()

console/tests/test_live_serial.py

@@ -16,7 +16,7 @@ from vrtManager.connection import connection_manager
 from accounts.models import UserInstance
 from computes.models import Compute
 from console import tokens
-from console.tests.test_socketiod_child import children, console, start_socketiod, socketio
+from console.tests.test_socketiod_child import consoles, console, start_socketiod, socketio
 from instances.models import Instance
 from instances.tests import livetest
 
@@ -73,7 +73,7 @@ class LiveSerialConsoleTestCase(TransactionTestCase):
                     console(self.port, token)
                 client.disconnect()
                 deadline = time.monotonic() + 5
-                while children(self.server.pid) and time.monotonic() < deadline:
+                while consoles(self.server.pid) and time.monotonic() < deadline:
                     time.sleep(0.05)
-                self.assertEqual(children(self.server.pid), [])
+                self.assertEqual(consoles(self.server.pid), [])
                 dom.destroy()

console/tests/test_socketiod_child.py

@@ -51,6 +51,11 @@ def children(pid):
     return found
 
 
+def consoles(pid):
+    """(pid, state) of the console workers: children of socketiod's gunicorn worker."""
+    return [c for worker, _ in children(pid) for c in children(worker)]
+
+
 def start_socketiod(test):
     """socketiod on a free port, reading test's database; stopped at test's cleanup."""
     tmp = tempfile.TemporaryDirectory()
@@ -116,13 +121,13 @@ class SocketiodProcessTestCase(TransactionTestCase):
                 self.assertNotIn("vm-a" if name == "vm-b" else "vm-b", output)
                 self.assertIn("console closed", output)
         end = time.monotonic() + 5
-        while children(self.server.pid) and time.monotonic() < end:
+        while consoles(self.server.pid) and time.monotonic() < end:
             time.sleep(0.05)
-        self.assertEqual(children(self.server.pid), [])
+        self.assertEqual(consoles(self.server.pid), [])
         self.assertIsNone(self.server.poll())
 
     def test_a_client_without_a_valid_token_is_refused(self):
         for token in ("", "22-no-such-vm"):
             with self.subTest(token=token), self.assertRaises(socketio.exceptions.ConnectionError):
                 self.console(token)
-        self.assertEqual(children(self.server.pid), [])
+        self.assertEqual(consoles(self.server.pid), [])