fix(gunicorn): at most 8 workers, and why the timeout stays 600 s

README.md

@@ -287,6 +287,7 @@ Where to set them:
 
 - **Docker:** the `environment:` list of `docker-compose.override.yml`; every process in the container gets it.
 - **Installer and manual installation:** edit `webvirtcloud/settings.py`, or put the same `environment=` line in both `[program:webvirtcloud]` and `[program:novncd]` of the supervisor config (novncd reads the database too) and export the variables before running `manage.py`.
+- **gunicorn** runs at most 8 workers (fewer on hosts with under 4 CPUs); to change it, set `GUNICORN_CMD_ARGS="--workers N"` in the `[program:webvirtcloud]` environment or in `docker-compose.override.yml`.
 
 ### HTTPS and reverse proxy
 

gunicorn.conf.py

@@ -73,14 +73,22 @@ backlog = 2048
 #
 
 
+# Sync workers, each holding its own libvirt connections and blocking for a
+# whole long operation; capped so a big host does not start dozens of them
+# against sqlite. Override with GUNICORN_CMD_ARGS="--workers N".
+MAX_WORKERS = 8
+
+
 def get_workers():
     procs = os.sysconf('SC_NPROCESSORS_ONLN')
-    return procs * 2 + 1 if procs > 0 else 3
+    return min(procs * 2 + 1, MAX_WORKERS) if procs > 0 else 3
 
 
 workers = get_workers()
 # worker_class = 'egg:gunicorn#eventlet'
 worker_connections = 1000
+# Must exceed instances.utils.MIGRATION_TIME_LIMIT (540 s) plus time to answer;
+# a lower value would kill a worker in the middle of a live migration.
 timeout = 600
 keepalive = 2
 

webvirtcloud/test_gunicorn_conf.py

@@ -0,0 +1,26 @@
+"""gunicorn runs a bounded number of sync workers."""
+
+import runpy
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from django.conf import settings
+
+CONF = str(Path(settings.BASE_DIR) / "gunicorn.conf.py")
+
+
+def load(cpus):
+    with patch("os.sysconf", return_value=cpus):
+        return runpy.run_path(CONF)
+
+
+class GunicornConfTestCase(unittest.TestCase):
+    def test_worker_count_is_capped(self):
+        self.assertEqual(load(64)["workers"], 8)
+
+    def test_small_hosts_keep_two_per_cpu_plus_one(self):
+        self.assertEqual(load(2)["workers"], 5)
+
+    def test_unknown_cpu_count_falls_back(self):
+        self.assertEqual(load(-1)["workers"], 3)