build: run the job worker in every install type

Dockerfile

@@ -31,7 +31,9 @@ EXPOSE 80
 # script writes this into settings.py); set it to the published port if not 80.
 ENV WS_PUBLIC_PORT=80
 
-# Use baseimage-docker's init system.
+# Use baseimage-docker's init system. my_init force-stops services after
+# KILL_PROCESS_TIMEOUT seconds (default 30); the worker drains jobs for 60 s.
+ENV KILL_PROCESS_TIMEOUT=80
 CMD ["/sbin/my_init"]
 
 RUN echo 'APT::Get::Clean=always;' >> /etc/apt/apt.conf.d/99AutomaticClean
@@ -69,17 +71,20 @@ RUN mkdir -p /etc/my_init.d \
 	/etc/service/nginx \
 	/etc/service/nginx-log-forwarder \
 	/etc/service/webvirtcloud \
+	/etc/service/webvirtcloud-worker \
 	/etc/service/novnc
 COPY conf/runit/10_webvirtcloud_init.sh	/etc/my_init.d/10_webvirtcloud_init.sh
 COPY conf/runit/nginx				/etc/service/nginx/run
 COPY conf/runit/nginx-log-forwarder	/etc/service/nginx-log-forwarder/run
 COPY conf/runit/novncd.sh			/etc/service/novnc/run
 COPY conf/runit/webvirtcloud.sh		/etc/service/webvirtcloud/run
+COPY conf/runit/webvirtcloud-worker.sh	/etc/service/webvirtcloud-worker/run
 RUN chmod +x /etc/my_init.d/10_webvirtcloud_init.sh \
 	/etc/service/nginx/run \
 	/etc/service/nginx-log-forwarder/run \
 	/etc/service/novnc/run \
-	/etc/service/webvirtcloud/run
+	/etc/service/webvirtcloud/run \
+	/etc/service/webvirtcloud-worker/run
 
 # X-Forwarded-Proto: https keeps the probe from being redirected when the
 # HTTPS profile is on (nginx passes the header on; see conf/nginx).

README.md

@@ -121,7 +121,7 @@ pip install -r conf/requirements.txt
 python3 manage.py migrate
 python3 manage.py collectstatic --noinput
 
-# 6. Supervisor (gunicorn and novncd) and nginx
+# 6. Supervisor (gunicorn, novncd and the job worker) and nginx
 cp conf/supervisor/webvirtcloud.conf /etc/supervisor/conf.d/
 cp conf/nginx/webvirtcloud.conf /etc/nginx/conf.d/
 rm -f /etc/nginx/sites-enabled/default
@@ -157,7 +157,7 @@ pip install -r conf/requirements.txt
 python3 manage.py migrate
 python3 manage.py collectstatic --noinput
 
-# 6. Supervisor (gunicorn and novncd, as nginx) and nginx
+# 6. Supervisor (gunicorn, novncd and the job worker, as nginx) and nginx
 sed 's/^user=.*/user=nginx/' conf/supervisor/webvirtcloud.conf > /etc/supervisord.d/webvirtcloud.ini
 cp conf/nginx/webvirtcloud.conf /etc/nginx/conf.d/
 # make sure the default server block in /etc/nginx/nginx.conf does not also listen on port 80
@@ -200,7 +200,7 @@ pip install -r conf/requirements.txt
 python3 manage.py migrate
 python3 manage.py collectstatic --noinput
 
-# 6. Supervisor (gunicorn and novncd, as nginx) and nginx
+# 6. Supervisor (gunicorn, novncd and the job worker, as nginx) and nginx
 mkdir -p /etc/supervisord.d
 sed 's/^user=.*/user=nginx/' conf/supervisor/webvirtcloud.conf > /etc/supervisord.d/webvirtcloud.ini
 # make sure /etc/supervisord.conf has: [include] files = /etc/supervisord.d/*.ini
@@ -295,7 +295,7 @@ If libvirt warns `Host SMBIOS information is not available`, install `dmidecode`
 Where to set them:
 
 - **Docker:** the `environment:` list of `docker-compose.override.yml`; every process in the container gets it.
-- **Installer and manual installation:** edit `webvirtcloud/settings.py`, or put the same `environment=` line in both `[program:webvirtcloud]` and `[program:novncd]` of the supervisor config (novncd reads the database too) and export the variables before running `manage.py`.
+- **Installer and manual installation:** edit `webvirtcloud/settings.py`, or put the same `environment=` line in `[program:webvirtcloud]`, `[program:novncd]` and `[program:webvirt-worker]` of the supervisor config (novncd and the worker read the database too; supervisor takes one `environment=` per program, so merge the worker's into its `WEBVIRTCLOUD_LOG_FILE` line) and export the variables before running `manage.py`.
 - **gunicorn** runs at most 8 workers (fewer on hosts with under 4 CPUs); to change it, set `GUNICORN_CMD_ARGS="--workers N"` in the `[program:webvirtcloud]` environment or in `docker-compose.override.yml`.
 
 ### HTTPS and reverse proxy
@@ -421,7 +421,7 @@ b=../webvirtcloud-backup-$(date +%Y%m%d-%H%M%S) &&
 mkdir -m 700 "$b" && git rev-parse HEAD > "$b/commit" &&
 git checkout master && git pull &&
 docker compose build &&
-docker compose stop &&
+docker compose stop -t 95 &&
 docker compose run --rm --no-deps -T --entrypoint /srv/webvirtcloud/venv/bin/python3 webvirtcloud \
   -c "import sqlite3,sys,tempfile; t=tempfile.mktemp(); sqlite3.connect('data/db.sqlite3').backup(sqlite3.connect(t)); sys.stdout.buffer.write(open(t,'rb').read())" > "$b/db.sqlite3" &&
 docker compose up -d
@@ -434,12 +434,12 @@ docker compose up -d
 1. Stop the panel, back up and pull:
 
    ```bash
-   supervisorctl stop webvirtcloud novncd
+   supervisorctl stop webvirtcloud novncd webvirt-worker
    # once: root may use the repo, which belongs to the service user
    git config --global --add safe.directory /srv/webvirtcloud
    b=/root/webvirtcloud-backup-$(date +%Y%m%d-%H%M%S)
    cd /srv/webvirtcloud &&
-   ! supervisorctl status webvirtcloud novncd | grep RUNNING &&
+   ! supervisorctl status webvirtcloud novncd webvirt-worker | grep RUNNING &&
    mkdir -m 700 "$b" &&
    venv/bin/python3 -c "import sqlite3,sys; sqlite3.connect('db.sqlite3').backup(sqlite3.connect(sys.argv[1]))" "$b/db.sqlite3" &&
    cp -p webvirtcloud/settings.py /etc/nginx/conf.d/webvirtcloud.conf "$b"/ &&
@@ -448,6 +448,16 @@ docker compose up -d
    git checkout master && git pull
    ```
 
+   Until step 4 installs the worker, `supervisorctl` reports "no such process" for `webvirt-worker` on a first update from a version without it. Then keep the supervisor config for step 4 and the restore (`supervisor.conf` in the backup directory):
+
+   ```bash
+   cp -p /etc/supervisor/conf.d/webvirtcloud.conf "$b/supervisor.conf"   # Debian / Ubuntu
+   ```
+
+   ```bash
+   cp -p /etc/supervisord.d/webvirtcloud.ini "$b/supervisor.conf"   # RHEL / openSUSE
+   ```
+
 2. Rebuild `settings.py` from the template (the file keeps its owner and mode), then put back your own values; the `diff` lists them:
 
    ```bash
@@ -469,17 +479,31 @@ docker compose up -d
    chmod -R go-rwx data webvirtcloud/settings.py db.sqlite3
    ```
 
-4. Update the nginx config the same way as `settings.py`:
+4. Update the nginx config the same way as `settings.py`, and the supervisor config (it gains the `webvirt-worker` program):
 
    ```bash
    cp conf/nginx/webvirtcloud.conf /etc/nginx/conf.d/webvirtcloud.conf
    diff "$b/webvirtcloud.conf" /etc/nginx/conf.d/webvirtcloud.conf
    ```
 
-   Put back your own lines (`server_name`, TLS, the noVNC port), then reload nginx and start the panel:
+   Debian / Ubuntu:
 
    ```bash
-   nginx -t && systemctl reload nginx && supervisorctl start webvirtcloud novncd
+   cp conf/supervisor/webvirtcloud.conf /etc/supervisor/conf.d/
+   diff "$b/supervisor.conf" /etc/supervisor/conf.d/webvirtcloud.conf
+   ```
+
+   RHEL / openSUSE:
+
+   ```bash
+   sed 's/^user=.*/user=nginx/' conf/supervisor/webvirtcloud.conf > /etc/supervisord.d/webvirtcloud.ini
+   diff "$b/supervisor.conf" /etc/supervisord.d/webvirtcloud.ini
+   ```
+
+   Put back your own lines (`server_name`, TLS, the noVNC port, supervisor `environment=` settings), then reload nginx and start the panel:
+
+   ```bash
+   nginx -t && systemctl reload nginx && supervisorctl update && supervisorctl start webvirtcloud novncd webvirt-worker
    ```
 
 Older versions collected static files into `static/`; `git clean -n static/` lists the leftovers, which can be deleted.
@@ -493,7 +517,7 @@ Docker, in the directory you installed from:
 ```bash
 cd webvirtcloud &&
 b=../webvirtcloud-backup-YYYYMMDD-HHMMSS &&
-docker compose stop &&
+docker compose stop -t 95 &&
 docker compose run --rm --no-deps -T --entrypoint /srv/webvirtcloud/venv/bin/python3 webvirtcloud \
   -c "import os,sqlite3,sys,tempfile; t=tempfile.mktemp(); open(t,'wb').write(sys.stdin.buffer.read()); [os.remove(p) for p in ('data/db.sqlite3-journal','data/db.sqlite3-wal','data/db.sqlite3-shm') if os.path.exists(p)]; sqlite3.connect(t).backup(sqlite3.connect('data/db.sqlite3'))" < "$b/db.sqlite3" &&
 git checkout "$(cat "$b/commit")" &&
@@ -505,9 +529,9 @@ Installer and manual installation, from a root shell (`nginx:nginx` on RHEL / op
 
 ```bash
 b=/root/webvirtcloud-backup-YYYYMMDD-HHMMSS
-supervisorctl stop webvirtcloud novncd
+supervisorctl stop webvirtcloud novncd webvirt-worker
 cd /srv/webvirtcloud &&
-! supervisorctl status webvirtcloud novncd | grep RUNNING &&
+! supervisorctl status webvirtcloud novncd webvirt-worker | grep RUNNING &&
 git checkout -- static/css/wvc-main.min.css dev/scss/wvc-main.scss &&
 git checkout "$(cat "$b/commit")" &&
 venv/bin/python3 -c "import os,sqlite3,sys; [os.remove(p) for p in ('db.sqlite3-journal','db.sqlite3-wal','db.sqlite3-shm') if os.path.exists(p)]; sqlite3.connect(sys.argv[1]).backup(sqlite3.connect('db.sqlite3'))" "$b/db.sqlite3" &&
@@ -516,12 +540,29 @@ cp "$b/webvirtcloud.conf" /etc/nginx/conf.d/webvirtcloud.conf &&
 venv/bin/pip3 install -r conf/requirements.txt &&
 venv/bin/python3 manage.py collectstatic --noinput &&
 chown -R www-data:www-data /srv/webvirtcloud &&
-chmod 600 db.sqlite3 webvirtcloud/settings.py &&
-nginx -t && systemctl reload nginx &&
-supervisorctl start webvirtcloud novncd
+chmod 600 db.sqlite3 webvirtcloud/settings.py
 ```
 
-A rollback leaves the clone on a detached HEAD at the restored commit; the update steps record that commit before they switch back to `master`.
+Restore the supervisor config, Debian / Ubuntu:
+
+```bash
+cp "$b/supervisor.conf" /etc/supervisor/conf.d/webvirtcloud.conf
+```
+
+RHEL / openSUSE:
+
+```bash
+cp "$b/supervisor.conf" /etc/supervisord.d/webvirtcloud.ini
+```
+
+Then reload nginx and start the panel:
+
+```bash
+nginx -t && systemctl reload nginx &&
+supervisorctl update && supervisorctl start webvirtcloud novncd webvirt-worker
+```
+
+When the restored version predates the worker, `supervisorctl` reports that `webvirt-worker` does not exist; the other programs start. A rollback leaves the clone on a detached HEAD at the restored commit; the update steps record that commit before they switch back to `master`.
 
 ### Behavior changes
 
@@ -559,6 +600,10 @@ A rollback leaves the clone on a detached HEAD at the restored commit; the updat
 - External snapshots use libvirt's snapshot API and need libvirt 9.9 or newer on the compute hosts. They work on UEFI VMs and running, paused or shut-off VMs, can save memory too or, on a running VM with a connected QEMU guest agent, freeze the guest's file systems (the default there), and a revert keeps disks added later (it detaches them). A snapshot is refused when a storage pool with the VM's disks has less free space than a share of those disks (10%, at least 1 GiB; both on the Settings page) plus, for the pool of the memory file, the VM's memory; this is checked before the snapshot only, nothing is reserved. A VM's snapshots form one chain: after a revert, delete the newer snapshots before taking another. Snapshot names no longer get an `s1.` prefix; existing ones keep theirs and can still be reverted and deleted, except that a snapshot an older version took of a UEFI VM can only be deleted. On NFS, libvirt removes merged snapshot files as the qemu user, who needs write access to the image directory; files it could not remove are named in a warning after the delete or revert. Deleting a VM with its data removes its snapshot files too, except a template it grew from and files another VM uses.
 - A storage volume a VM still needs (a disk, a backing file or a snapshot's file) is no longer deleted from the storage page or the API; the delete is refused with the reason. Only VMs on the same compute are checked.
 - Docker no longer publishes port 6080; the console goes through nginx on the panel's port (`WS_PUBLIC_PORT`, default 80).
+- Long operations run as jobs in a worker process (`manage.py run_worker`); follow them on the VM page and on the Jobs page.
+- Manual installs that keep their own `settings.py` copy the SQLite `OPTIONS` and `TEST` entries from `settings.py.template` and add `"operations"` to `INSTALLED_APPS`.
+- The SQLite database must be on a local file system (WAL).
+- Outside compose, stop the Docker container with `docker stop -t 95` so the worker can finish running jobs.
 
 ## Usage
 

conf/runit/10_webvirtcloud_init.sh

@@ -50,3 +50,6 @@ chmod 700 "/var/www/.ssh" 2>/dev/null || true
 # secret key and settings private to the app user (as the installer does)
 chmod 700 "$DATA_DIR"
 chmod 600 "$DATA_DIR/db.sqlite3" "$APP_DIR/webvirtcloud/settings.py"
+for f in "$DATA_DIR/db.sqlite3-wal" "$DATA_DIR/db.sqlite3-shm"; do
+    if [ -e "$f" ]; then chown www-data:www-data "$f"; chmod 600 "$f"; fi
+done

conf/runit/webvirtcloud-worker.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+# The job worker (one per installation). runit stops it with TERM; it drains
+# running jobs for 60 s and exits by itself.
+cd /srv/webvirtcloud || exit
+export WEBVIRTCLOUD_LOG_FILE=/srv/webvirtcloud/data/webvirtcloud-worker.log
+exec /sbin/setuser www-data /srv/webvirtcloud/venv/bin/python3 manage.py run_worker >> /var/log/webvirtcloud-worker.log 2>&1

conf/supervisor/webvirtcloud.conf

@@ -24,3 +24,15 @@ user=www-data
 autostart=false
 autorestart=true
 redirect_stderr=true
+
+# The job worker: one per installation. It drains running jobs for 60 s on
+# stop; supervisor waits a little longer before killing it.
+[program:webvirt-worker]
+command=/srv/webvirtcloud/venv/bin/python3 /srv/webvirtcloud/manage.py run_worker
+directory=/srv/webvirtcloud
+user=www-data
+environment=WEBVIRTCLOUD_LOG_FILE="/srv/webvirtcloud/data/webvirtcloud-worker.log"
+autostart=true
+autorestart=true
+stopwaitsecs=90
+redirect_stderr=true

conf/systemd/README.md

@@ -13,3 +13,4 @@ Should the executable be located in another location please adjust the path
 The error you should be seeing is "ssh executable not found"
 
 
+Use one of the two service managers: the installer sets up supervisor (`conf/supervisor/webvirtcloud.conf` runs the worker); these units are for manual installs without supervisor. Enabling both starts two workers; the second waits for the lease and does nothing.

conf/systemd/webvirt-worker.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=WebVirtCloud job worker
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=simple
+Restart=on-failure
+RestartSec=10
+User=www-data
+Group=www-data
+EnvironmentFile=/etc/default/webvirt
+ExecStart=/bin/bash -c "cd ${WEBVIRT_PATH}; WEBVIRTCLOUD_LOG_FILE=${WEBVIRT_PATH}/data/webvirtcloud-worker.log ${WEBVIRT_PATH}/venv/bin/python3 manage.py run_worker"
+TimeoutStopSec=90
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

docker-compose.yml

@@ -6,6 +6,8 @@ services:
     image: webvirtcloud:latest
     container_name: webvirtcloud
     restart: unless-stopped
+    # above my_init's 80 s, which is above the worker's 60 s drain
+    stop_grace_period: 95s
     ports:
       - "80:80"
     environment:

webvirtcloud/tests/test_worker_install.py

@@ -0,0 +1,41 @@
+"""Every install type runs the worker as www-data, with its own log file and
+a stop timeout above the worker's 60 s grace."""
+
+import re
+from pathlib import Path
+
+from django.conf import settings
+from django.test import SimpleTestCase
+
+BASE = Path(settings.BASE_DIR)
+
+
+class WorkerInstallTestCase(SimpleTestCase):
+    def test_docker(self):
+        dockerfile = (BASE / "Dockerfile").read_text()
+        self.assertIn("conf/runit/webvirtcloud-worker.sh", dockerfile)
+        self.assertIn("/etc/service/webvirtcloud-worker/run", dockerfile)
+        script = (BASE / "conf/runit/webvirtcloud-worker.sh").read_text()
+        for text in ("/sbin/setuser www-data", "manage.py run_worker",
+                     "WEBVIRTCLOUD_LOG_FILE=/srv/webvirtcloud/data/webvirtcloud-worker.log"):
+            self.assertIn(text, script)
+        self.assertIn("ENV KILL_PROCESS_TIMEOUT=80", dockerfile)
+        compose = (BASE / "docker-compose.yml").read_text()
+        self.assertRegex(compose, r"stop_grace_period:\s*9\ds")
+        # the worker's host identity lives in the data volume, not in a fixed hostname
+        self.assertIn("webvirtcloud-data:/srv/webvirtcloud/data", compose)
+        self.assertNotIn("hostname:", compose)
+
+    def test_supervisor(self):
+        conf = (BASE / "conf/supervisor/webvirtcloud.conf").read_text()
+        block = re.search(r"\[program:webvirt-worker\](.*?)(\n\[|\Z)", conf, re.S).group(1)
+        for text in ("manage.py run_worker", "user=www-data", "webvirtcloud-worker.log", "autorestart=true"):
+            self.assertIn(text, block)
+        self.assertRegex(block, r"stopwaitsecs=9\d")
+
+    def test_systemd_is_an_alternative(self):
+        unit = (BASE / "conf/systemd/webvirt-worker.service").read_text()
+        self.assertIn("run_worker", unit)
+        self.assertIn("Restart=on-failure", unit)
+        self.assertRegex(unit, r"TimeoutStopSec=9\d")
+        self.assertIn("one of the two", (BASE / "conf/systemd/README.md").read_text())