diff --git a/Dockerfile b/Dockerfile index 612a3a91..0e7e3f48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,9 @@ EXPOSE 80 # script writes this into settings.py); set it to the published port if not 80. ENV WS_PUBLIC_PORT=80 -# Use baseimage-docker's init system. +# Use baseimage-docker's init system. my_init force-stops services after +# KILL_PROCESS_TIMEOUT seconds (default 30); the worker drains jobs for 60 s. +ENV KILL_PROCESS_TIMEOUT=80 CMD ["/sbin/my_init"] RUN echo 'APT::Get::Clean=always;' >> /etc/apt/apt.conf.d/99AutomaticClean @@ -69,17 +71,20 @@ RUN mkdir -p /etc/my_init.d \ /etc/service/nginx \ /etc/service/nginx-log-forwarder \ /etc/service/webvirtcloud \ + /etc/service/webvirtcloud-worker \ /etc/service/novnc COPY conf/runit/10_webvirtcloud_init.sh /etc/my_init.d/10_webvirtcloud_init.sh COPY conf/runit/nginx /etc/service/nginx/run COPY conf/runit/nginx-log-forwarder /etc/service/nginx-log-forwarder/run COPY conf/runit/novncd.sh /etc/service/novnc/run COPY conf/runit/webvirtcloud.sh /etc/service/webvirtcloud/run +COPY conf/runit/webvirtcloud-worker.sh /etc/service/webvirtcloud-worker/run RUN chmod +x /etc/my_init.d/10_webvirtcloud_init.sh \ /etc/service/nginx/run \ /etc/service/nginx-log-forwarder/run \ /etc/service/novnc/run \ - /etc/service/webvirtcloud/run + /etc/service/webvirtcloud/run \ + /etc/service/webvirtcloud-worker/run # X-Forwarded-Proto: https keeps the probe from being redirected when the # HTTPS profile is on (nginx passes the header on; see conf/nginx). diff --git a/README.md b/README.md index 5334e940..1095abc7 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ pip install -r conf/requirements.txt python3 manage.py migrate python3 manage.py collectstatic --noinput -# 6. Supervisor (gunicorn and novncd) and nginx +# 6. Supervisor (gunicorn, novncd and the job worker) and nginx cp conf/supervisor/webvirtcloud.conf /etc/supervisor/conf.d/ cp conf/nginx/webvirtcloud.conf /etc/nginx/conf.d/ rm -f /etc/nginx/sites-enabled/default @@ -157,7 +157,7 @@ pip install -r conf/requirements.txt python3 manage.py migrate python3 manage.py collectstatic --noinput -# 6. Supervisor (gunicorn and novncd, as nginx) and nginx +# 6. Supervisor (gunicorn, novncd and the job worker, as nginx) and nginx sed 's/^user=.*/user=nginx/' conf/supervisor/webvirtcloud.conf > /etc/supervisord.d/webvirtcloud.ini cp conf/nginx/webvirtcloud.conf /etc/nginx/conf.d/ # make sure the default server block in /etc/nginx/nginx.conf does not also listen on port 80 @@ -200,7 +200,7 @@ pip install -r conf/requirements.txt python3 manage.py migrate python3 manage.py collectstatic --noinput -# 6. Supervisor (gunicorn and novncd, as nginx) and nginx +# 6. Supervisor (gunicorn, novncd and the job worker, as nginx) and nginx mkdir -p /etc/supervisord.d sed 's/^user=.*/user=nginx/' conf/supervisor/webvirtcloud.conf > /etc/supervisord.d/webvirtcloud.ini # make sure /etc/supervisord.conf has: [include] files = /etc/supervisord.d/*.ini @@ -295,7 +295,7 @@ If libvirt warns `Host SMBIOS information is not available`, install `dmidecode` Where to set them: - **Docker:** the `environment:` list of `docker-compose.override.yml`; every process in the container gets it. -- **Installer and manual installation:** edit `webvirtcloud/settings.py`, or put the same `environment=` line in both `[program:webvirtcloud]` and `[program:novncd]` of the supervisor config (novncd reads the database too) and export the variables before running `manage.py`. +- **Installer and manual installation:** edit `webvirtcloud/settings.py`, or put the same `environment=` line in `[program:webvirtcloud]`, `[program:novncd]` and `[program:webvirt-worker]` of the supervisor config (novncd and the worker read the database too; supervisor takes one `environment=` per program, so merge the worker's into its `WEBVIRTCLOUD_LOG_FILE` line) and export the variables before running `manage.py`. - **gunicorn** runs at most 8 workers (fewer on hosts with under 4 CPUs); to change it, set `GUNICORN_CMD_ARGS="--workers N"` in the `[program:webvirtcloud]` environment or in `docker-compose.override.yml`. ### HTTPS and reverse proxy @@ -421,7 +421,7 @@ b=../webvirtcloud-backup-$(date +%Y%m%d-%H%M%S) && mkdir -m 700 "$b" && git rev-parse HEAD > "$b/commit" && git checkout master && git pull && docker compose build && -docker compose stop && +docker compose stop -t 95 && docker compose run --rm --no-deps -T --entrypoint /srv/webvirtcloud/venv/bin/python3 webvirtcloud \ -c "import sqlite3,sys,tempfile; t=tempfile.mktemp(); sqlite3.connect('data/db.sqlite3').backup(sqlite3.connect(t)); sys.stdout.buffer.write(open(t,'rb').read())" > "$b/db.sqlite3" && docker compose up -d @@ -434,12 +434,12 @@ docker compose up -d 1. Stop the panel, back up and pull: ```bash - supervisorctl stop webvirtcloud novncd + supervisorctl stop webvirtcloud novncd webvirt-worker # once: root may use the repo, which belongs to the service user git config --global --add safe.directory /srv/webvirtcloud b=/root/webvirtcloud-backup-$(date +%Y%m%d-%H%M%S) cd /srv/webvirtcloud && - ! supervisorctl status webvirtcloud novncd | grep RUNNING && + ! supervisorctl status webvirtcloud novncd webvirt-worker | grep RUNNING && mkdir -m 700 "$b" && venv/bin/python3 -c "import sqlite3,sys; sqlite3.connect('db.sqlite3').backup(sqlite3.connect(sys.argv[1]))" "$b/db.sqlite3" && cp -p webvirtcloud/settings.py /etc/nginx/conf.d/webvirtcloud.conf "$b"/ && @@ -448,6 +448,16 @@ docker compose up -d git checkout master && git pull ``` + Until step 4 installs the worker, `supervisorctl` reports "no such process" for `webvirt-worker` on a first update from a version without it. Then keep the supervisor config for step 4 and the restore (`supervisor.conf` in the backup directory): + + ```bash + cp -p /etc/supervisor/conf.d/webvirtcloud.conf "$b/supervisor.conf" # Debian / Ubuntu + ``` + + ```bash + cp -p /etc/supervisord.d/webvirtcloud.ini "$b/supervisor.conf" # RHEL / openSUSE + ``` + 2. Rebuild `settings.py` from the template (the file keeps its owner and mode), then put back your own values; the `diff` lists them: ```bash @@ -469,17 +479,31 @@ docker compose up -d chmod -R go-rwx data webvirtcloud/settings.py db.sqlite3 ``` -4. Update the nginx config the same way as `settings.py`: +4. Update the nginx config the same way as `settings.py`, and the supervisor config (it gains the `webvirt-worker` program): ```bash cp conf/nginx/webvirtcloud.conf /etc/nginx/conf.d/webvirtcloud.conf diff "$b/webvirtcloud.conf" /etc/nginx/conf.d/webvirtcloud.conf ``` - Put back your own lines (`server_name`, TLS, the noVNC port), then reload nginx and start the panel: + Debian / Ubuntu: ```bash - nginx -t && systemctl reload nginx && supervisorctl start webvirtcloud novncd + cp conf/supervisor/webvirtcloud.conf /etc/supervisor/conf.d/ + diff "$b/supervisor.conf" /etc/supervisor/conf.d/webvirtcloud.conf + ``` + + RHEL / openSUSE: + + ```bash + sed 's/^user=.*/user=nginx/' conf/supervisor/webvirtcloud.conf > /etc/supervisord.d/webvirtcloud.ini + diff "$b/supervisor.conf" /etc/supervisord.d/webvirtcloud.ini + ``` + + Put back your own lines (`server_name`, TLS, the noVNC port, supervisor `environment=` settings), then reload nginx and start the panel: + + ```bash + nginx -t && systemctl reload nginx && supervisorctl update && supervisorctl start webvirtcloud novncd webvirt-worker ``` Older versions collected static files into `static/`; `git clean -n static/` lists the leftovers, which can be deleted. @@ -493,7 +517,7 @@ Docker, in the directory you installed from: ```bash cd webvirtcloud && b=../webvirtcloud-backup-YYYYMMDD-HHMMSS && -docker compose stop && +docker compose stop -t 95 && docker compose run --rm --no-deps -T --entrypoint /srv/webvirtcloud/venv/bin/python3 webvirtcloud \ -c "import os,sqlite3,sys,tempfile; t=tempfile.mktemp(); open(t,'wb').write(sys.stdin.buffer.read()); [os.remove(p) for p in ('data/db.sqlite3-journal','data/db.sqlite3-wal','data/db.sqlite3-shm') if os.path.exists(p)]; sqlite3.connect(t).backup(sqlite3.connect('data/db.sqlite3'))" < "$b/db.sqlite3" && git checkout "$(cat "$b/commit")" && @@ -505,9 +529,9 @@ Installer and manual installation, from a root shell (`nginx:nginx` on RHEL / op ```bash b=/root/webvirtcloud-backup-YYYYMMDD-HHMMSS -supervisorctl stop webvirtcloud novncd +supervisorctl stop webvirtcloud novncd webvirt-worker cd /srv/webvirtcloud && -! supervisorctl status webvirtcloud novncd | grep RUNNING && +! supervisorctl status webvirtcloud novncd webvirt-worker | grep RUNNING && git checkout -- static/css/wvc-main.min.css dev/scss/wvc-main.scss && git checkout "$(cat "$b/commit")" && venv/bin/python3 -c "import os,sqlite3,sys; [os.remove(p) for p in ('db.sqlite3-journal','db.sqlite3-wal','db.sqlite3-shm') if os.path.exists(p)]; sqlite3.connect(sys.argv[1]).backup(sqlite3.connect('db.sqlite3'))" "$b/db.sqlite3" && @@ -516,12 +540,29 @@ cp "$b/webvirtcloud.conf" /etc/nginx/conf.d/webvirtcloud.conf && venv/bin/pip3 install -r conf/requirements.txt && venv/bin/python3 manage.py collectstatic --noinput && chown -R www-data:www-data /srv/webvirtcloud && -chmod 600 db.sqlite3 webvirtcloud/settings.py && -nginx -t && systemctl reload nginx && -supervisorctl start webvirtcloud novncd +chmod 600 db.sqlite3 webvirtcloud/settings.py ``` -A rollback leaves the clone on a detached HEAD at the restored commit; the update steps record that commit before they switch back to `master`. +Restore the supervisor config, Debian / Ubuntu: + +```bash +cp "$b/supervisor.conf" /etc/supervisor/conf.d/webvirtcloud.conf +``` + +RHEL / openSUSE: + +```bash +cp "$b/supervisor.conf" /etc/supervisord.d/webvirtcloud.ini +``` + +Then reload nginx and start the panel: + +```bash +nginx -t && systemctl reload nginx && +supervisorctl update && supervisorctl start webvirtcloud novncd webvirt-worker +``` + +When the restored version predates the worker, `supervisorctl` reports that `webvirt-worker` does not exist; the other programs start. A rollback leaves the clone on a detached HEAD at the restored commit; the update steps record that commit before they switch back to `master`. ### Behavior changes @@ -559,6 +600,10 @@ A rollback leaves the clone on a detached HEAD at the restored commit; the updat - External snapshots use libvirt's snapshot API and need libvirt 9.9 or newer on the compute hosts. They work on UEFI VMs and running, paused or shut-off VMs, can save memory too or, on a running VM with a connected QEMU guest agent, freeze the guest's file systems (the default there), and a revert keeps disks added later (it detaches them). A snapshot is refused when a storage pool with the VM's disks has less free space than a share of those disks (10%, at least 1 GiB; both on the Settings page) plus, for the pool of the memory file, the VM's memory; this is checked before the snapshot only, nothing is reserved. A VM's snapshots form one chain: after a revert, delete the newer snapshots before taking another. Snapshot names no longer get an `s1.` prefix; existing ones keep theirs and can still be reverted and deleted, except that a snapshot an older version took of a UEFI VM can only be deleted. On NFS, libvirt removes merged snapshot files as the qemu user, who needs write access to the image directory; files it could not remove are named in a warning after the delete or revert. Deleting a VM with its data removes its snapshot files too, except a template it grew from and files another VM uses. - A storage volume a VM still needs (a disk, a backing file or a snapshot's file) is no longer deleted from the storage page or the API; the delete is refused with the reason. Only VMs on the same compute are checked. - Docker no longer publishes port 6080; the console goes through nginx on the panel's port (`WS_PUBLIC_PORT`, default 80). +- Long operations run as jobs in a worker process (`manage.py run_worker`); follow them on the VM page and on the Jobs page. +- Manual installs that keep their own `settings.py` copy the SQLite `OPTIONS` and `TEST` entries from `settings.py.template` and add `"operations"` to `INSTALLED_APPS`. +- The SQLite database must be on a local file system (WAL). +- Outside compose, stop the Docker container with `docker stop -t 95` so the worker can finish running jobs. ## Usage diff --git a/conf/runit/10_webvirtcloud_init.sh b/conf/runit/10_webvirtcloud_init.sh index eb101fdc..be6d8730 100755 --- a/conf/runit/10_webvirtcloud_init.sh +++ b/conf/runit/10_webvirtcloud_init.sh @@ -50,3 +50,6 @@ chmod 700 "/var/www/.ssh" 2>/dev/null || true # secret key and settings private to the app user (as the installer does) chmod 700 "$DATA_DIR" chmod 600 "$DATA_DIR/db.sqlite3" "$APP_DIR/webvirtcloud/settings.py" +for f in "$DATA_DIR/db.sqlite3-wal" "$DATA_DIR/db.sqlite3-shm"; do + if [ -e "$f" ]; then chown www-data:www-data "$f"; chmod 600 "$f"; fi +done diff --git a/conf/runit/webvirtcloud-worker.sh b/conf/runit/webvirtcloud-worker.sh new file mode 100755 index 00000000..6a2a599a --- /dev/null +++ b/conf/runit/webvirtcloud-worker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# The job worker (one per installation). runit stops it with TERM; it drains +# running jobs for 60 s and exits by itself. +cd /srv/webvirtcloud || exit +export WEBVIRTCLOUD_LOG_FILE=/srv/webvirtcloud/data/webvirtcloud-worker.log +exec /sbin/setuser www-data /srv/webvirtcloud/venv/bin/python3 manage.py run_worker >> /var/log/webvirtcloud-worker.log 2>&1 diff --git a/conf/supervisor/webvirtcloud.conf b/conf/supervisor/webvirtcloud.conf index d0e4c6ca..08823d3d 100644 --- a/conf/supervisor/webvirtcloud.conf +++ b/conf/supervisor/webvirtcloud.conf @@ -24,3 +24,15 @@ user=www-data autostart=false autorestart=true redirect_stderr=true + +# The job worker: one per installation. It drains running jobs for 60 s on +# stop; supervisor waits a little longer before killing it. +[program:webvirt-worker] +command=/srv/webvirtcloud/venv/bin/python3 /srv/webvirtcloud/manage.py run_worker +directory=/srv/webvirtcloud +user=www-data +environment=WEBVIRTCLOUD_LOG_FILE="/srv/webvirtcloud/data/webvirtcloud-worker.log" +autostart=true +autorestart=true +stopwaitsecs=90 +redirect_stderr=true diff --git a/conf/systemd/README.md b/conf/systemd/README.md index 4333f36e..b41990c7 100644 --- a/conf/systemd/README.md +++ b/conf/systemd/README.md @@ -13,3 +13,4 @@ Should the executable be located in another location please adjust the path The error you should be seeing is "ssh executable not found" +Use one of the two service managers: the installer sets up supervisor (`conf/supervisor/webvirtcloud.conf` runs the worker); these units are for manual installs without supervisor. Enabling both starts two workers; the second waits for the lease and does nothing. diff --git a/conf/systemd/webvirt-worker.service b/conf/systemd/webvirt-worker.service new file mode 100644 index 00000000..687f19bb --- /dev/null +++ b/conf/systemd/webvirt-worker.service @@ -0,0 +1,19 @@ +[Unit] +Description=WebVirtCloud job worker +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +Restart=on-failure +RestartSec=10 +User=www-data +Group=www-data +EnvironmentFile=/etc/default/webvirt +ExecStart=/bin/bash -c "cd ${WEBVIRT_PATH}; WEBVIRTCLOUD_LOG_FILE=${WEBVIRT_PATH}/data/webvirtcloud-worker.log ${WEBVIRT_PATH}/venv/bin/python3 manage.py run_worker" +TimeoutStopSec=90 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/docker-compose.yml b/docker-compose.yml index 977529f8..8add7d45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ services: image: webvirtcloud:latest container_name: webvirtcloud restart: unless-stopped + # above my_init's 80 s, which is above the worker's 60 s drain + stop_grace_period: 95s ports: - "80:80" environment: diff --git a/webvirtcloud/tests/test_worker_install.py b/webvirtcloud/tests/test_worker_install.py new file mode 100644 index 00000000..8887b16a --- /dev/null +++ b/webvirtcloud/tests/test_worker_install.py @@ -0,0 +1,41 @@ +"""Every install type runs the worker as www-data, with its own log file and +a stop timeout above the worker's 60 s grace.""" + +import re +from pathlib import Path + +from django.conf import settings +from django.test import SimpleTestCase + +BASE = Path(settings.BASE_DIR) + + +class WorkerInstallTestCase(SimpleTestCase): + def test_docker(self): + dockerfile = (BASE / "Dockerfile").read_text() + self.assertIn("conf/runit/webvirtcloud-worker.sh", dockerfile) + self.assertIn("/etc/service/webvirtcloud-worker/run", dockerfile) + script = (BASE / "conf/runit/webvirtcloud-worker.sh").read_text() + for text in ("/sbin/setuser www-data", "manage.py run_worker", + "WEBVIRTCLOUD_LOG_FILE=/srv/webvirtcloud/data/webvirtcloud-worker.log"): + self.assertIn(text, script) + self.assertIn("ENV KILL_PROCESS_TIMEOUT=80", dockerfile) + compose = (BASE / "docker-compose.yml").read_text() + self.assertRegex(compose, r"stop_grace_period:\s*9\ds") + # the worker's host identity lives in the data volume, not in a fixed hostname + self.assertIn("webvirtcloud-data:/srv/webvirtcloud/data", compose) + self.assertNotIn("hostname:", compose) + + def test_supervisor(self): + conf = (BASE / "conf/supervisor/webvirtcloud.conf").read_text() + block = re.search(r"\[program:webvirt-worker\](.*?)(\n\[|\Z)", conf, re.S).group(1) + for text in ("manage.py run_worker", "user=www-data", "webvirtcloud-worker.log", "autorestart=true"): + self.assertIn(text, block) + self.assertRegex(block, r"stopwaitsecs=9\d") + + def test_systemd_is_an_alternative(self): + unit = (BASE / "conf/systemd/webvirt-worker.service").read_text() + self.assertIn("run_worker", unit) + self.assertIn("Restart=on-failure", unit) + self.assertRegex(unit, r"TimeoutStopSec=9\d") + self.assertIn("one of the two", (BASE / "conf/systemd/README.md").read_text())