From 482f5bc16dc1ddfa846d652b87c97b20996f4ff4 Mon Sep 17 00:00:00 2001 From: catborise Date: Fri, 6 Nov 2026 09:11:43 +0300 Subject: [PATCH] fix(backup): TLS client certificates persist in Docker and are backed up libvirt reads a TLS compute's client certificate from the panel user's ~/.pki/libvirt. In Docker that was /var/www/.pki inside the container, so `docker compose up -d --build` dropped it and every TLS compute stopped connecting; no backup held it on either install type, although "Moving to a new machine" says the backup carries what the computes accept, and the README never said where the certificates go. Docker keeps /var/www/.pki in a webvirtcloud-pki volume, owned by www-data and private like ~/.ssh; the backup copies ~/.pki into pki/; the moving steps put it back; and "Adding a Compute Node" says where the files go. --- Dockerfile | 4 ++-- README.md | 12 +++++++++--- appsettings/management/commands/backup.py | 8 ++++++-- appsettings/tests/test_backup.py | 9 +++++++++ conf/runit/10_webvirtcloud_init.sh | 6 +++--- docker-compose.yml | 4 ++++ 6 files changed, 33 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 42e6c814..739b11bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -93,7 +93,7 @@ RUN chmod +x /etc/my_init.d/10_webvirtcloud_init.sh \ HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ CMD /usr/local/bin/webvirtcloud-healthcheck -# Declare mountable data directory for persistent SQLite and SSH keys, and the backups of both -VOLUME ["/srv/webvirtcloud/data", "/var/www/.ssh", "/srv/webvirtcloud/backups"] +# Declare mountable data directory for persistent SQLite, SSH keys and TLS certificates, and their backups +VOLUME ["/srv/webvirtcloud/data", "/var/www/.ssh", "/var/www/.pki", "/srv/webvirtcloud/backups"] WORKDIR /srv/webvirtcloud diff --git a/README.md b/README.md index 6dde2ca3..770655c0 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,8 @@ A compute node is a KVM host the panel manages over libvirt. docker compose exec -u www-data webvirtcloud ssh root@ true # no password or prompt: ready ``` + For TLS connections, put the CA certificate and the panel's client certificate and key in the panel user's `~/.pki/libvirt/` as `cacert.pem`, `clientcert.pem` and `clientkey.pem`, readable only by that user: `$H/.pki/libvirt/` with the installer; in Docker `/var/www/.pki/libvirt/` (the `webvirtcloud-pki` volume), owned by `www-data`. + 3. Install or update the `gstfsd` daemon on the compute node (the bootstrap script does this too). It sets the root password and SSH key of a shut-off VM: ```bash @@ -474,6 +476,7 @@ The panel backs itself up every day at 03:15 (cron in Docker, `webvirt-backup.ti - `db.sqlite3`: a consistent copy taken while the panel runs, checked with `PRAGMA integrity_check`; with PostgreSQL it is left out, so back up that database yourself - `secret_key`, `node_id` and `settings.py` - `ssh/`: the SSH keys of the panel's user, which the SSH computes accept +- `pki/`: the TLS client certificates of the panel's user, which the TLS computes accept - `commit`, outside Docker The database holds compute passwords and keys: the directories are readable only by the panel's user. They are on the same host as the panel; the [off-site copy](#off-site-copy) keeps them on another server too. A failed backup is reported in `docker logs` or by `systemctl status webvirt-backup` and deletes no older backup. @@ -716,7 +719,7 @@ A version from before v3.0.0 has no worker: `systemctl disable --now webvirt-wor ### Moving to a new machine -A [backup](#backups) moves the panel to a new machine, or back onto lost volumes: the database, the secret key (signed-in users stay signed in), the worker's id and the SSH keys the computes accept. Install the panel there first, the same version or a newer one, then copy the backup directory over and set `b` to it. The backup does not hold `docker-compose.override.yml`, the nginx config or `/etc/default/webvirtcloud`: bring your own lines over yourself. Users sign in with their passwords from the backup; the new installation's `data/admin_password` no longer applies. +A [backup](#backups) moves the panel to a new machine, or back onto lost volumes: the database, the secret key (signed-in users stay signed in), the worker's id and the SSH keys and TLS certificates the computes accept. Install the panel there first, the same version or a newer one, then copy the backup directory over and set `b` to it. The backup does not hold `docker-compose.override.yml`, the nginx config or `/etc/default/webvirtcloud`: bring your own lines over yourself. Users sign in with their passwords from the backup; the new installation's `data/admin_password` no longer applies. Docker, in the directory you installed from: @@ -728,6 +731,7 @@ docker compose run --rm --no-deps -T --entrypoint /srv/webvirtcloud/venv/bin/pyt docker compose cp "$b/secret_key" webvirtcloud:/srv/webvirtcloud/data/ && docker compose cp "$b/node_id" webvirtcloud:/srv/webvirtcloud/data/ && { [ ! -d "$b/ssh" ] || docker compose cp "$b/ssh/." webvirtcloud:/var/www/.ssh/; } && +{ [ ! -d "$b/pki" ] || docker compose cp "$b/pki/." webvirtcloud:/var/www/.pki/; } && docker compose up -d # the container migrates and fixes ownership and modes at start ``` @@ -740,9 +744,10 @@ cd /srv/webvirtcloud && venv/bin/python3 -c "import os,sqlite3,sys; [os.remove(p) for p in ('db.sqlite3-journal','db.sqlite3-wal','db.sqlite3-shm') if os.path.exists(p)]; sqlite3.connect(sys.argv[1]).backup(sqlite3.connect('db.sqlite3'))" "$b/db.sqlite3" && cp "$b/secret_key" "$b/node_id" data/ && mkdir -p ~www-data/.ssh && { [ ! -d "$b/ssh" ] || cp "$b"/ssh/* ~www-data/.ssh/; } && +mkdir -p ~www-data/.pki && { [ ! -d "$b/pki" ] || cp -r "$b"/pki/. ~www-data/.pki/; } && venv/bin/python3 manage.py migrate && -chown -R www-data:www-data data ~www-data/.ssh && -chmod 700 data ~www-data/.ssh && chmod 600 db.sqlite3 && +chown -R www-data:www-data data ~www-data/.ssh ~www-data/.pki && +chmod 700 data ~www-data/.ssh ~www-data/.pki && chmod 600 db.sqlite3 && systemctl start webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker ``` @@ -763,6 +768,7 @@ What an update from an older version changes for operators and users: - LDAP login's packages are in `conf/requirements-ldap.txt` and no longer installed by default, nor their build headers by the installer; the Docker image keeps them. An existing virtualenv keeps them, but only installing that file updates them (see [LDAP](#ldap)). - The themes ship compiled; picking one on the Settings page only saves its name, and the "Theme SASS Path" setting is gone. Browsers revalidate static files on each load: files cached before the update can stay until a hard reload (Ctrl+F5) of each page; an external caching proxy may need a purge. - The panel backs itself up every day ([Backups](#backups)); in Docker into a new volume, `webvirtcloud-backups`. +- In Docker, the TLS client certificates of TLS computes are kept in a new volume, `webvirtcloud-pki` (`/var/www/.pki`), and backed up; put them there once ([Adding a Compute Node](#adding-a-compute-node), step 2). - The SQLite database must be on a local file system (WAL). Outside compose, stop the Docker container with `docker stop -t 95` so the worker can finish running jobs. - Long operations run as jobs in a worker process (`manage.py run_worker`): creating, cloning and migrating VMs, cloning volumes, taking and reverting snapshots and deleting external snapshots. The page returns at once and the job panel and the Jobs page follow them; a migration can be cancelled. The API's VM create and migrate answer 202 with the job and need a new `submission_key` per request. The create form no longer takes a console password; set it on the VM page. - An ISO upload is tracked as a job that can be cancelled. The browser sends the file, so stay on the page until it ends; an upload left idle for an hour is ended and its partial file deleted. diff --git a/appsettings/management/commands/backup.py b/appsettings/management/commands/backup.py index e9817cfe..6d2d2c46 100644 --- a/appsettings/management/commands/backup.py +++ b/appsettings/management/commands/backup.py @@ -15,8 +15,8 @@ PREFIX = "webvirtcloud-backup-" class Command(BaseCommand): - help = ("Copy the database, secret key, worker id, settings and the SSH keys of the panel's user into a new " - "directory in backups/, then delete all but the newest --keep backups.") + help = ("Copy the database, secret key, worker id, settings and the SSH keys and TLS certificates of the " + "panel's user into a new directory in backups/, then delete all but the newest --keep backups.") def add_arguments(self, parser): parser.add_argument("--keep", type=int, default=7, help="how many backups to keep (default 7)") @@ -73,6 +73,10 @@ class Command(BaseCommand): for source in ssh.iterdir(): if source.is_file() and not source.is_symlink(): shutil.copyfile(source, to / "ssh" / source.name) + # the TLS client certificates libvirt reads (~/.pki/libvirt); a link stays a link + pki = Path.home() / ".pki" + if pki.is_dir(): + shutil.copytree(pki, to / "pki", symlinks=True) if (settings.BASE_DIR / ".git").exists(): commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=settings.BASE_DIR, capture_output=True, text=True) if commit.returncode == 0: diff --git a/appsettings/tests/test_backup.py b/appsettings/tests/test_backup.py index cde4ade0..0eb7914c 100644 --- a/appsettings/tests/test_backup.py +++ b/appsettings/tests/test_backup.py @@ -63,6 +63,15 @@ class BackupTests(SimpleTestCase): self.assertEqual(oct((backup / "db.sqlite3").stat().st_mode & 0o777), "0o600") self.assertEqual(oct((backup / "ssh/id_ed25519").stat().st_mode & 0o777), "0o600") + def test_a_backup_holds_the_tls_certificates_of_the_panel_user(self): + key = self.base / "home/.pki/libvirt/clientkey.pem" + key.parent.mkdir(parents=True) + key.write_text("tls key") + key.chmod(0o600) + copy = self.backup() / "pki/libvirt/clientkey.pem" + self.assertEqual(copy.read_text(), "tls key") + self.assertEqual(oct(copy.stat().st_mode & 0o777), "0o600") + def test_only_the_newest_backups_and_no_partial_one_are_kept(self): for name in ("webvirtcloud-backup-20260101-000000", "webvirtcloud-backup-20260102-000000", ".webvirtcloud-backup-20260103-000000"): diff --git a/conf/runit/10_webvirtcloud_init.sh b/conf/runit/10_webvirtcloud_init.sh index 6909517f..7ffb39b1 100755 --- a/conf/runit/10_webvirtcloud_init.sh +++ b/conf/runit/10_webvirtcloud_init.sh @@ -3,7 +3,7 @@ set -e APP_DIR="/srv/webvirtcloud" DATA_DIR="$APP_DIR/data" -mkdir -p "$DATA_DIR" "/var/www/.ssh" "$APP_DIR/backups" +mkdir -p "$DATA_DIR" "/var/www/.ssh" "/var/www/.pki" "$APP_DIR/backups" # If settings.py doesn't exist, generate from template if [ ! -f "$APP_DIR/webvirtcloud/settings.py" ]; then @@ -32,9 +32,9 @@ echo "* Running database migrations..." "$APP_DIR/venv/bin/python3" "$APP_DIR/manage.py" migrate --noinput # Set proper permissions on runtime and data directories -chown -R www-data:www-data "$DATA_DIR" "/var/www/.ssh" "$APP_DIR/backups" +chown -R www-data:www-data "$DATA_DIR" "/var/www/.ssh" "/var/www/.pki" "$APP_DIR/backups" chown www-data:www-data "$APP_DIR/webvirtcloud/settings.py" 2>/dev/null || true -chmod 700 "/var/www/.ssh" 2>/dev/null || true +chmod 700 "/var/www/.ssh" "/var/www/.pki" 2>/dev/null || true # The database holds compute passwords and password hashes; keep it, the # secret key and settings private to the app user (as the installer does) chmod 700 "$DATA_DIR" "$APP_DIR/backups" diff --git a/docker-compose.yml b/docker-compose.yml index 319cc50c..958eb5a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,8 @@ services: - webvirtcloud-data:/srv/webvirtcloud/data # Persistent SSH keys for compute node authentication - webvirtcloud-ssh:/var/www/.ssh + # Persistent TLS client certificates for TLS computes (.pki/libvirt) + - webvirtcloud-pki:/var/www/.pki # Daily backups of both (manage.py backup) - webvirtcloud-backups:/srv/webvirtcloud/backups @@ -45,5 +47,7 @@ volumes: driver: local webvirtcloud-ssh: driver: local + webvirtcloud-pki: + driver: local webvirtcloud-backups: driver: local