From 777057cb19f63190fdb43cf1aa68e7454f4c8d17 Mon Sep 17 00:00:00 2001 From: catborise Date: Tue, 3 Nov 2026 09:32:34 +0300 Subject: [PATCH] feat(appsettings): the panel backs itself up every day The only backups of the panel were the ones taken by hand before an update. Now `manage.py backup` writes a new directory under backups/ with a consistent copy of the sqlite database (the backup API, while the panel runs, checked with PRAGMA integrity_check), the secret key, the worker's node_id, settings.py, the SSH keys of the panel's user and, outside Docker, the commit. It keeps the newest 7 (--keep N) and deletes older ones only after a good backup; a failed one leaves nothing, and a second run while one runs fails. Files are mode 600, directories 700: the database holds compute passwords and keys. With PostgreSQL the database is left out. It runs daily at 03:15: in Docker from the image's cron into a new volume, webvirtcloud-backups; otherwise from webvirt-backup.timer, which the installer and the manual steps install. A backup has the layout of the update backups, so the restore steps take either. --- .dockerignore | 1 + .gitignore | 3 + Dockerfile | 5 +- README.md | 59 +++++++++---- appsettings/management/commands/backup.py | 79 +++++++++++++++++ appsettings/tests/test_backup.py | 101 ++++++++++++++++++++++ conf/runit/10_webvirtcloud_init.sh | 6 +- conf/runit/backup.cron | 2 + conf/systemd/webvirt-backup.service | 11 +++ conf/systemd/webvirt-backup.timer | 11 +++ docker-compose.yml | 4 + webvirtcloud.sh | 6 +- 12 files changed, 264 insertions(+), 24 deletions(-) create mode 100644 appsettings/management/commands/backup.py create mode 100644 appsettings/tests/test_backup.py create mode 100644 conf/runit/backup.cron create mode 100644 conf/systemd/webvirt-backup.service create mode 100644 conf/systemd/webvirt-backup.timer diff --git a/.dockerignore b/.dockerignore index 7d92d7c4..450efe7a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,6 +17,7 @@ **/.env* # Local databases and runtime data (secret key, admin password, update backups) +backups **/*.sqlite3* data **/*.bak diff --git a/.gitignore b/.gitignore index 2ffa41bd..8c5f941b 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ data/ # Local planning and agent workspace (superpowers) /docs/superpowers/ /.superpowers/ + +# Panel backups (manage.py backup) +/backups/ diff --git a/Dockerfile b/Dockerfile index 210f0464..be46843d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,6 +77,7 @@ COPY conf/runit/novncd.sh /etc/service/novnc/run COPY conf/runit/socketiod.sh /etc/service/socketiod/run COPY conf/runit/webvirtcloud.sh /etc/service/webvirtcloud/run COPY conf/runit/webvirtcloud-worker.sh /etc/service/webvirtcloud-worker/run +COPY --chmod=644 conf/runit/backup.cron /etc/cron.d/webvirtcloud-backup RUN chmod +x /etc/my_init.d/10_webvirtcloud_init.sh \ /etc/service/nginx/run \ /etc/service/nginx-log-forwarder/run \ @@ -90,7 +91,7 @@ RUN chmod +x /etc/my_init.d/10_webvirtcloud_init.sh \ HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ CMD /srv/webvirtcloud/venv/bin/python3 -c "import urllib.request as u; u.urlopen(u.Request('http://127.0.0.1/accounts/login/', headers={'X-Forwarded-Proto': 'https'}), timeout=5)" -# Declare mountable data directory for persistent SQLite and SSH keys -VOLUME ["/srv/webvirtcloud/data", "/var/www/.ssh"] +# Declare mountable data directory for persistent SQLite and SSH keys, and the backups of both +VOLUME ["/srv/webvirtcloud/data", "/var/www/.ssh", "/srv/webvirtcloud/backups"] WORKDIR /srv/webvirtcloud diff --git a/README.md b/README.md index 9b930022..555931f8 100644 --- a/README.md +++ b/README.md @@ -58,9 +58,10 @@ WebVirtCloud and `virt-manager`/`virsh` can manage the same libvirt host **concu 1. [Installation](#installation): [installer](#installer), [Docker](#docker), [manual](#manual-installation), [first login](#first-login) 2. [Adding a Compute Node](#adding-a-compute-node) 3. [Configuration](#configuration): [environment variables](#environment-variables), [HTTPS and reverse proxy](#https-and-reverse-proxy), [cloud-init datasource](#cloud-init-datasource), [LDAP](#ldap) -4. [Updating](#updating) -5. [Usage](#usage): [users and permissions](#users-roles-and-permissions), [REST API](#rest-api), [screenshots](#screenshots) -6. [Development](#development) +4. [Backups](#backups) +5. [Updating](#updating) +6. [Usage](#usage): [users and permissions](#users-roles-and-permissions), [REST API](#rest-api), [screenshots](#screenshots) +7. [Development](#development) ## Installation @@ -127,15 +128,15 @@ pip install -r conf/requirements.txt python3 manage.py migrate python3 manage.py collectstatic --noinput -# 6. systemd units (gunicorn, novncd, socketiod and the job worker) and nginx -cp conf/systemd/webvirt-{cloud,novncd,socketiod,worker}.service /etc/systemd/system/ +# 6. systemd units (gunicorn, novncd, socketiod, the job worker and the daily backup) and nginx +cp conf/systemd/webvirt-{cloud,novncd,socketiod,worker,backup}.service conf/systemd/webvirt-backup.timer /etc/systemd/system/ cp conf/nginx/webvirtcloud.conf /etc/nginx/conf.d/ rm -f /etc/nginx/sites-enabled/default # 7. Ownership, private files, start chown -R www-data:www-data /srv/webvirtcloud chmod 700 data && chmod 600 webvirtcloud/settings.py db.sqlite3 -systemctl daemon-reload && systemctl enable --now webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker +systemctl daemon-reload && systemctl enable --now webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker webvirt-backup.timer nginx -t && systemctl restart nginx ``` @@ -164,8 +165,9 @@ pip install -r conf/requirements.txt python3 manage.py migrate python3 manage.py collectstatic --noinput -# 6. systemd units (gunicorn, novncd, socketiod and the job worker, as nginx) and nginx -for u in cloud novncd socketiod worker; do sed 's/^\(User\|Group\)=.*/\1=nginx/' conf/systemd/webvirt-$u.service > /etc/systemd/system/webvirt-$u.service; done +# 6. systemd units (gunicorn, novncd, socketiod, the job worker and the daily backup, as nginx) and nginx +for u in cloud novncd socketiod worker backup; do sed 's/^\(User\|Group\)=.*/\1=nginx/' conf/systemd/webvirt-$u.service > /etc/systemd/system/webvirt-$u.service; done +cp conf/systemd/webvirt-backup.timer /etc/systemd/system/ cp conf/nginx/webvirtcloud.conf /etc/nginx/conf.d/ # make sure the default server block in /etc/nginx/nginx.conf does not also listen on port 80 @@ -177,7 +179,7 @@ restorecon -R /srv/webvirtcloud 2>/dev/null || true setsebool -P httpd_can_network_connect on 2>/dev/null || true firewall-cmd --add-service=http --permanent 2>/dev/null || true firewall-cmd --reload 2>/dev/null || true -systemctl daemon-reload && systemctl enable --now webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker +systemctl daemon-reload && systemctl enable --now webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker webvirt-backup.timer systemctl enable nginx && nginx -t && systemctl restart nginx ``` @@ -205,15 +207,16 @@ pip install -r conf/requirements.txt python3 manage.py migrate python3 manage.py collectstatic --noinput -# 6. systemd units (gunicorn, novncd, socketiod and the job worker, as nginx) and nginx -for u in cloud novncd socketiod worker; do sed 's/^\(User\|Group\)=.*/\1=nginx/' conf/systemd/webvirt-$u.service > /etc/systemd/system/webvirt-$u.service; done +# 6. systemd units (gunicorn, novncd, socketiod, the job worker and the daily backup, as nginx) and nginx +for u in cloud novncd socketiod worker backup; do sed 's/^\(User\|Group\)=.*/\1=nginx/' conf/systemd/webvirt-$u.service > /etc/systemd/system/webvirt-$u.service; done +cp conf/systemd/webvirt-backup.timer /etc/systemd/system/ cp conf/nginx/webvirtcloud.conf /etc/nginx/conf.d/ # make sure the default server block in /etc/nginx/nginx.conf does not also listen on port 80 # 7. Ownership, private files, start chown -R nginx:nginx /srv/webvirtcloud chmod 700 data && chmod 600 webvirtcloud/settings.py db.sqlite3 -systemctl daemon-reload && systemctl enable --now webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker +systemctl daemon-reload && systemctl enable --now webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker webvirt-backup.timer systemctl enable nginx && nginx -t && systemctl restart nginx ``` @@ -436,6 +439,26 @@ A VM without graphics (no `` device, a ``) opens a - One console per VM, at most 20 at a time; a second one is refused with the reason, except a superuser's, which takes the console over. - novncd and socketiod log to the journal (`journalctl -u webvirt-novncd -u webvirt-socketiod`) or, in Docker, to `docker logs`: a console that was refused or could not connect says why there. +## Backups + +The panel backs itself up every day at 03:15 (cron in Docker, `webvirt-backup.timer` otherwise) into a new directory under `/srv/webvirtcloud/backups`, in Docker the `webvirtcloud-backups` volume, and keeps the newest 7. A backup holds: + +- `db.sqlite3`: a consistent copy taken while the panel runs, checked with `PRAGMA integrity_check`; with PostgreSQL it is left out, so back up that database yourself +- `secret_key`, `node_id` and `settings.py` +- `ssh/`: the SSH keys of the panel's user, which the SSH computes accept +- `commit`, outside Docker + +The database holds compute passwords and keys: the directories are readable only by the panel's user. They are on the same host as the panel; copy them to another machine as well. A failed backup is reported in `docker logs` or by `systemctl status webvirt-backup` and deletes no older backup. + +Take one now: + +```bash +docker compose exec webvirtcloud /sbin/setuser www-data venv/bin/python3 manage.py backup # Docker +systemctl start webvirt-backup && journalctl -u webvirt-backup -n 3 # otherwise +``` + +To restore one, follow [Restore and rollback](#restore-and-rollback) with `b` set to its directory. In Docker, copy it out of the volume first (`docker compose cp webvirtcloud:/srv/webvirtcloud/backups/webvirtcloud-backup-YYYYMMDD-HHMMSS ..`) and leave out the `git checkout` and `build` lines: the backup has no commit. On a new machine, also put back `secret_key` and `node_id` into `data/` and `ssh/` into the panel user's `~/.ssh`. + ## Updating Every update starts with a backup in its own directory (mode 700, named by date) that holds the database, the commit it was taken from and, for installer and manual installations, `settings.py` and the nginx config. The database is copied with SQLite's backup API, which also finishes a write interrupted by a crash. This covers sqlite only; with PostgreSQL (`WEBVIRTCLOUD_DB_HOST`), back up that database yourself before updating. @@ -529,19 +552,20 @@ Move the `environment=` values of that config into `/etc/default/webvirtcloud` ( Put back your own lines (`server_name`, TLS, the noVNC port). Install the systemd units; your own service settings stay in `/etc/default/webvirtcloud` and in drop-ins. Debian / Ubuntu: ```bash - cp conf/systemd/webvirt-{cloud,novncd,socketiod,worker}.service /etc/systemd/system/ + cp conf/systemd/webvirt-{cloud,novncd,socketiod,worker,backup}.service conf/systemd/webvirt-backup.timer /etc/systemd/system/ ``` RHEL / openSUSE: ```bash - for u in cloud novncd socketiod worker; do sed 's/^\(User\|Group\)=.*/\1=nginx/' conf/systemd/webvirt-$u.service > /etc/systemd/system/webvirt-$u.service; done + for u in cloud novncd socketiod worker backup; do sed 's/^\(User\|Group\)=.*/\1=nginx/' conf/systemd/webvirt-$u.service > /etc/systemd/system/webvirt-$u.service; done + cp conf/systemd/webvirt-backup.timer /etc/systemd/system/ ``` Then reload nginx and start the panel: ```bash - nginx -t && systemctl reload nginx && systemctl daemon-reload && systemctl enable --now webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker + nginx -t && systemctl reload nginx && systemctl daemon-reload && systemctl enable --now webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker webvirt-backup.timer ``` Older versions collected static files into `static/`; `git clean -n static/` lists the leftovers, which can be deleted. @@ -572,10 +596,10 @@ cd /srv/webvirtcloud && ! systemctl is-active --quiet webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker && git checkout -- $(git ls-files -m -- static/css dev/scss) && git checkout "$(cat "$b/commit")" && -tar -xf "$b/local-migrations.tar" && +{ [ ! -f "$b/local-migrations.tar" ] || tar -xf "$b/local-migrations.tar"; } && venv/bin/python3 -c "import os,sqlite3,sys; [os.remove(p) for p in ('db.sqlite3-journal','db.sqlite3-wal','db.sqlite3-shm') if os.path.exists(p)]; sqlite3.connect(sys.argv[1]).backup(sqlite3.connect('db.sqlite3'))" "$b/db.sqlite3" && cp "$b/settings.py" webvirtcloud/settings.py && -cp "$b/webvirtcloud.conf" /etc/nginx/conf.d/webvirtcloud.conf && +{ [ ! -f "$b/webvirtcloud.conf" ] || cp "$b/webvirtcloud.conf" /etc/nginx/conf.d/webvirtcloud.conf; } && venv/bin/pip3 install -r conf/requirements.txt && venv/bin/python3 manage.py collectstatic --noinput && chown -R www-data:www-data /srv/webvirtcloud && @@ -650,6 +674,7 @@ A version from before v3.0.0 has no worker: `systemctl disable --now webvirt-wor - Outside compose, stop the Docker container with `docker stop -t 95` so the worker can finish running jobs. - Installer and manual installations run the panel as systemd units (`webvirt-cloud`, `webvirt-novncd`, `webvirt-socketiod`, `webvirt-worker`) and computes run `webvirt-gstfsd`, instead of supervisor; see [Moving from supervisor to systemd](#moving-from-supervisor-to-systemd). Service settings go into `/etc/default/webvirtcloud`, service output into the journal (`journalctl -u webvirt-cloud`); the app log stays in `data/`. - LDAP login's packages are in `conf/requirements-ldap.txt` and no longer installed by default, nor their build headers by the installer; the Docker image keeps them. An existing virtualenv keeps them, but only installing that file updates them (see [LDAP](#ldap) and step 3). +- The panel backs itself up every day ([Backups](#backups)); in Docker into a new volume, `webvirtcloud-backups`. Installer and manual installations get it with the units of step 4. ## Usage diff --git a/appsettings/management/commands/backup.py b/appsettings/management/commands/backup.py new file mode 100644 index 00000000..e9817cfe --- /dev/null +++ b/appsettings/management/commands/backup.py @@ -0,0 +1,79 @@ +import fcntl +import os +import shutil +import sqlite3 +import subprocess +from contextlib import closing +from datetime import datetime +from pathlib import Path + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.db import connection + +PREFIX = "webvirtcloud-backup-" + + +class Command(BaseCommand): + help = ("Copy the database, secret key, worker id, settings and the SSH keys of the panel's user into a new " + "directory in backups/, then delete all but the newest --keep backups.") + + def add_arguments(self, parser): + parser.add_argument("--keep", type=int, default=7, help="how many backups to keep (default 7)") + + def handle(self, *args, keep, **options): + if keep < 1: + raise CommandError("--keep must be at least 1") + os.umask(0o077) # the database holds compute passwords and keys + backups = settings.BASE_DIR / "backups" + backups.mkdir(mode=0o700, exist_ok=True) + with open(backups / ".lock", "w") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise CommandError("Another backup is running") from None + self.stdout.write(str(self.backup(backups, keep))) + + def backup(self, backups, keep): + name = PREFIX + datetime.now().strftime("%Y%m%d-%H%M%S") + partial = backups / f".{name}" + partial.mkdir() + try: + self.copy(partial) + partial.rename(backups / name) + except BaseException: + shutil.rmtree(partial, ignore_errors=True) + raise + # only after a good backup: a failing one never costs an older one; + # the partial ones are what a killed run left behind + for old in [*sorted(backups.glob(PREFIX + "*"))[:-keep], *backups.glob(f".{PREFIX}*")]: + shutil.rmtree(old) + return backups / name + + def copy(self, to): + database = connection.settings_dict + if database["ENGINE"] == "django.db.backends.sqlite3": + # the backup API copies a consistent state while the panel writes + read_only = Path(database["NAME"]).resolve().as_uri() + "?mode=ro" + with closing(sqlite3.connect(read_only, uri=True, timeout=30)) as source, \ + closing(sqlite3.connect(to / "db.sqlite3")) as copy: + source.backup(copy) + result = copy.execute("PRAGMA integrity_check").fetchone()[0] + if result != "ok": + raise CommandError(f"The copy of the database is damaged: {result}") + else: + self.stderr.write("The database is not sqlite and is not in the backup: back it up with its own tools.") + data = settings.BASE_DIR / "data" + for source in (data / "secret_key", data / "node_id", settings.BASE_DIR / "webvirtcloud" / "settings.py"): + if source.is_file(): + shutil.copyfile(source, to / source.name) + ssh = Path.home() / ".ssh" + if ssh.is_dir(): + (to / "ssh").mkdir() + for source in ssh.iterdir(): + if source.is_file() and not source.is_symlink(): + shutil.copyfile(source, to / "ssh" / source.name) + if (settings.BASE_DIR / ".git").exists(): + commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=settings.BASE_DIR, capture_output=True, text=True) + if commit.returncode == 0: + (to / "commit").write_text(commit.stdout) diff --git a/appsettings/tests/test_backup.py b/appsettings/tests/test_backup.py new file mode 100644 index 00000000..cde4ade0 --- /dev/null +++ b/appsettings/tests/test_backup.py @@ -0,0 +1,101 @@ +"""manage.py backup: a directory of what a new panel needs to take over, +readable only by the panel's user, the newest --keep of them kept.""" + +import fcntl +import os +import shutil +import sqlite3 +import tempfile +from contextlib import closing +from io import StringIO +from pathlib import Path +from unittest import skipUnless +from unittest.mock import patch + +from django.core.management import call_command +from django.core.management.base import CommandError +from django.db import connection +from django.test import SimpleTestCase, override_settings + +from appsettings.management.commands.backup import Command + + +SQLITE_ONLY = skipUnless(connection.vendor == "sqlite", "the backup copies only a sqlite database") + + +class BackupTests(SimpleTestCase): + def setUp(self): + self.base = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.base) + self.addCleanup(os.umask, os.umask(0o022)) # the command sets its own + for path, text in (("data/secret_key", "key"), ("data/node_id", "a1b2c3"), + ("webvirtcloud/settings.py", "DEBUG = False"), ("home/.ssh/id_ed25519", "private")): + (self.base / path).parent.mkdir(parents=True, exist_ok=True) + (self.base / path).write_text(text) + (self.base / "home/.ssh/agent").symlink_to(self.base / "data/secret_key") + overridden = override_settings(BASE_DIR=self.base) + overridden.enable() + self.addCleanup(overridden.disable) + home = patch.object(Path, "home", return_value=self.base / "home") + home.start() + self.addCleanup(home.stop) + self.backups = self.base / "backups" + + def backup(self, *args): + out = StringIO() + call_command("backup", *args, stdout=out) + return Path(out.getvalue().strip()) + + def kept(self): + return sorted(p.name for p in self.backups.iterdir() if p.name != ".lock") + + @SQLITE_ONLY + def test_a_backup_holds_the_database_keys_and_settings_for_the_panel_user_only(self): + backup = self.backup() + with closing(sqlite3.connect(backup / "db.sqlite3")) as db: + tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type = 'table'")} + self.assertIn("computes_compute", tables) + self.assertEqual(sorted(p.name for p in backup.iterdir()), ["db.sqlite3", "node_id", "secret_key", "settings.py", "ssh"]) + self.assertEqual((backup / "secret_key").read_text(), "key") + self.assertEqual([p.name for p in (backup / "ssh").iterdir()], ["id_ed25519"]) # not what a link points to + self.assertEqual(oct(self.backups.stat().st_mode & 0o777), "0o700") + self.assertEqual(oct(backup.stat().st_mode & 0o777), "0o700") + self.assertEqual(oct((backup / "db.sqlite3").stat().st_mode & 0o777), "0o600") + self.assertEqual(oct((backup / "ssh/id_ed25519").stat().st_mode & 0o777), "0o600") + + def test_only_the_newest_backups_and_no_partial_one_are_kept(self): + for name in ("webvirtcloud-backup-20260101-000000", "webvirtcloud-backup-20260102-000000", + ".webvirtcloud-backup-20260103-000000"): + (self.backups / name).mkdir(parents=True) + backup = self.backup("--keep", "2") + self.assertEqual(self.kept(), ["webvirtcloud-backup-20260102-000000", backup.name]) + + def test_a_failed_backup_leaves_nothing_and_deletes_no_older_one(self): + (self.backups / "webvirtcloud-backup-20260101-000000").mkdir(parents=True) + with patch.object(Command, "copy", side_effect=OSError("No space left on device")), self.assertRaises(OSError): + self.backup("--keep", "1") + self.assertEqual(self.kept(), ["webvirtcloud-backup-20260101-000000"]) + + @SQLITE_ONLY + def test_a_damaged_copy_fails(self): + with patch("appsettings.management.commands.backup.sqlite3") as sqlite, \ + self.assertRaisesRegex(CommandError, "damaged: row 3 missing from index"): + sqlite.connect.return_value.execute.return_value.fetchone.return_value = ("row 3 missing from index",) + self.backup() + self.assertEqual(self.kept(), []) + + def test_a_backup_fails_while_another_runs(self): + self.backups.mkdir() + with open(self.backups / ".lock", "w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + with self.assertRaisesRegex(CommandError, "Another backup is running"): + self.backup() + self.assertEqual(self.kept(), []) + + def test_a_database_other_than_sqlite_is_left_out(self): + errors = StringIO() + with patch.dict(connection.settings_dict, ENGINE="django.db.backends.postgresql"): + call_command("backup", stdout=StringIO(), stderr=errors) + backup = self.backups / self.kept()[0] + self.assertNotIn("db.sqlite3", [p.name for p in backup.iterdir()]) + self.assertIn("back it up with its own tools", errors.getvalue()) diff --git a/conf/runit/10_webvirtcloud_init.sh b/conf/runit/10_webvirtcloud_init.sh index 8c407a32..774612f1 100755 --- a/conf/runit/10_webvirtcloud_init.sh +++ b/conf/runit/10_webvirtcloud_init.sh @@ -3,7 +3,7 @@ set -e APP_DIR="/srv/webvirtcloud" DATA_DIR="$APP_DIR/data" -mkdir -p "$DATA_DIR" "/var/www/.ssh" +mkdir -p "$DATA_DIR" "/var/www/.ssh" "$APP_DIR/backups" # If settings.py doesn't exist, generate from template if [ ! -f "$APP_DIR/webvirtcloud/settings.py" ]; then @@ -32,12 +32,12 @@ echo "* Running database migrations..." "$APP_DIR/venv/bin/python3" "$APP_DIR/manage.py" migrate --noinput # Set proper permissions on runtime and data directories -chown -R www-data:www-data "$DATA_DIR" "/var/www/.ssh" +chown -R www-data:www-data "$DATA_DIR" "/var/www/.ssh" "$APP_DIR/backups" chown www-data:www-data "$APP_DIR/webvirtcloud/settings.py" 2>/dev/null || true chmod 700 "/var/www/.ssh" 2>/dev/null || true # The database holds compute passwords and password hashes; keep it, the # secret key and settings private to the app user (as the installer does) -chmod 700 "$DATA_DIR" +chmod 700 "$DATA_DIR" "$APP_DIR/backups" chmod 600 "$DATA_DIR/db.sqlite3" "$APP_DIR/webvirtcloud/settings.py" for f in "$DATA_DIR/db.sqlite3-wal" "$DATA_DIR/db.sqlite3-shm"; do if [ -e "$f" ]; then chown www-data:www-data "$f"; chmod 600 "$f"; fi diff --git a/conf/runit/backup.cron b/conf/runit/backup.cron new file mode 100644 index 00000000..b7896f94 --- /dev/null +++ b/conf/runit/backup.cron @@ -0,0 +1,2 @@ +# Daily panel backup into /srv/webvirtcloud/backups (the webvirtcloud-backups volume); the output goes to docker logs +15 3 * * * root . /etc/container_environment.sh && cd /srv/webvirtcloud && /sbin/setuser www-data venv/bin/python3 manage.py backup > /proc/1/fd/1 2>&1 diff --git a/conf/systemd/webvirt-backup.service b/conf/systemd/webvirt-backup.service new file mode 100644 index 00000000..a17657f2 --- /dev/null +++ b/conf/systemd/webvirt-backup.service @@ -0,0 +1,11 @@ +[Unit] +Description=WebVirtCloud panel backup + +[Service] +Type=oneshot +User=www-data +Group=www-data +UMask=0077 +WorkingDirectory=/srv/webvirtcloud +EnvironmentFile=-/etc/default/webvirtcloud +ExecStart=/srv/webvirtcloud/venv/bin/python3 /srv/webvirtcloud/manage.py backup diff --git a/conf/systemd/webvirt-backup.timer b/conf/systemd/webvirt-backup.timer new file mode 100644 index 00000000..d17bde3f --- /dev/null +++ b/conf/systemd/webvirt-backup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Daily WebVirtCloud panel backup + +[Timer] +OnCalendar=*-*-* 03:15 +# a backup missed while the server was off runs at the next boot +Persistent=true +RandomizedDelaySec=10min + +[Install] +WantedBy=timers.target diff --git a/docker-compose.yml b/docker-compose.yml index cb14d720..d6ce3171 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,9 +31,13 @@ services: - webvirtcloud-data:/srv/webvirtcloud/data # Persistent SSH keys for compute node authentication - webvirtcloud-ssh:/var/www/.ssh + # Daily backups of both (manage.py backup) + - webvirtcloud-backups:/srv/webvirtcloud/backups volumes: webvirtcloud-data: driver: local webvirtcloud-ssh: driver: local + webvirtcloud-backups: + driver: local diff --git a/webvirtcloud.sh b/webvirtcloud.sh index d563e683..960f5da1 100755 --- a/webvirtcloud.sh +++ b/webvirtcloud.sh @@ -210,11 +210,13 @@ configure_nginx () { configure_systemd () { # The panel's services run as the web server's user; a re-run restarts them. units="webvirt-cloud webvirt-novncd webvirt-socketiod webvirt-worker" - for unit in $units; do + for unit in $units webvirt-backup; do sed "s/^\\(User\\|Group\\)=.*/\\1=$nginx_group/" "$APP_PATH/conf/systemd/$unit.service" > "/etc/systemd/system/$unit.service" || exit 1 done + cp "$APP_PATH/conf/systemd/webvirt-backup.timer" /etc/systemd/system/ || exit 1 # shellcheck disable=SC2086 - systemctl daemon-reload && systemctl enable $units && systemctl restart $units || exit 1 + systemctl daemon-reload && systemctl enable $units && systemctl restart $units && + systemctl enable --now webvirt-backup.timer || exit 1 } create_user () {