From 8e23138ef72b2956c8fb2866933ab06aa2074b16 Mon Sep 17 00:00:00 2001 From: catborise Date: Wed, 21 Oct 2026 08:56:49 +0300 Subject: [PATCH] test: the suite runs with manage.py test --parallel Django clones a SQLite test database by copying its main file, while migrate's writes were still in the WAL file, so every clone had no tables. The project test runner writes the WAL back before cloning. Its pool workers are no longer daemonic, so the operations process tests can start their own workers. tblib carries the traceback of a failure out of a worker instead of ending the run. The suite takes 35 s on 8 CPUs instead of 133 s. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- dev/requirements.txt | 2 ++ webvirtcloud/settings.py.template | 3 +++ webvirtcloud/test_runner.py | 39 +++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 webvirtcloud/test_runner.py diff --git a/README.md b/README.md index 974104f4..26824b53 100644 --- a/README.md +++ b/README.md @@ -695,7 +695,7 @@ python manage.py test # all tests, including vrtManager ruff check . ``` -Each app keeps its tests in a `tests/` package, so `python manage.py test instances` runs one app. The suite includes mocked regression tests and disk/clone tests on libvirt's in-process `test:///default` driver. Set `WEBVIRTCLOUD_DB_HOST` (and the other `WEBVIRTCLOUD_DB_*` variables) to run it against PostgreSQL, as CI does. +Each app keeps its tests in a `tests/` package, so `python manage.py test instances` runs one app; `--parallel` spreads the tests over the CPUs. The suite includes mocked regression tests and disk/clone tests on libvirt's in-process `test:///default` driver. Set `WEBVIRTCLOUD_DB_HOST` (and the other `WEBVIRTCLOUD_DB_*` variables) to run it against PostgreSQL, as CI does. The compute tests try a local libvirt and skip when none answers; the instance tests run only when `TEST_LIBVIRT_HOST` is set. A green result with skips does not establish live KVM compatibility. The instance tests create `wvc-test-*` VMs and a temporary `wvc-test` pool (`/var/lib/libvirt/wvc-test`), remove them afterwards, and fail if any other VM or volume on the host changed: diff --git a/dev/requirements.txt b/dev/requirements.txt index 7dc6868a..8a3de2f3 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -7,3 +7,5 @@ django-debug-toolbar==8.0.0 django-debug-toolbar-template-profiler==2.1.0 pip-audit==2.10.1 ruff==0.17.0 +# tracebacks of failures in manage.py test --parallel +tblib==3.2.2 diff --git a/webvirtcloud/settings.py.template b/webvirtcloud/settings.py.template index b63d266d..f0542cf1 100644 --- a/webvirtcloud/settings.py.template +++ b/webvirtcloud/settings.py.template @@ -178,6 +178,9 @@ LOGIN_URL = "/accounts/login/" if sys.argv[1:2] == ["test"]: PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] +# --parallel on SQLite (WAL) and with the process tests +TEST_RUNNER = "webvirtcloud.test_runner.TestRunner" + # Failed logins (django-axes): a user is locked for the address the logins # come from. The limit and the lock time are on the Settings page # (LOGIN_FAILURE_LIMIT, LOGIN_LOCKOUT_MINUTES); unlock with diff --git a/webvirtcloud/test_runner.py b/webvirtcloud/test_runner.py new file mode 100644 index 00000000..f2dcc9ed --- /dev/null +++ b/webvirtcloud/test_runner.py @@ -0,0 +1,39 @@ +"""manage.py test --parallel on the SQLite database in WAL mode.""" + +import multiprocessing + +from django.test.runner import DiscoverRunner, ParallelTestSuite + + +def _init_worker(*args, **kwargs): + # Pool processes are daemonic, and a daemonic process may have no + # children; the operations process tests start workers of their own. + multiprocessing.current_process().daemon = False + ParallelTestSuite.init_worker(*args, **kwargs) + + +class _ParallelTestSuite(ParallelTestSuite): + init_worker = _init_worker + + +class TestRunner(DiscoverRunner): + parallel_test_suite = _ParallelTestSuite + + def setup_databases(self, **kwargs): + # Django clones a SQLite test database by copying its main file, but + # migrate's writes are still in the WAL file (operations/apps.py): + # create the database alone, write the WAL back, then clone. + parallel, self.parallel = self.parallel, 0 + old_config = super().setup_databases(**kwargs) + self.parallel = parallel + for connection, _, created in old_config: + if not created or parallel < 2: + continue + if connection.vendor == "sqlite": + with connection.cursor() as cursor: + cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)") + for index in range(parallel): + connection.creation.clone_test_db( + suffix=str(index + 1), verbosity=self.verbosity, keepdb=self.keepdb + ) + return old_config