fix(tests): a killed test run no longer breaks the next one

Django replaces a SQLite test database and its parallel clones by the main
file only. A run that is killed, or whose worker pool crashes on a failure
it cannot send back, leaves the clones' WAL files behind; the next run's
fresh copies then replay that run's writes, and hundreds of tests fail with
"Cannot operate on a closed database" or foreign key errors until a clean
run checkpoints the stale WAL away. The test runner now removes the test
database's and its clones' WAL files before it creates and clones them
(not with --keepdb, whose databases they belong to).

webvirtcloud/test_runner.py

@@ -1,8 +1,21 @@
 """manage.py test --parallel on the SQLite database in WAL mode."""
 
+import contextlib
+import os
+
+from django.db import connections
 from django.test.runner import DiscoverRunner
 
 
+def remove_wal_files(name):
+    """Django replaces a SQLite test database by its main file only. A run
+    that was killed or whose pool crashed leaves the database's WAL behind,
+    and SQLite would replay that run's writes into the new database."""
+    for suffix in ("-wal", "-shm"):
+        with contextlib.suppress(FileNotFoundError):
+            os.remove(f"{name}{suffix}")
+
+
 class TestRunner(DiscoverRunner):
     def build_suite(self, *args, **kwargs):
         suite = super().build_suite(*args, **kwargs)
@@ -19,6 +32,9 @@ class TestRunner(DiscoverRunner):
         # migrate's writes are still in the WAL file (operations/apps.py):
         # create the database alone, write the WAL back, then clone.
         parallel, self.parallel = self.parallel, 0
+        test_files = [c.settings_dict["TEST"]["NAME"] for c in connections.all() if c.vendor == "sqlite"]
+        for name in filter(None, test_files) if not self.keepdb else ():
+            remove_wal_files(name)
         old_config = super().setup_databases(**kwargs)
         self.parallel = parallel
         for connection, _, created in old_config:
@@ -30,6 +46,8 @@ class TestRunner(DiscoverRunner):
                     if cursor.fetchone()[0]:
                         raise RuntimeError("The test database's WAL could not be written back before cloning")
             for index in range(parallel):
+                if connection.vendor == "sqlite" and not self.keepdb:
+                    remove_wal_files(connection.creation.get_test_db_clone_settings(str(index + 1))["NAME"])
                 connection.creation.clone_test_db(
                     suffix=str(index + 1), verbosity=self.verbosity, keepdb=self.keepdb
                 )

webvirtcloud/tests/test_test_runner.py

@@ -0,0 +1,21 @@
+"""A killed test run leaves its SQLite WAL files behind; the next run removes
+them before it creates and clones the test database, or SQLite would replay
+the old run's writes into the new one."""
+
+import tempfile
+from pathlib import Path
+
+from django.test import SimpleTestCase
+
+from webvirtcloud.test_runner import remove_wal_files
+
+
+class RemoveWalFilesTests(SimpleTestCase):
+    def test_the_wal_files_go_and_the_database_stays(self):
+        with tempfile.TemporaryDirectory() as directory:
+            name = Path(directory) / "test_db_1.sqlite3"
+            for path in (name, Path(f"{name}-wal"), Path(f"{name}-shm")):
+                path.write_bytes(b"x")
+            remove_wal_files(name)
+            remove_wal_files(name)  # none left: nothing to do
+            self.assertEqual(sorted(p.name for p in Path(directory).iterdir()), ["test_db_1.sqlite3"])