mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-09 19:56:48 +00:00
addc7ccfcb
Two computes decide the owners of the same UUID: the one the VM left buries them in a tombstone (or a compute delete does), the one it went to restores them. On PostgreSQL the two ran at once: the restore still saw the old row of the uncommitted bury, skipped, and the VM ended up without owners while the tombstone stayed. SQLite was never affected: its IMMEDIATE transactions take the write lock at the start. The reconcile and the compute delete now take one transaction-level advisory lock (pg_advisory_xact_lock) when their database step starts, so ownership decisions run one after another on PostgreSQL as they do on SQLite. Reading libvirt stays outside the lock.
82 lines
3.1 KiB
Python
82 lines
3.1 KiB
Python
"""Two reconciles of different computes decide the owners of the same UUID
|
|
one after the other: the move's restore sees the bury committed. SQLite's
|
|
IMMEDIATE transactions serialize them; PostgreSQL takes the ownership lock."""
|
|
|
|
import threading
|
|
import time
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from accounts.models import UserInstance
|
|
from computes import utils
|
|
from computes.models import Compute
|
|
from django.contrib.auth import get_user_model
|
|
from django.db import connection
|
|
from django.test import TransactionTestCase
|
|
from instances.models import Instance, InstanceTombstone
|
|
|
|
UUID = "44444444-4444-4444-4444-444444444444"
|
|
|
|
|
|
class Domain:
|
|
def __init__(self, name, uuid):
|
|
self._name, self._uuid = name, uuid
|
|
|
|
def name(self):
|
|
return self._name
|
|
|
|
def UUIDString(self):
|
|
return self._uuid
|
|
|
|
|
|
class OwnershipRaceTests(TransactionTestCase):
|
|
def compute(self, name, domains):
|
|
c = Compute.objects.create(name=name, hostname=f"{name}.example", login="", password="", type=1)
|
|
c.__dict__["proxy"] = MagicMock()
|
|
c.__dict__["status"] = True
|
|
c.proxy.wvm.listAllDomains.return_value = [Domain(n, u) for n, u in domains]
|
|
return c
|
|
|
|
def test_move_seen_by_both_reconciles_at_once(self):
|
|
a = self.compute("host-a", []) # the VM left A (virsh migrate)
|
|
self.race(a, lambda: utils.refresh_instance_database(a))
|
|
|
|
def test_a_compute_delete_and_a_reconcile_at_once(self):
|
|
a = self.compute("host-a", [("vm", UUID)]) # deleted while B sees the VM
|
|
self.race(a, a.delete)
|
|
|
|
def race(self, a, decide_a):
|
|
"""decide_a buries the VM's owners on A and waits with its transaction
|
|
open while B's reconcile finds the VM; B must restore them."""
|
|
owner = get_user_model().objects.create_user("owner", password="pw")
|
|
b = self.compute("host-b", [("vm", UUID)])
|
|
vm = Instance.objects.create(compute=a, name="vm", uuid=UUID)
|
|
UserInstance.objects.create(instance=vm, user=owner, is_change=True)
|
|
|
|
buried, go = threading.Event(), threading.Event()
|
|
real_forget = utils.forget
|
|
|
|
def slow_forget(inst):
|
|
real_forget(inst) # B's row does not exist yet: owners go to a tombstone
|
|
buried.set()
|
|
go.wait(10) # A's transaction stays open here
|
|
|
|
def run(decide):
|
|
try:
|
|
decide()
|
|
finally:
|
|
connection.close()
|
|
|
|
with patch.object(utils, "forget", slow_forget):
|
|
ta = threading.Thread(target=run, args=(decide_a,))
|
|
ta.start()
|
|
self.assertTrue(buried.wait(10))
|
|
tb = threading.Thread(target=run, args=(lambda: utils.refresh_instance_database(b),))
|
|
tb.start()
|
|
time.sleep(1) # B would decide now if nothing held it
|
|
go.set()
|
|
ta.join(30)
|
|
tb.join(30)
|
|
|
|
on_b = Instance.objects.get(compute=b, uuid=UUID)
|
|
self.assertEqual(list(on_b.userinstance_set.values_list("user__username", flat=True)), ["owner"])
|
|
self.assertFalse(InstanceTombstone.objects.exists())
|