mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-11 12:46:49 +00:00
refactor(computes): the lock helpers take claims instead of file locks
This commit is contained in:
catborise
parent
fca4a5d91c
commit
68812ddbdf
18 changed files with 545 additions and 318 deletions
|
|
@ -5,6 +5,7 @@ from computes.models import Compute
|
|||
from django.contrib.auth import get_user_model
|
||||
from django.test import TestCase
|
||||
from instances.models import Flavor, Instance
|
||||
from operations import claims
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
User = get_user_model()
|
||||
|
|
@ -178,12 +179,13 @@ class APISecurityTestCase(TestCase):
|
|||
|
||||
def test_busy_instance_api_does_not_report_success(self):
|
||||
self.client.force_login(self.regular_user)
|
||||
with patch("instances.views.libvirt_instance_lock", side_effect=TimeoutError("busy")), patch(
|
||||
with patch("instances.views.libvirt_instance_lock", side_effect=claims.Busy("instance:wvc-test")), patch(
|
||||
"instances.models.wvmInstance"
|
||||
) as proxy:
|
||||
response = self.client.post(
|
||||
f"/api/v1/computes/{self.compute.pk}/instances/{self.instance1.pk}/poweron/", {}
|
||||
)
|
||||
self.assertEqual(response.status_code, 409)
|
||||
self.assertIn("busy", response.json()["detail"])
|
||||
self.assertIn("short operation", response.json()["detail"])
|
||||
self.assertIsNone(response.json()["operation"])
|
||||
proxy.assert_not_called()
|
||||
|
|
|
|||
|
|
@ -1,18 +1,20 @@
|
|||
import threading
|
||||
from types import SimpleNamespace
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
from django.db import connection
|
||||
from django.test import TransactionTestCase
|
||||
|
||||
from computes.utils import libvirt_compute_lock, libvirt_instance_lock
|
||||
from operations import claims
|
||||
|
||||
|
||||
class OperationLocksTestCase(SimpleTestCase):
|
||||
class OperationLocksTestCase(TransactionTestCase):
|
||||
def setUp(self):
|
||||
self.compute = SimpleNamespace(pk=987654321)
|
||||
self.first = SimpleNamespace(pk=987654321, compute=self.compute)
|
||||
self.second = SimpleNamespace(pk=987654322, compute=self.compute)
|
||||
self.first = SimpleNamespace(pk=987654321, uuid="wvc-test-uuid-1", compute=self.compute)
|
||||
self.second = SimpleNamespace(pk=987654322, uuid="wvc-test-uuid-2", compute=self.compute)
|
||||
|
||||
def test_sibling_vm_can_run_while_same_vm_and_reconcile_are_excluded(self):
|
||||
def test_sibling_vm_can_run_while_same_vm_and_vm_set_changes_are_refused(self):
|
||||
acquired = threading.Event()
|
||||
release = threading.Event()
|
||||
errors = []
|
||||
|
|
@ -21,37 +23,39 @@ class OperationLocksTestCase(SimpleTestCase):
|
|||
try:
|
||||
with libvirt_instance_lock(self.first):
|
||||
acquired.set()
|
||||
if not release.wait(5):
|
||||
raise TimeoutError("Test did not release the VM lock")
|
||||
if not release.wait(10):
|
||||
raise AssertionError("Test did not release the VM lock")
|
||||
except Exception as err:
|
||||
errors.append(err)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
worker = threading.Thread(target=hold_first_vm)
|
||||
worker.start()
|
||||
try:
|
||||
self.assertTrue(acquired.wait(2))
|
||||
with libvirt_instance_lock(self.second, timeout=0.1):
|
||||
self.assertTrue(acquired.wait(10))
|
||||
with libvirt_instance_lock(self.second):
|
||||
pass
|
||||
with self.assertRaises(TimeoutError):
|
||||
with libvirt_instance_lock(self.first, timeout=0.05):
|
||||
with self.assertRaises(claims.Busy):
|
||||
with libvirt_instance_lock(self.first):
|
||||
pass
|
||||
with self.assertRaises(TimeoutError):
|
||||
with libvirt_compute_lock(self.compute, timeout=0.05):
|
||||
with self.assertRaises(claims.Busy):
|
||||
with libvirt_compute_lock(self.compute):
|
||||
pass
|
||||
finally:
|
||||
release.set()
|
||||
worker.join(5)
|
||||
worker.join(10)
|
||||
self.assertFalse(worker.is_alive())
|
||||
self.assertEqual(errors, [])
|
||||
with libvirt_compute_lock(self.compute, timeout=0.1):
|
||||
with libvirt_compute_lock(self.compute):
|
||||
pass
|
||||
|
||||
def test_instance_lock_is_reentrant_without_releasing_the_outer_lock(self):
|
||||
def test_instance_lock_nests_and_cannot_be_upgraded(self):
|
||||
with libvirt_instance_lock(self.first):
|
||||
with libvirt_instance_lock(self.first):
|
||||
pass
|
||||
with self.assertRaisesRegex(RuntimeError, "Cannot upgrade"):
|
||||
with libvirt_compute_lock(self.compute, timeout=0.05):
|
||||
with self.assertRaisesRegex(RuntimeError, "upgrade"):
|
||||
with libvirt_compute_lock(self.compute):
|
||||
pass
|
||||
|
||||
def test_compute_exclusive_owner_can_take_an_instance_lock(self):
|
||||
|
|
|
|||
|
|
@ -236,85 +236,144 @@ class ComputeConcurrencyTestCase(TransactionTestCase):
|
|||
self.assertEqual(exceptions, [])
|
||||
self.assertEqual(Instance.objects.filter(compute=compute).count(), 2)
|
||||
|
||||
def test_refresh_fails_closed_on_lock_error(self):
|
||||
from unittest.mock import patch, MagicMock
|
||||
def test_refresh_skips_a_busy_compute(self):
|
||||
from unittest.mock import MagicMock
|
||||
from computes.utils import refresh_instance_database
|
||||
from instances.models import Instance
|
||||
from operations import claims
|
||||
from operations.models import Claim
|
||||
|
||||
compute = Compute.objects.create(
|
||||
name="lock-fail-test",
|
||||
hostname="127.0.0.1",
|
||||
name="lock-busy-test",
|
||||
hostname="192.0.2.10",
|
||||
login="test",
|
||||
password="pwd",
|
||||
type=CONN_SOCKET,
|
||||
)
|
||||
Instance.objects.create(compute=compute, name="existing-vm", uuid="existing-uuid-1234")
|
||||
Instance.objects.create(compute=compute, name="wvc-test-existing", uuid="existing-uuid-1234")
|
||||
|
||||
# Libvirt returns empty list - if it did not fail closed, existing-vm would be deleted!
|
||||
# Libvirt returns an empty list: a refresh that ran would delete the VM
|
||||
mock_proxy = MagicMock()
|
||||
mock_proxy.wvm.listAllDomains.return_value = []
|
||||
compute.proxy = mock_proxy
|
||||
compute.status = True # reachable
|
||||
compute.status = True
|
||||
|
||||
# 1. When lock directory cannot be acquired
|
||||
with patch("computes.utils._get_lock_directory", return_value=None):
|
||||
refresh_instance_database(compute)
|
||||
self.assertEqual(Instance.objects.filter(compute=compute).count(), 1)
|
||||
claims.admit({claims.compute_key(compute.id): Claim.EXCLUSIVE}, holder="other")
|
||||
refresh_instance_database(compute)
|
||||
self.assertEqual(Instance.objects.filter(compute=compute).count(), 1)
|
||||
mock_proxy.wvm.listAllDomains.assert_not_called()
|
||||
|
||||
# 2. When lock file is a symlink
|
||||
with patch("os.path.islink", return_value=True):
|
||||
refresh_instance_database(compute)
|
||||
self.assertEqual(Instance.objects.filter(compute=compute).count(), 1)
|
||||
def _reconcile_compute(self, name, domains):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
# 3. When lock file cannot be opened
|
||||
with patch("os.open", side_effect=OSError("Permission denied")):
|
||||
refresh_instance_database(compute)
|
||||
self.assertEqual(Instance.objects.filter(compute=compute).count(), 1)
|
||||
compute = Compute.objects.create(name=name, hostname="192.0.2.10", login="test", password="pwd",
|
||||
type=CONN_SOCKET)
|
||||
compute.proxy = MagicMock()
|
||||
compute.proxy.wvm.listAllDomains.side_effect = domains
|
||||
compute.status = True
|
||||
return compute
|
||||
|
||||
# 4. When flock times out
|
||||
curr_time = [0.0]
|
||||
def advance_time():
|
||||
curr_time[0] += 20.0
|
||||
return curr_time[0]
|
||||
def test_a_vm_operation_runs_while_a_reconcile_holds_the_compute(self):
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
from django.db import connection
|
||||
from computes.utils import libvirt_instance_lock, refresh_instance_database
|
||||
|
||||
with patch("fcntl.flock", side_effect=BlockingIOError("Resource temporarily unavailable")):
|
||||
with patch("time.monotonic", side_effect=advance_time):
|
||||
outcome = []
|
||||
|
||||
def vm_operation():
|
||||
try:
|
||||
with libvirt_instance_lock(SimpleNamespace(uuid="wvc-test-uuid-1", compute=compute)):
|
||||
outcome.append("ran")
|
||||
except Exception as err:
|
||||
outcome.append(err)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def list_domains():
|
||||
thread = threading.Thread(target=vm_operation)
|
||||
thread.start()
|
||||
thread.join(10)
|
||||
return []
|
||||
|
||||
compute = self._reconcile_compute("reconcile-shared-test", list_domains)
|
||||
refresh_instance_database(compute)
|
||||
self.assertEqual(outcome, ["ran"])
|
||||
|
||||
def test_a_second_reconcile_of_the_compute_is_skipped_meanwhile(self):
|
||||
"""Two reconciles would read libvirt at different times and could
|
||||
commit in reverse order, an older listing undoing a newer one."""
|
||||
import threading
|
||||
from django.db import connection
|
||||
from computes.utils import refresh_instance_database
|
||||
|
||||
calls = []
|
||||
|
||||
def second_reconcile():
|
||||
try:
|
||||
refresh_instance_database(compute)
|
||||
self.assertEqual(Instance.objects.filter(compute=compute).count(), 1)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def list_domains():
|
||||
calls.append("list")
|
||||
if len(calls) == 1:
|
||||
thread = threading.Thread(target=second_reconcile)
|
||||
thread.start()
|
||||
thread.join(10)
|
||||
return []
|
||||
|
||||
compute = self._reconcile_compute("reconcile-twice-test", list_domains)
|
||||
refresh_instance_database(compute)
|
||||
self.assertEqual(calls, ["list"])
|
||||
|
||||
def test_a_reconcile_runs_while_a_job_holds_the_compute_shared(self):
|
||||
from computes.utils import refresh_instance_database
|
||||
from instances.models import Instance
|
||||
from operations import claims
|
||||
from operations.models import Claim, Operation
|
||||
|
||||
dom = type("Dom", (), {"UUIDString": lambda self: "33333333-4444-5555-6666-777777777777",
|
||||
"name": lambda self: "wvc-test-vm"})()
|
||||
compute = self._reconcile_compute("reconcile-job-test", lambda: [dom])
|
||||
op = Operation.objects.create(kind="test.job", username="wvc-test-user", submission_key="key-00000001",
|
||||
state=Operation.RUNNING)
|
||||
Claim.objects.create(operation=op, resource=claims.compute_key(compute.id), mode=Claim.SHARED)
|
||||
refresh_instance_database(compute)
|
||||
self.assertTrue(Instance.objects.filter(compute=compute, name="wvc-test-vm").exists())
|
||||
|
||||
def test_libvirt_compute_lock_held_by_another_holder_is_busy(self):
|
||||
from computes.utils import libvirt_compute_lock
|
||||
from operations import claims
|
||||
from operations.models import Claim
|
||||
|
||||
def test_libvirt_compute_lock_thread_timeout(self):
|
||||
from computes.utils import libvirt_compute_lock, _get_compute_thread_lock
|
||||
compute = Compute.objects.create(
|
||||
name="thread-lock-test",
|
||||
hostname="127.0.0.1",
|
||||
hostname="192.0.2.10",
|
||||
login="test",
|
||||
password="pwd",
|
||||
type=CONN_SOCKET,
|
||||
)
|
||||
tlock = _get_compute_thread_lock(compute.id)
|
||||
tlock.acquire()
|
||||
try:
|
||||
with self.assertRaises(TimeoutError):
|
||||
with libvirt_compute_lock(compute, timeout=0.05):
|
||||
pass
|
||||
finally:
|
||||
tlock.release()
|
||||
claims.admit({claims.compute_key(compute.id): Claim.EXCLUSIVE}, holder="other")
|
||||
with self.assertRaises(claims.Busy):
|
||||
with libvirt_compute_lock(compute):
|
||||
pass
|
||||
|
||||
def test_libvirt_compute_lock_reentrant(self):
|
||||
from computes.utils import libvirt_compute_lock
|
||||
compute = Compute.objects.create(
|
||||
name="reentrant-lock-test",
|
||||
hostname="127.0.0.1",
|
||||
hostname="192.0.2.10",
|
||||
login="test",
|
||||
password="pwd",
|
||||
type=CONN_SOCKET,
|
||||
)
|
||||
executed = []
|
||||
with libvirt_compute_lock(compute, timeout=1.0):
|
||||
with libvirt_compute_lock(compute):
|
||||
executed.append("outer")
|
||||
with libvirt_compute_lock(compute, timeout=1.0):
|
||||
with libvirt_compute_lock(compute):
|
||||
executed.append("inner")
|
||||
with libvirt_compute_lock(compute, timeout=1.0):
|
||||
with libvirt_compute_lock(compute):
|
||||
executed.append("deep")
|
||||
self.assertEqual(executed, ["outer", "inner", "deep"])
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,5 @@
|
|||
from contextlib import contextmanager
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from datetime import timedelta
|
||||
|
||||
from accounts.models import UserInstance
|
||||
|
|
@ -15,142 +9,40 @@ from django.db import transaction
|
|||
from django.utils import timezone
|
||||
from computes.models import Compute
|
||||
from instances.models import Instance, InstanceTombstone
|
||||
from operations import claims
|
||||
from operations.models import Claim
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_COMPUTE_THREAD_LOCKS = {}
|
||||
_COMPUTE_THREAD_LOCKS_GUARD = threading.Lock()
|
||||
|
||||
|
||||
def _get_compute_thread_lock(compute_pk):
|
||||
with _COMPUTE_THREAD_LOCKS_GUARD:
|
||||
if compute_pk not in _COMPUTE_THREAD_LOCKS:
|
||||
_COMPUTE_THREAD_LOCKS[compute_pk] = threading.Lock()
|
||||
return _COMPUTE_THREAD_LOCKS[compute_pk]
|
||||
|
||||
|
||||
def _get_lock_directory():
|
||||
candidates = [
|
||||
"/srv/webvirtcloud/data/locks",
|
||||
os.path.join(tempfile.gettempdir(), f"webvirtcloud_locks_{os.geteuid()}"),
|
||||
]
|
||||
for candidate in candidates:
|
||||
try:
|
||||
if os.path.islink(candidate):
|
||||
continue
|
||||
if not os.path.exists(candidate):
|
||||
os.makedirs(candidate, mode=0o700, exist_ok=True)
|
||||
if not os.path.isdir(candidate) or os.path.islink(candidate):
|
||||
continue
|
||||
|
||||
stat_info = os.stat(candidate)
|
||||
# Ensure candidate directory is owned by the current process user
|
||||
if stat_info.st_uid != os.geteuid():
|
||||
continue
|
||||
# Ensure candidate directory has private permissions (mode 0700)
|
||||
if (stat.S_IMODE(stat_info.st_mode) & 0o077) != 0:
|
||||
try:
|
||||
os.chmod(candidate, 0o700)
|
||||
except OSError:
|
||||
continue
|
||||
stat_info = os.stat(candidate)
|
||||
if (stat.S_IMODE(stat_info.st_mode) & 0o077) != 0:
|
||||
continue
|
||||
|
||||
test_file = os.path.join(candidate, f".test_write_{os.getpid()}_{threading.get_ident()}")
|
||||
with open(test_file, "w") as tf:
|
||||
tf.write("1")
|
||||
os.unlink(test_file)
|
||||
return candidate
|
||||
except (OSError, PermissionError):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
_thread_local = threading.local()
|
||||
|
||||
|
||||
def _get_thread_held_locks():
|
||||
if not hasattr(_thread_local, "compute_locks"):
|
||||
_thread_local.compute_locks = {}
|
||||
return _thread_local.compute_locks
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _libvirt_lock(key, filename, timeout=15.0, shared=False):
|
||||
"""Use flock for both threads and processes; exclusive locks also use a mutex."""
|
||||
held_locks = _get_thread_held_locks()
|
||||
if key in held_locks:
|
||||
if held_locks[key] and not shared:
|
||||
raise RuntimeError("Cannot upgrade a shared compute lock inside a VM operation")
|
||||
yield
|
||||
return
|
||||
|
||||
thread_lock = None if shared else _get_compute_thread_lock(key)
|
||||
start_time = time.monotonic()
|
||||
if thread_lock is not None and not thread_lock.acquire(timeout=timeout):
|
||||
raise TimeoutError(f"Timeout waiting for lock on {key}")
|
||||
try:
|
||||
lock_dir = _get_lock_directory()
|
||||
if not lock_dir:
|
||||
raise OSError(f"Could not acquire secure lock directory for {key}")
|
||||
path = os.path.join(lock_dir, filename)
|
||||
if os.path.islink(path):
|
||||
raise PermissionError(f"Lock file {path} is a symlink")
|
||||
fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
||||
acquired = False
|
||||
try:
|
||||
mode = fcntl.LOCK_SH if shared else fcntl.LOCK_EX
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(fd, mode | fcntl.LOCK_NB)
|
||||
acquired = True
|
||||
break
|
||||
except BlockingIOError:
|
||||
if time.monotonic() - start_time >= timeout:
|
||||
raise TimeoutError(f"Timeout waiting for lock on {key}")
|
||||
time.sleep(0.05)
|
||||
held_locks[key] = shared
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
held_locks.pop(key, None)
|
||||
finally:
|
||||
try:
|
||||
if acquired:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
finally:
|
||||
os.close(fd)
|
||||
finally:
|
||||
if thread_lock is not None:
|
||||
thread_lock.release()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def libvirt_compute_lock(compute, timeout=15.0, *, shared=False):
|
||||
"""Exclude reconciliation/migration while allowing independent VM operations."""
|
||||
def libvirt_compute_lock(compute, *, shared=False):
|
||||
"""The compute for the block: exclusive keeps out every VM operation and
|
||||
reconcile on it; shared lets sibling VMs and reconciles go on. Busy if it is taken."""
|
||||
compute_pk = getattr(compute, "pk", getattr(compute, "id", None))
|
||||
if not compute_pk:
|
||||
yield
|
||||
return
|
||||
with _libvirt_lock(compute_pk, f"sync_compute_{compute_pk}.lock", timeout, shared):
|
||||
with claims.hold({claims.compute_key(compute_pk): Claim.SHARED if shared else Claim.EXCLUSIVE}):
|
||||
yield
|
||||
|
||||
|
||||
@contextmanager
|
||||
def user_quota_lock(user, timeout=15.0):
|
||||
"""One user's quota check and the change it allows run one at a time, on every compute.
|
||||
Take it before any compute or VM lock."""
|
||||
with _libvirt_lock(("quota", user.pk), f"quota_user_{user.pk}.lock", timeout):
|
||||
def user_quota_lock(user):
|
||||
"""One user's quota check and the change it allows run one at a time."""
|
||||
with claims.hold({claims.quota_key(user.pk): Claim.EXCLUSIVE}):
|
||||
yield
|
||||
|
||||
|
||||
@contextmanager
|
||||
def libvirt_instance_lock(instance, timeout=15.0):
|
||||
"""Serialize one VM and share the compute barrier with other VM operations."""
|
||||
with libvirt_compute_lock(instance.compute, timeout, shared=True):
|
||||
with _libvirt_lock(("instance", instance.pk), f"instance_{instance.pk}.lock", timeout):
|
||||
yield
|
||||
def libvirt_instance_lock(instance):
|
||||
"""One VM for the block, sharing its compute with sibling VMs."""
|
||||
compute = instance.compute
|
||||
with claims.hold({
|
||||
claims.compute_key(getattr(compute, "pk", getattr(compute, "id", None))): Claim.SHARED,
|
||||
claims.instance_key(instance.uuid): Claim.EXCLUSIVE,
|
||||
}):
|
||||
yield
|
||||
|
||||
|
||||
def _move_ownership(source, target):
|
||||
|
|
@ -219,11 +111,16 @@ def refresh_instance_database(compute):
|
|||
return
|
||||
|
||||
try:
|
||||
with libvirt_compute_lock(compute, timeout=15.0):
|
||||
# The compute shared: VM operations and jobs on it go on; changes of
|
||||
# its VM set (create, clone, delete, migration, define) hold it
|
||||
# exclusive and stay out until the end. One reconcile of a compute at
|
||||
# a time: two would read libvirt at different times and could commit
|
||||
# in reverse order, an older listing undoing a newer one.
|
||||
with claims.hold({claims.compute_key(compute_pk): Claim.SHARED,
|
||||
claims.reconcile_key(compute_pk): Claim.EXCLUSIVE}):
|
||||
# libvirt is asked before the transaction: a silent host can keep a
|
||||
# call waiting for 30 s, and on SQLite an open transaction makes every
|
||||
# other write of the app fail meanwhile. The compute lock still keeps
|
||||
# create/delete/migration of this compute out until the end.
|
||||
# other write of the app fail meanwhile.
|
||||
if hasattr(compute, "status") and compute.status is False:
|
||||
return
|
||||
|
||||
|
|
@ -294,6 +191,6 @@ def refresh_instance_database(compute):
|
|||
)
|
||||
if created and not Instance.objects.filter(uuid=uuid).exclude(compute=compute).exists():
|
||||
_restore(inst)
|
||||
except (OSError, PermissionError, TimeoutError) as e:
|
||||
logger.error("Could not synchronize instances for compute %s: %s; failing closed", compute_pk, e)
|
||||
except claims.Busy:
|
||||
logger.debug("Compute %s is busy; not synchronized now", compute_pk)
|
||||
return
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue