refactor(computes): the lock helpers take claims instead of file locks

computes/tests/test_api_security.py

@@ -5,6 +5,7 @@ from computes.models import Compute
 from django.contrib.auth import get_user_model
 from django.test import TestCase
 from instances.models import Flavor, Instance
+from operations import claims
 from rest_framework.test import APIClient
 
 User = get_user_model()
@@ -178,12 +179,13 @@ class APISecurityTestCase(TestCase):
 
     def test_busy_instance_api_does_not_report_success(self):
         self.client.force_login(self.regular_user)
-        with patch("instances.views.libvirt_instance_lock", side_effect=TimeoutError("busy")), patch(
+        with patch("instances.views.libvirt_instance_lock", side_effect=claims.Busy("instance:wvc-test")), patch(
             "instances.models.wvmInstance"
         ) as proxy:
             response = self.client.post(
                 f"/api/v1/computes/{self.compute.pk}/instances/{self.instance1.pk}/poweron/", {}
             )
         self.assertEqual(response.status_code, 409)
-        self.assertIn("busy", response.json()["detail"])
+        self.assertIn("short operation", response.json()["detail"])
+        self.assertIsNone(response.json()["operation"])
         proxy.assert_not_called()

computes/tests/test_operation_locks.py

@@ -1,18 +1,20 @@
 import threading
 from types import SimpleNamespace
 
-from django.test import SimpleTestCase
+from django.db import connection
+from django.test import TransactionTestCase
 
 from computes.utils import libvirt_compute_lock, libvirt_instance_lock
+from operations import claims
 
 
-class OperationLocksTestCase(SimpleTestCase):
+class OperationLocksTestCase(TransactionTestCase):
     def setUp(self):
         self.compute = SimpleNamespace(pk=987654321)
-        self.first = SimpleNamespace(pk=987654321, compute=self.compute)
-        self.second = SimpleNamespace(pk=987654322, compute=self.compute)
+        self.first = SimpleNamespace(pk=987654321, uuid="wvc-test-uuid-1", compute=self.compute)
+        self.second = SimpleNamespace(pk=987654322, uuid="wvc-test-uuid-2", compute=self.compute)
 
-    def test_sibling_vm_can_run_while_same_vm_and_reconcile_are_excluded(self):
+    def test_sibling_vm_can_run_while_same_vm_and_vm_set_changes_are_refused(self):
         acquired = threading.Event()
         release = threading.Event()
         errors = []
@@ -21,37 +23,39 @@ class OperationLocksTestCase(SimpleTestCase):
             try:
                 with libvirt_instance_lock(self.first):
                     acquired.set()
-                    if not release.wait(5):
-                        raise TimeoutError("Test did not release the VM lock")
+                    if not release.wait(10):
+                        raise AssertionError("Test did not release the VM lock")
             except Exception as err:
                 errors.append(err)
+            finally:
+                connection.close()
 
         worker = threading.Thread(target=hold_first_vm)
         worker.start()
         try:
-            self.assertTrue(acquired.wait(2))
-            with libvirt_instance_lock(self.second, timeout=0.1):
+            self.assertTrue(acquired.wait(10))
+            with libvirt_instance_lock(self.second):
                 pass
-            with self.assertRaises(TimeoutError):
-                with libvirt_instance_lock(self.first, timeout=0.05):
+            with self.assertRaises(claims.Busy):
+                with libvirt_instance_lock(self.first):
                     pass
-            with self.assertRaises(TimeoutError):
-                with libvirt_compute_lock(self.compute, timeout=0.05):
+            with self.assertRaises(claims.Busy):
+                with libvirt_compute_lock(self.compute):
                     pass
         finally:
             release.set()
-            worker.join(5)
+            worker.join(10)
         self.assertFalse(worker.is_alive())
         self.assertEqual(errors, [])
-        with libvirt_compute_lock(self.compute, timeout=0.1):
+        with libvirt_compute_lock(self.compute):
             pass
 
-    def test_instance_lock_is_reentrant_without_releasing_the_outer_lock(self):
+    def test_instance_lock_nests_and_cannot_be_upgraded(self):
         with libvirt_instance_lock(self.first):
             with libvirt_instance_lock(self.first):
                 pass
-            with self.assertRaisesRegex(RuntimeError, "Cannot upgrade"):
-                with libvirt_compute_lock(self.compute, timeout=0.05):
+            with self.assertRaisesRegex(RuntimeError, "upgrade"):
+                with libvirt_compute_lock(self.compute):
                     pass
 
     def test_compute_exclusive_owner_can_take_an_instance_lock(self):

computes/tests/test_views.py

@@ -236,85 +236,144 @@ class ComputeConcurrencyTestCase(TransactionTestCase):
         self.assertEqual(exceptions, [])
         self.assertEqual(Instance.objects.filter(compute=compute).count(), 2)
 
-    def test_refresh_fails_closed_on_lock_error(self):
-        from unittest.mock import patch, MagicMock
+    def test_refresh_skips_a_busy_compute(self):
+        from unittest.mock import MagicMock
         from computes.utils import refresh_instance_database
         from instances.models import Instance
+        from operations import claims
+        from operations.models import Claim
 
         compute = Compute.objects.create(
-            name="lock-fail-test",
-            hostname="127.0.0.1",
+            name="lock-busy-test",
+            hostname="192.0.2.10",
             login="test",
             password="pwd",
             type=CONN_SOCKET,
         )
-        Instance.objects.create(compute=compute, name="existing-vm", uuid="existing-uuid-1234")
+        Instance.objects.create(compute=compute, name="wvc-test-existing", uuid="existing-uuid-1234")
 
-        # Libvirt returns empty list - if it did not fail closed, existing-vm would be deleted!
+        # Libvirt returns an empty list: a refresh that ran would delete the VM
         mock_proxy = MagicMock()
         mock_proxy.wvm.listAllDomains.return_value = []
         compute.proxy = mock_proxy
-        compute.status = True  # reachable
+        compute.status = True
 
-        # 1. When lock directory cannot be acquired
-        with patch("computes.utils._get_lock_directory", return_value=None):
-            refresh_instance_database(compute)
-            self.assertEqual(Instance.objects.filter(compute=compute).count(), 1)
+        claims.admit({claims.compute_key(compute.id): Claim.EXCLUSIVE}, holder="other")
+        refresh_instance_database(compute)
+        self.assertEqual(Instance.objects.filter(compute=compute).count(), 1)
+        mock_proxy.wvm.listAllDomains.assert_not_called()
 
-        # 2. When lock file is a symlink
-        with patch("os.path.islink", return_value=True):
-            refresh_instance_database(compute)
-            self.assertEqual(Instance.objects.filter(compute=compute).count(), 1)
+    def _reconcile_compute(self, name, domains):
+        from unittest.mock import MagicMock
 
-        # 3. When lock file cannot be opened
-        with patch("os.open", side_effect=OSError("Permission denied")):
-            refresh_instance_database(compute)
-            self.assertEqual(Instance.objects.filter(compute=compute).count(), 1)
+        compute = Compute.objects.create(name=name, hostname="192.0.2.10", login="test", password="pwd",
+                                         type=CONN_SOCKET)
+        compute.proxy = MagicMock()
+        compute.proxy.wvm.listAllDomains.side_effect = domains
+        compute.status = True
+        return compute
 
-        # 4. When flock times out
-        curr_time = [0.0]
-        def advance_time():
-            curr_time[0] += 20.0
-            return curr_time[0]
+    def test_a_vm_operation_runs_while_a_reconcile_holds_the_compute(self):
+        import threading
+        from types import SimpleNamespace
+        from django.db import connection
+        from computes.utils import libvirt_instance_lock, refresh_instance_database
 
-        with patch("fcntl.flock", side_effect=BlockingIOError("Resource temporarily unavailable")):
-            with patch("time.monotonic", side_effect=advance_time):
+        outcome = []
+
+        def vm_operation():
+            try:
+                with libvirt_instance_lock(SimpleNamespace(uuid="wvc-test-uuid-1", compute=compute)):
+                    outcome.append("ran")
+            except Exception as err:
+                outcome.append(err)
+            finally:
+                connection.close()
+
+        def list_domains():
+            thread = threading.Thread(target=vm_operation)
+            thread.start()
+            thread.join(10)
+            return []
+
+        compute = self._reconcile_compute("reconcile-shared-test", list_domains)
+        refresh_instance_database(compute)
+        self.assertEqual(outcome, ["ran"])
+
+    def test_a_second_reconcile_of_the_compute_is_skipped_meanwhile(self):
+        """Two reconciles would read libvirt at different times and could
+        commit in reverse order, an older listing undoing a newer one."""
+        import threading
+        from django.db import connection
+        from computes.utils import refresh_instance_database
+
+        calls = []
+
+        def second_reconcile():
+            try:
                 refresh_instance_database(compute)
-                self.assertEqual(Instance.objects.filter(compute=compute).count(), 1)
+            finally:
+                connection.close()
+
+        def list_domains():
+            calls.append("list")
+            if len(calls) == 1:
+                thread = threading.Thread(target=second_reconcile)
+                thread.start()
+                thread.join(10)
+            return []
+
+        compute = self._reconcile_compute("reconcile-twice-test", list_domains)
+        refresh_instance_database(compute)
+        self.assertEqual(calls, ["list"])
+
+    def test_a_reconcile_runs_while_a_job_holds_the_compute_shared(self):
+        from computes.utils import refresh_instance_database
+        from instances.models import Instance
+        from operations import claims
+        from operations.models import Claim, Operation
+
+        dom = type("Dom", (), {"UUIDString": lambda self: "33333333-4444-5555-6666-777777777777",
+                               "name": lambda self: "wvc-test-vm"})()
+        compute = self._reconcile_compute("reconcile-job-test", lambda: [dom])
+        op = Operation.objects.create(kind="test.job", username="wvc-test-user", submission_key="key-00000001",
+                                      state=Operation.RUNNING)
+        Claim.objects.create(operation=op, resource=claims.compute_key(compute.id), mode=Claim.SHARED)
+        refresh_instance_database(compute)
+        self.assertTrue(Instance.objects.filter(compute=compute, name="wvc-test-vm").exists())
+
+    def test_libvirt_compute_lock_held_by_another_holder_is_busy(self):
+        from computes.utils import libvirt_compute_lock
+        from operations import claims
+        from operations.models import Claim
 
-    def test_libvirt_compute_lock_thread_timeout(self):
-        from computes.utils import libvirt_compute_lock, _get_compute_thread_lock
         compute = Compute.objects.create(
             name="thread-lock-test",
-            hostname="127.0.0.1",
+            hostname="192.0.2.10",
             login="test",
             password="pwd",
             type=CONN_SOCKET,
         )
-        tlock = _get_compute_thread_lock(compute.id)
-        tlock.acquire()
-        try:
-            with self.assertRaises(TimeoutError):
-                with libvirt_compute_lock(compute, timeout=0.05):
-                    pass
-        finally:
-            tlock.release()
+        claims.admit({claims.compute_key(compute.id): Claim.EXCLUSIVE}, holder="other")
+        with self.assertRaises(claims.Busy):
+            with libvirt_compute_lock(compute):
+                pass
 
     def test_libvirt_compute_lock_reentrant(self):
         from computes.utils import libvirt_compute_lock
         compute = Compute.objects.create(
             name="reentrant-lock-test",
-            hostname="127.0.0.1",
+            hostname="192.0.2.10",
             login="test",
             password="pwd",
             type=CONN_SOCKET,
         )
         executed = []
-        with libvirt_compute_lock(compute, timeout=1.0):
+        with libvirt_compute_lock(compute):
             executed.append("outer")
-            with libvirt_compute_lock(compute, timeout=1.0):
+            with libvirt_compute_lock(compute):
                 executed.append("inner")
-                with libvirt_compute_lock(compute, timeout=1.0):
+                with libvirt_compute_lock(compute):
                     executed.append("deep")
         self.assertEqual(executed, ["outer", "inner", "deep"])