feat(instances): VM clone runs as a job

instances/tests/test_clone_disks.py

@@ -11,6 +11,7 @@ from accounts.models import UserInstance
 from appsettings.models import AppSettings
 from computes.models import Compute
 from instances.models import Instance
+from operations.models import Operation
 
 LIVE = [{"dev": "vda", "path": "/pool/live.qcow2", "storage": "pool", "image": "live.qcow2", "size": 1 << 30}]
 SAVED = [{"dev": "vdb", "path": "/pool/saved.qcow2", "storage": "pool", "image": "saved.qcow2", "size": 20 << 30}]
@@ -30,16 +31,16 @@ class CloneDisksTestCase(TestCase):
             wvm.return_value.get_vcpu.return_value = 1
             wvm.return_value.get_memory.return_value = 512
             wvm.return_value.get_net_devices.return_value = []
-            wvm.return_value.clone_instance.return_value = "abcdefab-0000-1111-2222-333333333333"
-            response = self.client.post(reverse("instances:clone", args=[self.vm.id]), post)
+            wvm.return_value.get_status.return_value = 5
+            response = self.client.post(reverse("instances:clone", args=[self.vm.id]), {"submission_key": "key-clone-disk1", **post})
             self.messages = [str(m) for m in response.wsgi_request._messages]
-        return quota, wvm.return_value.clone_instance
+        return quota
 
     def test_the_quota_counts_the_disks_the_clone_copies(self):
         owner = get_user_model().objects.create_user("cd_owner", password="x")
         owner.user_permissions.add(Permission.objects.get(codename="clone_instances"))
         UserInstance.objects.create(instance=self.vm, user=owner, is_change=True)
-        quota, _ = self.clone(owner, name="cd-clone")
+        quota = self.clone(owner, name="cd-clone")
         self.assertEqual(quota.call_args.args[4], 20)
 
     def test_automatic_disk_names_are_for_the_disks_the_clone_copies(self):
@@ -48,7 +49,7 @@ class CloneDisksTestCase(TestCase):
         with patch("instances.views.utils.get_clone_free_names", return_value=["auto-1"]), patch(
             "instances.views.utils.get_dhcp_mac_address", return_value="52:54:00:00:00:01"
         ):
-            _, clone_instance = self.clone(admin)
-        data = clone_instance.call_args.args[0]
+            self.clone(admin)
+        data = Operation.objects.get().intent["clone"]
         self.assertIn("disk-vdb", data)
         self.assertNotIn("disk-vda", data)

instances/tests/test_clone_job.py

@@ -0,0 +1,112 @@
+"""VM clone runs as a job: the page queues it, the worker clones and records it."""
+
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import libvirt
+from django.contrib.auth import get_user_model
+from django.core.exceptions import PermissionDenied
+from django.test import TransactionTestCase
+from django.urls import reverse
+
+from accounts.models import UserInstance
+from appsettings.settings import get_settings
+from computes.models import Compute
+from instances.jobs import CloneInstance
+from instances.models import Instance
+from operations.jobs import Context
+from vrtManager.util import OperationError
+from operations.models import Claim, Operation, acquire
+
+
+def proxy():
+    p = MagicMock()
+    p.get_status.return_value = 5
+    p.get_disk_devices.return_value = [{"dev": "vda", "image": "src.qcow2", "size": 1 << 30}]
+    p.get_net_devices.return_value = [{"mac": "52:54:00:aa:bb:cc"}]
+    p.get_vcpu.return_value = 1
+    p.get_memory.return_value = 512
+    return p
+
+
+def pool(volumes):
+    p = MagicMock()
+    p.listVolumes.return_value = volumes
+    return p
+
+
+class CloneJobTestCase(TransactionTestCase):
+    def setUp(self):
+        get_settings()
+        self.admin = get_user_model().objects.create_superuser("clone_admin", "a@example.com", "pw")
+        self.client.force_login(self.admin)
+        self.compute = Compute.objects.create(name="clone-c", hostname="192.0.2.30", login="root", password="", type=1)
+        self.source = Instance.objects.create(compute=self.compute, name="src", uuid="12121212-3434-5656-7878-909090909090")
+        patcher = patch("instances.models.wvmInstance", return_value=proxy())
+        self.proxy = patcher.start().return_value
+        self.addCleanup(patcher.stop)
+
+    def test_the_page_queues_the_clone_and_the_worker_makes_and_records_it(self):
+        response = self.client.post(reverse("instances:clone", args=[self.source.pk]),
+                                    {"name": "copy", "disk-vda": "copy.qcow2", "clone-net-mac-0": "52:54:00:00:00:01",
+                                     "submission_key": "key-clone001"})
+        self.assertRedirects(response, reverse("instances:instance", args=[self.source.pk]),
+                             fetch_redirect_response=False)
+        op = Operation.objects.get()
+        c = self.compute.pk
+        self.assertEqual(set(Claim.objects.values_list("resource", "mode")), {
+            (f"compute:{c}", "shared"), (f"reconcile:{c}", "exclusive"),
+            (f"instance:{self.source.uuid}", "shared"), (f"name:{c}:copy", "exclusive")})
+        self.proxy.clone_instance.assert_not_called()
+
+        acquire("wvc-test:1:a", "wvc-test")
+        Operation.objects.filter(pk=op.pk).update(state=Operation.RUNNING, worker="wvc-test:1:a")
+        op.refresh_from_db()
+        CloneInstance().run(op, Context(op, SimpleNamespace(identity="wvc-test:1:a")))
+        self.proxy.clone_instance.assert_called_once_with(op.intent["clone"], op.intent["uuid"])
+        clone = Instance.objects.get(uuid=op.intent["uuid"])
+        self.assertTrue(UserInstance.objects.filter(instance=clone, user=self.admin).exists())
+
+    def queued(self):
+        self.client.post(reverse("instances:clone", args=[self.source.pk]),
+                         {"name": "copy", "disk-vda": "copy.qcow2", "clone-net-mac-0": "52:54:00:00:00:01",
+                          "submission_key": "key-clone002"})
+        return Operation.objects.get()
+
+    def test_the_worker_refuses_a_clone_whose_access_was_revoked(self):
+        op = self.queued()
+        self.admin.is_superuser = False
+        self.admin.save()
+        with self.assertRaises(PermissionDenied):
+            CloneInstance().precheck(op, None)
+
+    def test_the_worker_refuses_a_clone_whose_source_moved(self):
+        op = self.queued()
+        other = Compute.objects.create(name="clone-d", hostname="192.0.2.31", login="root", password="", type=1)
+        Instance.objects.filter(pk=self.source.pk).update(compute=other)
+        with self.assertRaises(OperationError):
+            CloneInstance().precheck(op, None)
+
+    @patch("instances.jobs.wvmConnect")
+    def test_recovery(self, conn):
+        op = Operation(user=self.admin, intent={"compute": self.compute.pk, "uuid": "abababab-0000-1111-2222-333333333333",
+                                                "name": "copy", "clone": {"name": "copy", "disk-vda": "copy.qcow2"}})
+        self.assertEqual(CloneInstance().reconcile(op).state, Operation.SUCCEEDED)  # the VM is defined
+        self.assertTrue(Instance.objects.filter(uuid=op.intent["uuid"]).exists())
+        missing = libvirt.libvirtError("no domain")
+        missing.err = (libvirt.VIR_ERR_NO_DOMAIN, libvirt.VIR_FROM_QEMU, "no domain")
+        conn.return_value.wvm.lookupByUUIDString.side_effect = missing
+        conn.return_value.wvm.listAllStoragePools.return_value = [pool(["copy.qcow2", "other.qcow2"])]
+        outcome = CloneInstance().reconcile(op)
+        self.assertEqual(outcome.state, Operation.UNCERTAIN)
+        self.assertIn("copy.qcow2", outcome.error)
+
+    @patch("instances.jobs.wvmConnect")
+    def test_a_clone_that_left_nothing_failed(self, conn):
+        op = Operation(user=self.admin, intent={"compute": self.compute.pk, "uuid": "abababab-0000-1111-2222-333333333333",
+                                                "name": "copy", "clone": {"name": "copy", "disk-vda": "copy.qcow2"}})
+        missing = libvirt.libvirtError("no domain")
+        missing.err = (libvirt.VIR_ERR_NO_DOMAIN, libvirt.VIR_FROM_QEMU, "no domain")
+        conn.return_value.wvm.lookupByUUIDString.side_effect = missing
+        conn.return_value.wvm.listAllStoragePools.return_value = [pool(["other.qcow2"])]
+        self.assertEqual(CloneInstance().reconcile(op).state, Operation.FAILED)

instances/tests/test_clone_permission.py

@@ -17,6 +17,7 @@ from appsettings.settings import app_settings
 from computes.models import Compute
 from instances.models import Instance
 from instances.utils import get_dhcp_mac_address
+from operations.models import Operation
 
 SRC_DISK = {
     "dev": "vda",
@@ -71,9 +72,9 @@ class ClonePermissionTestCase(TestCase):
             proxy = mock_wvm.return_value
             proxy.get_disk_devices.return_value = [SRC_DISK]
             proxy.get_net_devices.return_value = [{"mac": "52:54:00:aa:bb:cc"}]
+            proxy.get_status.return_value = 5
             proxy.get_vcpu.return_value = 1
             proxy.get_memory.return_value = 512
-            proxy.clone_instance.return_value = "abababab-cdcd-efef-0101-232323232323"
             yield proxy
 
     def _clone(self, user, data):
@@ -91,50 +92,49 @@ class ClonePermissionTestCase(TestCase):
             "disk-vda": "clone-vm.qcow2",
             "clone-title": "",
             "clone-description": "",
+            "submission_key": "key-clone-perm1",
         }
         data.update(extra)
         return data
 
     def test_viewer_cannot_clone_someone_elses_vm(self):
-        with self.mocked_libvirt() as proxy:
+        with self.mocked_libvirt():
             res = self._clone(self.viewer, self._valid_post())
         self.assertEqual(res.status_code, 403)
-        proxy.clone_instance.assert_not_called()
+        self.assertFalse(Operation.objects.exists())
 
     def test_owner_without_is_change_cannot_clone(self):
-        with self.mocked_libvirt() as proxy:
+        with self.mocked_libvirt():
             res = self._clone(self.owner_readonly, self._valid_post())
         self.assertEqual(res.status_code, 403)
-        proxy.clone_instance.assert_not_called()
+        self.assertFalse(Operation.objects.exists())
 
     def test_a_clone_stays_on_the_host_of_its_vm(self):
         # the removed "VM Clone Auto Migrate" setting sent it to a random host
         other = Compute.objects.create(name="other-host", hostname="198.51.100.20", login="", password="", type=4)
-        with self.mocked_libvirt() as proxy, \
+        with self.mocked_libvirt(), \
              patch("instances.views.utils.migrate_instance") as migrate, \
              patch.object(app_settings, "CLONE_INSTANCE_AUTO_MIGRATE", "True", create=True), \
              patch("instances.views.Compute.objects.order_by") as random_host:
             random_host.return_value.first.return_value = other
             self._clone(self.owner_change, self._valid_post())
-        proxy.clone_instance.assert_called_once()
         migrate.assert_not_called()
-        self.assertEqual(Instance.objects.get(name="clone-vm").compute, self.instance.compute)
+        self.assertEqual(Operation.objects.get().intent["compute"], self.instance.compute_id)
 
     def test_viewer_can_clone_a_template(self):
         self.instance.is_template = True
         self.instance.save(update_fields=["is_template"])
-        with self.mocked_libvirt() as proxy:
+        with self.mocked_libvirt():
             self._clone(self.viewer, self._valid_post())
-        proxy.clone_instance.assert_called_once()
+        self.assertEqual(Operation.objects.count(), 1)
 
     def test_posted_disk_owner_and_unknown_keys_are_ignored(self):
-        with self.mocked_libvirt() as proxy:
+        with self.mocked_libvirt():
             self._clone(
                 self.owner_change,
                 self._valid_post(disk_owner_uid="4242", disk_owner_gid="4242", evil="x"),
             )
-        proxy.clone_instance.assert_called_once()
-        clone_data = proxy.clone_instance.call_args[0][0]
+        clone_data = Operation.objects.get().intent["clone"]
         self.assertEqual(
             clone_data["disk_owner_uid"],
             int(app_settings.INSTANCE_VOLUME_DEFAULT_OWNER_UID),
@@ -146,53 +146,37 @@ class ClonePermissionTestCase(TestCase):
         self.assertNotIn("evil", clone_data)
 
     def test_non_superuser_disk_names_are_derived_by_the_server(self):
-        with self.mocked_libvirt() as proxy:
+        with self.mocked_libvirt():
             self._clone(
                 self.owner_change,
                 self._valid_post(**{"disk-vda": "../../../etc/passwd"}),
             )
-        proxy.clone_instance.assert_called_once()
-        clone_data = proxy.clone_instance.call_args[0][0]
+        clone_data = Operation.objects.get().intent["clone"]
         self.assertEqual(clone_data["disk-vda"], "clone-vm.qcow2")
 
     def test_superuser_unsafe_disk_name_is_rejected(self):
-        with self.mocked_libvirt() as proxy:
+        with self.mocked_libvirt():
             self._clone(
                 self.superuser,
                 self._valid_post(**{"disk-vda": "x</name><evil/>"}),
             )
-        proxy.clone_instance.assert_not_called()
+        self.assertFalse(Operation.objects.exists())
 
     def test_superuser_invalid_second_mac_is_rejected(self):
-        with self.mocked_libvirt() as proxy:
+        with self.mocked_libvirt():
             self._clone(
                 self.superuser,
                 self._valid_post(**{"clone-net-mac-1": "not-a-mac"}),
             )
-        proxy.clone_instance.assert_not_called()
-
-    def test_clone_holds_the_compute_exclusively(self):
-        # Clones of different VMs must not race for the destination name.
-        calls = []
-
-        @contextmanager
-        def record(compute, *, shared=False):
-            calls.append(shared)
-            yield
-
-        with self.mocked_libvirt() as proxy, patch("instances.views.libvirt_compute_lock", record):
-            self._clone(self.superuser, self._valid_post())
-        proxy.clone_instance.assert_called_once()
-        self.assertEqual(calls, [False])
+        self.assertFalse(Operation.objects.exists())
 
     def test_superuser_valid_clone_passes_posted_values(self):
-        with self.mocked_libvirt() as proxy:
+        with self.mocked_libvirt():
             self._clone(
                 self.superuser,
                 self._valid_post(**{"disk-vda": "custom-name.qcow2", "meta-vda": "true"}),
             )
-        proxy.clone_instance.assert_called_once()
-        clone_data = proxy.clone_instance.call_args[0][0]
+        clone_data = Operation.objects.get().intent["clone"]
         self.assertEqual(clone_data["disk-vda"], "custom-name.qcow2")
         self.assertEqual(clone_data["meta-vda"], "true")
         self.assertEqual(clone_data["clone-net-mac-0"], "52:54:00:12:34:56")
@@ -203,8 +187,7 @@ class ClonePermissionTestCase(TestCase):
         with self.mocked_libvirt() as proxy:
             proxy.get_disk_devices.return_value = [SRC_DISK, empty_disk]
             self._clone(self.owner_change, self._valid_post())
-        proxy.clone_instance.assert_called_once()
-        clone_data = proxy.clone_instance.call_args[0][0]
+        clone_data = Operation.objects.get().intent["clone"]
         self.assertEqual(clone_data["disk-vda"], "clone-vm.qcow2")
         self.assertIsNone(clone_data["disk-vdb"])
 
@@ -213,9 +196,9 @@ class ClonePermissionTestCase(TestCase):
         with self.mocked_libvirt() as proxy:
             proxy.get_disk_devices.return_value = [odd_disk]
             self._clone(self.owner_change, self._valid_post())
-        proxy.clone_instance.assert_called_once()
+        self.assertEqual(Operation.objects.count(), 1)
         self.assertEqual(
-            proxy.clone_instance.call_args[0][0]["disk-vda"], "my-disk-2-clone.qcow2"
+            Operation.objects.get().intent["clone"]["disk-vda"], "my-disk-2-clone.qcow2"
         )
 
 

instances/tests/test_live_clone.py

@@ -68,7 +68,7 @@ class LiveCloneTestCase(SimpleTestCase):
         before = self.pool_volumes()
 
         with self.assertRaises(Exception):
-            vm.clone_instance(self.clone_data("clone-dst"))
+            vm.clone_instance(self.clone_data("clone-dst"), "abababab-0000-1111-2222-333333333333")
 
         self.assertEqual(self.pool_volumes(), before, "copies left behind or a volume deleted")
         self.assertTrue(livetest.volume_exists(self.conn, taken))
@@ -81,7 +81,7 @@ class LiveCloneTestCase(SimpleTestCase):
         before = self.pool_volumes()
 
         with self.assertRaises(Exception):
-            vm.clone_instance(self.clone_data("clone-run-copy"))
+            vm.clone_instance(self.clone_data("clone-run-copy"), "abababab-0000-1111-2222-333333333333")
 
         self.assertEqual(self.pool_volumes(), before)
 
@@ -94,7 +94,7 @@ class LiveCloneTestCase(SimpleTestCase):
         source = vm.get_nvram()
 
         uuid = vm.clone_instance(
-            {"name": P + "clone-uefi-copy", "disk-vda": P + "clone-uefi-copy-a.qcow2", "disk_owner_uid": 0, "disk_owner_gid": 0}
+            {"name": P + "clone-uefi-copy", "disk-vda": P + "clone-uefi-copy-a.qcow2", "disk_owner_uid": 0, "disk_owner_gid": 0}, "abababab-0000-1111-2222-333333333333"
         )
 
         clone = self.conn.lookupByUUIDString(uuid)
@@ -111,7 +111,7 @@ class LiveCloneTestCase(SimpleTestCase):
 
     def test_clone_of_a_stopped_vm(self):
         _, vm = self.source("clone-ok")
-        uuid = vm.clone_instance(self.clone_data("clone-ok-copy"))
+        uuid = vm.clone_instance(self.clone_data("clone-ok-copy"), "abababab-0000-1111-2222-333333333333")
         clone = self.conn.lookupByUUIDString(uuid)
         self.assertEqual(clone.name(), P + "clone-ok-copy")
         self.assertTrue({P + "clone-ok-copy-a.qcow2", P + "clone-ok-copy-b.qcow2"} <= self.pool_volumes())

instances/tests/test_moved_while_waiting.py

@@ -69,74 +69,3 @@ class MovedWhileWaitingTestCase(TestCase):
         self.assertTrue(Instance.objects.filter(pk=self.vm.pk).exists())
         messages = [str(m) for m in response.wsgi_request._messages]
         self.assertIn("The VM was moved meanwhile; try again.", messages)
-
-    def clone(self, user, change):
-        @contextmanager
-        def compute_lock_meanwhile(compute, *, shared=False):
-            change()
-            yield
-
-        self.client.force_login(user)
-        with patch("instances.views.libvirt_compute_lock", compute_lock_meanwhile), patch(
-            "instances.models.wvmInstance"
-        ) as wvm, patch.object(Compute, "status", new_callable=PropertyMock, return_value=True), patch(
-            "instances.views.utils.check_user_quota", return_value=""  # not what this tests
-        ):
-            wvm.return_value.get_disk_devices.return_value = []
-            wvm.return_value.get_vcpu.return_value = 1
-            wvm.return_value.get_memory.return_value = 512
-            response = self.client.post(reverse("instances:clone", args=[self.vm.id]), {"name": "moved-clone"})
-        return response, wvm.return_value.clone_instance.called
-
-    def test_a_clone_of_a_vm_moved_while_it_waited_is_not_made(self):
-        from accounts.models import UserInstance
-        from django.contrib.auth.models import Permission
-
-        # an owner who may clone it (a superuser would also choose MACs)
-        owner = get_user_model().objects.create_user("clone_owner", password="x")
-        owner.user_permissions.add(Permission.objects.get(codename="clone_instances"))
-        UserInstance.objects.create(instance=self.vm, user=owner, is_change=True)
-        response, cloned = self.clone(
-            owner, lambda: Instance.objects.filter(pk=self.vm.pk).update(compute=self.target)
-        )
-        self.assertFalse(cloned)
-        self.assertIn("The VM was moved meanwhile; try again.", [str(m) for m in response.wsgi_request._messages])
-
-    def test_a_template_unmarked_while_the_request_waited_is_not_cloned_by_a_viewer(self):
-        from django.contrib.auth.models import Permission
-
-        viewer = get_user_model().objects.create_user("clone_viewer", password="x")
-        for codename in ("view_instances", "clone_instances"):
-            viewer.user_permissions.add(Permission.objects.get(codename=codename))
-        Instance.objects.filter(pk=self.vm.pk).update(is_template=True)
-        response, cloned = self.clone(viewer, lambda: Instance.objects.filter(pk=self.vm.pk).update(is_template=False))
-        self.assertFalse(cloned)
-        self.assertEqual(response.status_code, 403)
-
-    def test_a_clone_checks_the_quota_against_the_source_as_it_is_under_the_lock(self):
-        from accounts.models import UserInstance
-        from django.contrib.auth.models import Permission
-
-        owner = get_user_model().objects.create_user("clone_quota", password="x")
-        owner.user_permissions.add(Permission.objects.get(codename="clone_instances"))
-        UserInstance.objects.create(instance=self.vm, user=owner, is_change=True)
-        self.client.force_login(owner)
-        with patch("instances.models.wvmInstance") as wvm:
-            wvm.return_value.get_disk_devices.return_value = []
-            wvm.return_value.get_vcpu.return_value = 1
-            wvm.return_value.get_memory.return_value = 512
-
-            @contextmanager
-            def compute_lock_after_a_resize(compute, *, shared=False):
-                wvm.return_value.get_vcpu.return_value = 8  # resized meanwhile
-                yield
-
-            with patch("instances.views.libvirt_compute_lock", compute_lock_after_a_resize), patch.object(
-                Compute, "status", new_callable=PropertyMock, return_value=True
-            ), patch(
-                "instances.views.utils.check_user_quota",
-                side_effect=lambda user, count, cpu, memory, disk: "cpu" if cpu > 4 else "",
-            ):
-                response = self.client.post(reverse("instances:clone", args=[self.vm.id]), {"name": "grown-clone"})
-        self.assertFalse(wvm.return_value.clone_instance.called)
-        self.assertTrue(any("quota" in str(m) for m in response.wsgi_request._messages))