fix(instances): a quiesced snapshot or live clone stops early on a silent guest agent

A quiesced snapshot and a live clone freeze the guest's file systems
through libvirt, which asks the agent without a timeout: with the agent
hung, the job waited, holding the VM, with no end.

The agent must now first answer a guest-ping within 10 s; a frozen or
dead agent fails the job in seconds, before anything is frozen. The
freeze itself stays libvirt's, without a timeout: a timed freeze was
tried and left the guest frozen, since the freeze still ran in the
guest when the agent came back, after its thaw had failed.

vrtManager/instance.py

@@ -1668,6 +1668,7 @@ class wvmInstance(wvmConnect):
         if not memory:
             flags |= VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY
         if quiesce:
+            self.ping_agent()
             flags |= VIR_DOMAIN_SNAPSHOT_CREATE_QUIESCE
         self._snapshotCreateXML(xml, flags)
         self._refresh_pools_after_snapshot()
@@ -2259,7 +2260,15 @@ class wvmInstance(wvmConnect):
     def abort_backup(self):
         self.instance.abortJob()
 
+    def ping_agent(self):
+        """A frozen or dead agent fails here in AGENT_TIMEOUT seconds. The
+        freeze that follows is libvirt's, without a timeout: a timed freeze
+        could still end in the guest after its thaw failed, leaving the guest
+        frozen."""
+        agent_command(self.instance, "guest-ping")
+
     def fs_freeze(self):
+        self.ping_agent()
         self.instance.fsFreeze(None, 0)
 
     def fs_thaw(self):

vrtManager/tests/test_external_snapshots.py

@@ -14,6 +14,7 @@ if not settings.configured:
 
 import libvirt
 from libvirt import libvirtError
+from vrtManager import instance as instance_module
 from vrtManager import util
 from vrtManager.instance import PERSISTENT_XML, wvmInstance
 
@@ -274,10 +275,23 @@ class CreateTestCase(unittest.TestCase):
     def test_a_quiesced_snapshot_freezes_the_guest_file_systems(self):
         p = proxy()
         p.is_agent_ready = lambda: True
-        p.create_external_snapshot("q", quiesce=True)
+        with patch.object(instance_module, "qemuAgentCommand", return_value='{"return": {}}') as agent:
+            p.create_external_snapshot("q", quiesce=True)
+        self.assertEqual(agent.call_args.args[1:], ('{"execute": "guest-ping"}', 10, 0))
         _, flags = created(p)
         self.assertEqual(flags, DISK_ONLY_ATOMIC | libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_QUIESCE)
 
+    def test_a_quiesced_snapshot_needs_an_agent_that_answers(self):
+        # libvirt's quiesce asks the agent without a timeout: a hung agent is
+        # found by a timed ping first
+        p = proxy()
+        p.is_agent_ready = lambda: True
+        with patch.object(instance_module, "qemuAgentCommand",
+                          side_effect=libvirtError("guest agent didn't respond to command within '10' seconds")):
+            with self.assertRaises(libvirtError):
+                p.create_external_snapshot("q", quiesce=True)
+        p._snapshotCreateXML.assert_not_called()
+
     def test_quiesce_needs_a_running_vm_with_its_agent_and_no_memory(self):
         # libvirt accepts quiesce only for disk-only snapshots, through the agent
         for status, ready, memory, pattern in (

vrtManager/tests/test_live_clone.py

@@ -2,11 +2,12 @@
 copied disks into new volumes, and the clone definition both paths share."""
 
 import unittest
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 import libvirt
 from lxml import etree
 
+from vrtManager import instance as instance_module
 from vrtManager import util
 from vrtManager.instance import LIVE_CLONE_MIN_LIBVIRT, VIR_DOMAIN_BACKUP_BEGIN_PRESERVE_SHUTDOWN_DOMAIN, wvmInstance
 
@@ -199,13 +200,26 @@ class BackupTestCase(unittest.TestCase):
 
     def test_freeze_thaw_abort(self):
         p = proxy()
-        p.fs_freeze()
+        with patch.object(instance_module, "qemuAgentCommand", return_value='{"return": {}}') as agent:
+            p.fs_freeze()
+        # a timed ping first, then libvirt's own freeze
+        self.assertEqual(agent.call_args.args[1:], ('{"execute": "guest-ping"}', 10, 0))
         p.instance.fsFreeze.assert_called_once_with(None, 0)
         p.fs_thaw()
         p.instance.fsThaw.assert_called_once_with(None, 0)
         p.abort_backup()
         p.instance.abortJob.assert_called_once_with()
 
+    def test_an_agent_that_does_not_answer_is_not_asked_to_freeze(self):
+        # libvirt's freeze waits without a timeout; a timed freeze could end in
+        # the guest after its thaw had already failed, leaving it frozen
+        p = proxy()
+        with patch.object(instance_module, "qemuAgentCommand",
+                          side_effect=libvirt.libvirtError("guest agent didn't respond to command within '10' seconds")):
+            with self.assertRaises(libvirt.libvirtError):
+                p.fs_freeze()
+        p.instance.fsFreeze.assert_not_called()
+
 
 class CreateCloneTargetTestCase(unittest.TestCase):
     def test_an_empty_volume_at_the_source_capacity(self):