feat(instances): a running VM gets its root password and SSH key through its guest agent

The Access tab's Root Password and SSH Keys worked only on a shut-off
VM, through gstfsd on the compute. A running VM whose QEMU guest agent
is connected now gets them through the agent: guest-set-user-password
with the SHA-512 crypt hash, and guest-ssh-add-authorized-keys adding
the key to root's keys. A shut-off VM still goes through gstfsd; any
other VM is refused with "Shut the VM down, or start its QEMU guest
agent, and try again", and the buttons are disabled.

Each is one agent command with a 10 s timeout. libvirt's own calls for
them (setUserPassword, authorizedSSHKeysSet) wait for a hung agent
indefinitely, holding a worker and the VM's lock, and the domain's
agent timeout would change every other agent call of the VM too.

The agent's or libvirt's error is shown as is through error_text, so
users who do not see the hosts get no connection details, and the
attempt is logged as failed. The README names the SELinux boolean a
RHEL-family guest needs before its agent may add SSH keys.

README.md

@@ -269,6 +269,8 @@ A compute node is a KVM host the panel manages over libvirt.
 
    It listens on 127.0.0.1:16510 only, which serves a panel on the same host. For a remote panel, run `sudo systemctl edit webvirt-gstfsd`, add `[Service]` and `Environment=GSTFSD_BIND_HOST=0.0.0.0 GSTFSD_ALLOW_REMOTE=1`, and restart it. It is unauthenticated: allow TCP 16510 only from the panel's IP.
 
+   A running VM gets them through its QEMU guest agent instead, without gstfsd. On RHEL-family guests, SELinux lets the agent add SSH keys only after `sudo setsebool -P virt_qemu_ga_manage_ssh on` inside the guest.
+
    A compute that ran gstfsd under supervisor: remove its program first (`/etc/supervisor/conf.d/gstfsd.conf` on Debian / Ubuntu, `/etc/supervisord.d/gstfsd.ini` on RHEL / openSUSE), run `sudo supervisorctl update`, and remove supervisor once nothing else uses it.
 
 4. Add the compute on the panel's Computes page.

instances/templates/instances/access_tab.html

@@ -88,7 +88,7 @@
         </div>
         {% if app_settings.SHOW_ACCESS_ROOT_PASSWORD == 'True' and can_change %}
         <div role="tabpanel" class="tab-pane tab-pane-bordered" id="rootpasswd">
-            <p>{% trans "You need shut down your instance and enter a new root password." %}</p>
+            <p>{% trans "Shut the VM down, or run it with its QEMU guest agent, and enter a new root password." %}</p>
             <form action="{% url 'instances:rootpasswd' instance.id %}" method="post"
                 role="form" aria-label="Add root password to instance form">
                 {% csrf_token %}
@@ -98,7 +98,7 @@
                             placeholder="{% trans "Enter Password" %}" maxlength="24">
                     </div>
                 </div>
-                {% if instance.status == 5 %}
+                {% if instance.status == 5 or instance.status == 1 and instance.guest_agent_ready %}
                 <input type="submit" class="btn btn-lg btn-success float-end" name="rootpasswd"
                     value="{% trans "Reset Root Password" %}">
                 {% else %}
@@ -111,7 +111,7 @@
         {% endif %}
         {% if app_settings.SHOW_ACCESS_SSH_KEYS == 'True' and can_change %}
         <div role="tabpanel" class="tab-pane tab-pane-bordered" id="sshkeys">
-            <p>{% trans "You need shut down your instance and choose your public key." %}</p>
+            <p>{% trans "Shut the VM down, or run it with its QEMU guest agent, and choose your public key. The key is added to root's keys." %}</p>
             <form action="{% url 'instances:add_public_key' instance.id %}"method="post"
                 role="form" aria-label="Add public key to instance form">
                 {% csrf_token %}
@@ -128,7 +128,7 @@
                         </select>
                     </div>
                 </div>
-                {% if instance.status == 5 %}
+                {% if instance.status == 5 or instance.status == 1 and instance.guest_agent_ready %}
                 <input type="submit" class="btn btn-lg btn-success float-end" name="addpublickey"
                     value="{% trans "Add Public Key" %}">
                 {% else %}

instances/tests/test_root_pass.py

@@ -2,11 +2,13 @@ import json
 import socket
 from unittest.mock import MagicMock, patch
 
+from django.contrib.messages import get_messages
 from django.test import RequestFactory, TestCase
 from django.urls import reverse
 
 from computes.models import Compute
 from instances.models import Instance
+from libvirt import libvirtError
 
 
 class SetRootPassTestCase(TestCase):
@@ -115,3 +117,69 @@ class SetRootPassTestCase(TestCase):
 
         self.assertEqual(response.status_code, 302)
         mock_sock.settimeout.assert_called()
+
+
+class GuestAgentAccessTestCase(TestCase):
+    """A running VM whose guest agent answers gets its root password and SSH
+    key through the agent; gstfsd serves only a shut-off VM."""
+
+    def setUp(self):
+        from accounts.models import UserSSHKey
+        from django.contrib.auth import get_user_model
+
+        self.client.login(request=RequestFactory().get("/"), username="admin", password="admin")
+        compute = Compute.objects.create(name="agent-compute", hostname="192.0.2.1", type=1)
+        self.instance = Instance.objects.create(compute=compute, name="agent-vm",
+                                                uuid="11111111-2222-3333-4444-555555555555")
+        admin = get_user_model().objects.get(username="admin")
+        self.key = UserSSHKey.objects.create(user=admin, keyname="laptop", keypublic="ssh-ed25519 AAAA laptop")
+        patcher = patch.object(Instance, "proxy")
+        self.proxy = patcher.start()
+        self.addCleanup(patcher.stop)
+        self.proxy.get_status.return_value = 1  # running
+        self.proxy.is_agent_ready.return_value = True
+
+    def post(self, name, data):
+        with patch("instances.views.gstfsd_request") as gstfsd:
+            response = self.client.post(reverse(f"instances:{name}", args=[self.instance.id]), data)
+        gstfsd.assert_not_called()
+        return [str(m) for m in get_messages(response.wsgi_request)]
+
+    def test_the_agent_sets_the_root_password(self):
+        self.assertEqual(self.post("rootpasswd", {"passwd": "S3cret-pass"}), ["Reset root password"])
+        user, crypted = self.proxy.set_user_password.call_args.args
+        self.assertEqual(user, "root")
+        self.assertTrue(crypted.startswith("$6$"))
+
+    def test_the_agent_adds_the_ssh_key(self):
+        self.assertEqual(self.post("add_public_key", {"sshkeyid": self.key.id}),
+                         ["Installed new SSH public key laptop"])
+        self.proxy.add_authorized_key.assert_called_once_with("root", "ssh-ed25519 AAAA laptop")
+
+    def test_a_running_vm_without_its_agent_is_refused(self):
+        self.proxy.is_agent_ready.return_value = False
+        for name, data in (("rootpasswd", {"passwd": "S3cret-pass"}), ("add_public_key", {"sshkeyid": self.key.id})):
+            # the latest message: the redirect is not followed, so earlier ones stay
+            self.assertEqual(self.post(name, data)[-1], "Shut the VM down, or start its QEMU guest agent, and try again")
+        self.proxy.set_user_password.assert_not_called()
+        self.proxy.add_authorized_key.assert_not_called()
+
+    def test_an_agent_that_refuses_is_an_error_message(self):
+        self.proxy.set_user_password.side_effect = libvirtError("guest agent command failed: user root not found")
+        self.assertEqual(self.post("rootpasswd", {"passwd": "S3cret-pass"}),
+                         ["guest agent command failed: user root not found"])
+
+    def test_a_user_who_does_not_see_hosts_gets_no_connection_details(self):
+        from accounts.models import UserInstance
+        from django.contrib.auth import get_user_model
+        from libvirt import VIR_ERR_SYSTEM_ERROR, VIR_FROM_RPC
+
+        owner = get_user_model().objects.create_user("agent-owner", password="pw")
+        UserInstance.objects.create(user=owner, instance=self.instance, is_change=True)
+        self.client.force_login(owner)
+        err = libvirtError("Cannot recv data: 192.0.2.1:16509 Connection reset by peer")
+        err.get_error_domain = lambda: VIR_FROM_RPC
+        err.get_error_code = lambda: VIR_ERR_SYSTEM_ERROR
+        self.proxy.set_user_password.side_effect = err
+        self.assertEqual(self.post("rootpasswd", {"passwd": "S3cret-pass"}),
+                         ["The host of this virtual machine cannot be reached right now."])

instances/tests/test_vm_audit_rows.py

@@ -77,6 +77,15 @@ class VMRowTests(TestCase):
         self.assertEqual(self.rows(), [("VM vm: reset root password: gstfsd error: timed out", False),
                                        ("VM vm: reset root password", True)])
 
+    def test_an_agent_error_is_a_failed_row_without_the_password(self):
+        self.proxy.get_status.return_value = 1
+        self.proxy.is_agent_ready.return_value = True
+        self.proxy.set_user_password.side_effect = libvirtError("guest agent didn't respond to command within '10' seconds")
+        self.post("rootpasswd", {"passwd": "S3cret-pass"})
+        self.assertEqual(self.rows(), [
+            ("VM vm: reset root password: guest agent didn't respond to command within '10' seconds", False)])
+        self.assertFalse(Logs.objects.filter(message__contains="$6$").exists())
+
     def test_console_password_rows_never_hold_the_password(self):
         self.proxy.get_console_type.return_value = "vnc"
         self.proxy.console_listens_locally.return_value = True

instances/views.py

@@ -863,6 +863,11 @@ def gstfsd_request(hostname, data):
     return reply
 
 
+# root password and SSH key: gstfsd changes a shut-off VM's disk, the QEMU
+# guest agent a running VM
+AGENT_OR_SHUT_OFF = _("Shut the VM down, or start its QEMU guest agent, and try again")
+
+
 @require_POST
 @serialize_instance_mutation
 def set_root_pass(request, pk):
@@ -875,17 +880,20 @@ def set_root_pass(request, pk):
             # gstfsd opens the domain by name: send the host's current one.
             data = {"action": "password", "passwd": passwd_hash, "vname": instance.proxy.instance.name()}
 
-            if instance.proxy.get_status() == 5:
+            status = instance.proxy.get_status()
+            if status == 5 or (status == 1 and instance.guest_agent_ready):
                 try:
                     with audit_vm(request, instance, "reset root password"):
-                        gstfsd_request(instance.compute.hostname, data)
-                except util.OperationError as err:
-                    messages.error(request, err)
+                        if status == 5:
+                            gstfsd_request(instance.compute.hostname, data)
+                        else:
+                            instance.proxy.set_user_password("root", passwd_hash)
+                except (util.OperationError, libvirtError) as err:
+                    messages.error(request, error_text(request, err))
                 else:
                     messages.success(request, _("Reset root password"))
             else:
-                msg = _("Please shutdown down your instance and then try again")
-                messages.error(request, msg)
+                messages.error(request, AGENT_OR_SHUT_OFF)
     return get_safe_redirect(
         request, default=reverse("instances:instance", args=[instance.id]) + "#access"
     )
@@ -910,19 +918,22 @@ def add_public_key(request, pk):
             "vname": instance.proxy.instance.name(),
         }
 
-        if instance.proxy.get_status() == 5:
+        status = instance.proxy.get_status()
+        if status == 5 or (status == 1 and instance.guest_agent_ready):
             try:
                 with audit_vm(request, instance, f"install SSH key {publickey.keyname}"):
-                    gstfsd_request(instance.compute.hostname, data)
-            except util.OperationError as err:
-                messages.error(request, err)
+                    if status == 5:
+                        gstfsd_request(instance.compute.hostname, data)
+                    else:
+                        instance.proxy.add_authorized_key("root", publickey.keypublic)
+            except (util.OperationError, libvirtError) as err:
+                messages.error(request, error_text(request, err))
             else:
                 messages.success(request, _("Installed new SSH public key %(keyname)s") % {
                     "keyname": publickey.keyname
                 })
         else:
-            msg = _("Please shutdown down your instance and then try again")
-            messages.error(request, msg)
+            messages.error(request, AGENT_OR_SHUT_OFF)
     return get_safe_redirect(
         request, default=reverse("instances:instance", args=[instance.id]) + "#access"
     )

vrtManager/instance.py

@@ -1,3 +1,4 @@
+import base64
 import contextlib
 import io
 import ipaddress
@@ -73,6 +74,8 @@ PERSISTENT_XML = VIR_DOMAIN_XML_INACTIVE | VIR_DOMAIN_XML_SECURE
 SNAPSHOT_NAME = re.compile(r"[A-Za-z0-9_-][A-Za-z0-9._-]{0,63}")
 # the longest window wvmInstance.usage() averages over
 USAGE_MAX_WINDOW = 30
+# seconds the guest agent has to set a password or add a key
+AGENT_TIMEOUT = 10
 # A running VM is cloned with a push backup that keeps the VM alive when the
 # guest shuts down during the copy: libvirt 11.10.
 LIVE_CLONE_MIN_LIBVIRT = 11_010_000
@@ -2522,6 +2525,24 @@ class wvmInstance(wvmConnect):
 
         return util.get_xml_path(self._XMLDesc(0), func=_get_agent)
 
+    def _agent_command(self, command, arguments):
+        """One guest agent command that waits AGENT_TIMEOUT seconds at most:
+        libvirt's own calls for it wait for a hung agent indefinitely, and
+        the domain's agent timeout is shared by every agent call."""
+        request = json.dumps({"execute": command, "arguments": arguments})
+        qemuAgentCommand(self.instance, request, AGENT_TIMEOUT, 0)
+
+    def set_user_password(self, user, crypted):
+        """Set a user's password in the running guest through its agent;
+        crypted is a crypt(3) hash."""
+        password = base64.b64encode(crypted.encode()).decode()
+        self._agent_command("guest-set-user-password", {"username": user, "password": password, "crypted": True})
+
+    def add_authorized_key(self, user, key):
+        """Add an SSH public key to the user's authorized keys in the running
+        guest through its agent."""
+        self._agent_command("guest-ssh-add-authorized-keys", {"username": user, "keys": [key], "reset": False})
+
     def is_agent_ready(self):
         """
         Return connected state of an agent.

vrtManager/tests/test_agent_access.py

@@ -0,0 +1,49 @@
+"""A running guest's root password and SSH keys through its QEMU guest
+agent: the password as a crypt(3) hash, the key added to those it has. Each
+is one agent command with a timeout: libvirt's own calls for them would wait
+for a hung agent indefinitely, and setting the domain's agent timeout would
+change it for every other agent call too."""
+
+import base64
+import json
+import unittest
+from unittest.mock import MagicMock, patch
+
+from django.conf import settings
+
+if not settings.configured:
+    settings.configure(MAC_OUI="52:54:10")
+
+from libvirt import libvirtError
+from vrtManager import instance as module
+from vrtManager.instance import wvmInstance
+
+
+class AgentAccessTestCase(unittest.TestCase):
+    def setUp(self):
+        self.vm = wvmInstance.__new__(wvmInstance)  # no connection: only these calls
+        self.vm.instance = MagicMock()
+        patcher = patch.object(module, "qemuAgentCommand", return_value='{"return":{}}')
+        self.agent = patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def sent(self):
+        (dom, command, timeout, flags), = [c.args for c in self.agent.call_args_list]
+        self.assertIs(dom, self.vm.instance)
+        self.assertEqual((timeout, flags), (10, 0))
+        return json.loads(command)
+
+    def test_the_password_goes_as_a_hash(self):
+        self.vm.set_user_password("root", "$6$salt$hash")
+        self.assertEqual(self.sent(), {"execute": "guest-set-user-password", "arguments": {
+            "username": "root", "password": base64.b64encode(b"$6$salt$hash").decode(), "crypted": True}})
+
+    def test_the_key_is_added_to_the_users_keys(self):
+        self.vm.add_authorized_key("root", "ssh-ed25519 AAAA laptop")
+        self.assertEqual(self.sent(), {"execute": "guest-ssh-add-authorized-keys", "arguments": {
+            "username": "root", "keys": ["ssh-ed25519 AAAA laptop"], "reset": False}})
+
+    def test_the_agents_error_is_passed_on(self):
+        self.agent.side_effect = libvirtError("guest agent didn't respond to command within '10' seconds")
+        with self.assertRaisesRegex(libvirtError, "within '10' seconds"):
+            self.vm.set_user_password("root", "$6$salt$hash")