feat(instances): a new VM can get a cloud-init seed with its hostname and SSH keys

Cloud images read their first-boot settings with cloud-init. The panel
served them only over HTTP, which a VM reaches only with the network
and a reverse DNS name for its address. A new VM can now carry them
itself: Cloud-init seed (Advanced) on the create form, or cloud_init in
the API.

The job writes a NoCloud seed, an ISO labelled cidata, into the disk's
directory pool as cidata-<uuid>.iso and puts it in the VM's CD-ROM
drive. Its meta-data holds the VM's UUID as instance-id and its name as
hostname, written as JSON so a VM named 123 or yes keeps a string
hostname; its user-data is what the HTTP datasource serves, with the
creating user's SSH keys. The ISO is built with pycdlib and written
through a libvirt stream, as the NVRAM copy is, now one helper for both.

A seed needs a CD-ROM bus and a directory pool, else the create is
refused. A failed create deletes the seed with the new disk, recovery
names one left behind, and deleting the VM with its disks deletes its
seed, found in the pools by name even when ejected. A clone's drive is
left empty: the seed is the source VM's identity.

instances/api/serializers.py

@@ -83,6 +83,7 @@ class CreateInstanceSerializer(serializers.ModelSerializer):
             "meta_prealloc",
             "virtio",
             "qemu_ga",
+            "cloud_init",
             "video",
             "listener_addr",
             "submission_key",

instances/jobs.py

@@ -5,6 +5,7 @@ import re
 import time
 
 from django.core.exceptions import PermissionDenied
+from django.template.loader import render_to_string
 from django.utils.translation import gettext_noop
 from django.utils.translation import gettext_noop as _
 from libvirt import (VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE, VIR_DOMAIN_JOB_NONE, VIR_DOMAIN_SNAPSHOT_LIST_EXTERNAL,
@@ -14,7 +15,7 @@ from vrtManager.connection import wvmConnect
 from vrtManager.create import wvmCreate
 from vrtManager.instance import wvmInstance
 
-from accounts.models import UserInstance
+from accounts.models import UserInstance, UserSSHKey
 from appsettings.settings import app_settings
 from computes.models import Compute
 from instances import utils
@@ -613,6 +614,17 @@ class CloneInstance(Kind):
         return _log(op, state, f"clone from {source}", vm=op.intent["name"])
 
 
+# pools whose volumes are files a CD-ROM can read
+SEED_POOL_TYPES = ("dir", "fs", "netfs")
+
+
+def _seed_pool(conn, data):
+    """The pool of a new VM's cloud-init seed: the one of its disk."""
+    if data["storage"]:
+        return conn.get_storage(data["storage"])
+    return conn.get_storage_by_vol_path(conn.get_volume_path(data["images"].split(",")[0]))
+
+
 @register
 class CreateInstance(Kind):
     name = "instance.create"
@@ -622,7 +634,7 @@ class CreateInstance(Kind):
     def normalize(self, raw):
         # what the API leaves out means what create_instance did by default
         params = {"template": "", "images": "", "firmware": "", "net_model": "virtio", "add_cdrom": "sata",
-                  "add_input": "default", "meta_prealloc": False, "hdd_size": 0}
+                  "add_input": "default", "meta_prealloc": False, "hdd_size": 0, "cloud_init": False}
         params.update({k: v for k, v in raw.items() if k not in ("submission_key", "create", "csrfmiddlewaretoken")})
         params.update(compute=int(raw["compute"]), devices=[list(d) for d in raw.get("devices", [])])
         if "UEFI" in params["firmware"] and "secboot" in params["firmware"].partition(":")[2]:
@@ -653,6 +665,12 @@ class CreateInstance(Kind):
             raise util.OperationError(_("Invalid cache mode"))
         if data["add_cdrom"] != "None" and data["add_cdrom"] not in cdrom_buses:
             raise util.OperationError(_("A CD-ROM cannot use the %(bus)s bus") % {"bus": data["add_cdrom"]})
+        if data["cloud_init"]:
+            if data["add_cdrom"] == "None":
+                raise util.OperationError(_("The cloud-init seed is read from a CD-ROM: choose its bus"))
+            pool = _seed_pool(conn, data)
+            if util.get_xml_path(pool.XMLDesc(0), "/pool/@type") not in SEED_POOL_TYPES:
+                raise util.OperationError(_("The cloud-init seed needs a directory storage pool for the disk"))
         try:
             utils.nic_macs(data["mac"], data["networks"])
         except ValueError as err:
@@ -715,6 +733,14 @@ class CreateInstance(Kind):
                 for vol, (device, bus) in zip(data["images"].split(","), data["devices"]):
                     volume_list.append(volume(conn.get_volume_path(vol), device, bus))
 
+            if data["cloud_init"]:
+                ctx.set_phase(gettext_noop("Writing the cloud-init seed"))
+                keys = UserSSHKey.objects.filter(user=op.user).values_list("keypublic", flat=True)
+                user_data = render_to_string("user_data", {"instance_keys": list(keys)})
+                seed = conn.create_seed(_seed_pool(conn, data).name(), op.intent["uuid"], data["name"], user_data)
+                created.append(seed)
+                volume_list.append(volume(seed, "cdrom", data["add_cdrom"]))
+
             ctx.set_phase(gettext_noop("Defining the VM"))
             conn.create_instance(
                 name=data["name"],
@@ -754,8 +780,10 @@ class CreateInstance(Kind):
 
     def reconcile(self, op):
         data = op.params
-        names = [data["name"] + ext for ext in ("", ".img", ".qcow", ".qcow2")]
-        return vm_outcome(op, owner=False, volumes=names if data["hdd_size"] or data["template"] else [])
+        names = [data["name"] + ext for ext in ("", ".img", ".qcow", ".qcow2")] if data["hdd_size"] or data["template"] else []
+        if data.get("cloud_init"):
+            names.append(f"cidata-{op.intent['uuid']}.iso")
+        return vm_outcome(op, owner=False, volumes=names)
 
     def log_entry(self, op, state):
         return _log(op, state, "create")

instances/models.py

@@ -375,6 +375,8 @@ class CreateInstance(models.Model):
     qemu_ga = models.BooleanField(default=False)
     mac = models.CharField(max_length=255, blank=True)  # one per NIC, comma separated
     add_cdrom = models.CharField(max_length=16)
+    # a NoCloud seed on a CD-ROM: the hostname and the creating user's SSH keys
+    cloud_init = models.BooleanField(default=False)
     add_input = models.CharField(max_length=16)
     video = models.CharField(
         max_length=16, error_messages={"required": _("Please select a video driver")}

instances/templates/create_instance_w2.html

@@ -329,6 +329,11 @@
                                     <input id="vm_qemu_ga" class="form-check-input" type="checkbox" name="qemu_ga" value="true"{% if vm_form.qemu_ga.value %} checked{% endif %}>
                                     <label for="vm_qemu_ga" class="form-check-label">{% trans "Guest agent" %}</label>
                                 </div>
+                                <div class="form-check">
+                                    <input id="vm_cloud_init" class="form-check-input" type="checkbox" name="cloud_init" value="true"{% if vm_form.cloud_init.value %} checked{% endif %} aria-describedby="vm_cloud_init_help">
+                                    <label for="vm_cloud_init" class="form-check-label">{% trans "Cloud-init seed" %}</label>
+                                    <div id="vm_cloud_init_help" class="form-text">{% trans "The hostname and your SSH keys on the CD-ROM, for cloud images" %}</div>
+                                </div>
                                 {% if virtio_support %}
                                 <div class="form-check">
                                     <input id="vm_virtio" class="form-check-input" type="checkbox" name="virtio" value="{{ virtio_support }}"{% if vm_form.virtio.value %} checked{% endif %}>

instances/tests/test_cloud_init_create.py

@@ -0,0 +1,98 @@
+"""A new VM can get a cloud-init NoCloud seed: its hostname and the creating
+user's SSH keys on a CD-ROM, read at first boot without the network."""
+
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import libvirt
+from accounts.models import UserSSHKey
+from appsettings.settings import get_settings
+from computes.models import Compute
+from django.contrib.auth import get_user_model
+from django.test import TransactionTestCase
+from instances.jobs import CreateInstance
+from operations.jobs import Context
+from operations.models import Operation, acquire
+from vrtManager.util import OperationError
+
+from instances.tests.test_create_job import BODY
+
+SEED = "/srv/default/cidata-x.iso"
+
+
+class CloudInitCreateTestCase(TransactionTestCase):
+    def setUp(self):
+        get_settings()
+        self.admin = get_user_model().objects.create_superuser("ci_admin", "a@example.com", "pw")
+        self.client.force_login(self.admin)
+        UserSSHKey.objects.create(user=self.admin, keyname="laptop", keypublic="ssh-ed25519 AAAA laptop")
+        self.compute = Compute.objects.create(name="ci-c", hostname="192.0.2.41", login="root", password="", type=1)
+        patcher = patch("instances.jobs.wvmCreate")
+        self.conn = patcher.start().return_value
+        self.addCleanup(patcher.stop)
+        self.conn.get_instances.return_value = []
+        self.conn.get_cache_modes.return_value = {"none": "Disable cache"}
+        self.conn.get_disk_bus_types.return_value = ["virtio", "sata"]
+        self.conn.create_volume.return_value = "/srv/default/api-vm.qcow2"
+        self.conn.create_seed.return_value = SEED
+        self.conn.get_storage.return_value.XMLDesc.return_value = "<pool type='dir'/>"
+        self.conn.get_storage.return_value.name.return_value = "default"
+
+    def post(self, **fields):
+        url = f"/api/v1/computes/{self.compute.pk}/instances/create/x86_64/q35/"
+        return self.client.post(url, {**BODY, **fields}, content_type="application/json")
+
+    def run_job(self):
+        op = Operation.objects.get()
+        acquire("wvc-test:1:a", "wvc-test")
+        Operation.objects.filter(pk=op.pk).update(state=Operation.RUNNING, worker="wvc-test:1:a")
+        op.refresh_from_db()
+        CreateInstance().run(op, Context(op, SimpleNamespace(identity="wvc-test:1:a")))
+        return op
+
+    def test_the_seed_goes_into_the_disks_pool_and_a_cdrom(self):
+        self.assertEqual(self.post(cloud_init=True).status_code, 202)
+        op = self.run_job()
+        pool, uuid, hostname, user_data = self.conn.create_seed.call_args.args
+        self.assertEqual((pool, uuid, hostname), ("default", op.intent["uuid"], "api-vm"))
+        # the same user-data the HTTP datasource serves
+        self.assertEqual(user_data.split(), ["#cloud-config", "ssh_authorized_keys:", "-", "ssh-ed25519", "AAAA", "laptop"])
+        cdrom = self.conn.create_instance.call_args.kwargs["volumes"][-1]
+        self.assertEqual((cdrom["path"], cdrom["device"], cdrom["bus"]), (SEED, "cdrom", "sata"))
+
+    def test_without_cloud_init_there_is_no_seed(self):
+        self.post()
+        self.run_job()
+        self.conn.create_seed.assert_not_called()
+
+    def test_a_seed_needs_a_cdrom_and_a_directory_pool(self):
+        kind = CreateInstance()
+        params = kind.normalize({**BODY, "compute": self.compute.pk, "arch": "x86_64", "machine": "q35",
+                                 "cloud_init": True, "add_cdrom": "None"})
+        with self.assertRaisesRegex(OperationError, "CD-ROM"):
+            kind.validate(self.admin, params)
+        self.conn.get_storage.return_value.XMLDesc.return_value = "<pool type='rbd'/>"
+        self.assertEqual(self.post(cloud_init=True, submission_key="key-rbd00001").status_code, 400)
+        self.assertFalse(Operation.objects.exists())
+
+    def test_a_failed_create_deletes_the_seed_too(self):
+        self.post(cloud_init=True)
+        self.conn.create_instance.side_effect = libvirt.libvirtError("define failed")
+        with self.assertRaises(libvirt.libvirtError):
+            self.run_job()
+        deleted = [c.args[0] for c in self.conn.delete_volume.call_args_list]
+        self.assertEqual(deleted, ["/srv/default/api-vm.qcow2", SEED])
+
+    @patch("instances.jobs.wvmConnect")
+    def test_recovery_names_a_seed_left_behind(self, conn):
+        op = Operation(params={**BODY, "template": "", "cloud_init": True},
+                       intent={"compute": self.compute.pk, "uuid": "abababab-0000-1111-2222-333333333333",
+                               "name": "api-vm"})
+        missing = libvirt.libvirtError("no domain")
+        missing.get_error_code = lambda: libvirt.VIR_ERR_NO_DOMAIN
+        conn.return_value.wvm.lookupByUUIDString.side_effect = missing
+        conn.return_value.wvm.listAllStoragePools.return_value = [
+            SimpleNamespace(listVolumes=lambda: ["cidata-abababab-0000-1111-2222-333333333333.iso"])]
+        outcome = CreateInstance().reconcile(op)
+        self.assertEqual(outcome.state, Operation.UNCERTAIN)
+        self.assertIn("cidata-abababab-0000-1111-2222-333333333333.iso", outcome.error)

instances/tests/test_create_form_input.py

@@ -76,6 +76,16 @@ class CreateFormInputTests(TestCase):
         self.assertRedirects(response, reverse("instances", args=[compute.id]), fetch_redirect_response=False)
         self.assertEqual(Operation.objects.get().kind, "instance.create")
 
+    def test_the_cloud_init_seed_is_chosen_on_the_form(self):
+        _, response = render_create_page(self)
+        (box,) = html.fromstring(response.content).xpath("//input[@name='cloud_init']")
+        self.assertIsNone(box.get("checked"))
+        # the test driver has no CD-ROM bus to pass the job's checks: what the form hands over
+        queued = Operation(pk=1, params={"machine": "pc"})
+        with patch("instances.views.jobs.submit", return_value=queued) as submit:
+            render_create_page(self, valid(disk_source="new", hdd_size="10", storage="default-pool", cloud_init="true"))
+        self.assertIs(submit.call_args.args[2]["cloud_init"], True)
+
     def test_two_disk_sources_are_refused(self):
         # the page sends only the chosen source; a crafted request with two queues nothing
         _, response = render_create_page(self, valid(