webvirtcloud/instances/jobs.py

"""Jobs of the instances app (operations framework, spec section 5)."""

import math
import re

from django.core.exceptions import PermissionDenied
from django.utils.translation import gettext_noop
from django.utils.translation import gettext_noop as _
from libvirt import (VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE, VIR_DOMAIN_JOB_NONE, VIR_DOMAIN_SNAPSHOT_LIST_EXTERNAL,
                     VIR_ERR_NO_DOMAIN, libvirtError)
from vrtManager import util
from vrtManager.connection import wvmConnect
from vrtManager.create import wvmCreate
from vrtManager.instance import wvmInstance

from accounts.models import UserInstance
from appsettings.settings import app_settings
from computes.models import Compute
from instances import utils
from instances.models import Instance
from operations import claims
from operations.jobs import Kind, Outcome, Plan, register
from operations.models import Claim, Operation


def _authorized(user, pk):
    """The instance views' rule: change access and the snapshot permission."""
    from instances.views import get_instance

    if user is None or not user.is_active:
        raise PermissionDenied
    instance = get_instance(user, pk, perm_type="change")
    if not user.has_perm("instances.snapshot_instances"):
        raise PermissionDenied
    return instance


def _proxy(instance):
    c = instance.compute
    return wvmInstance(c.hostname, c.login, c.password, c.type, instance.name, uuid=instance.uuid)


def _instance(pk):
    return Instance.objects.select_related("compute").get(pk=pk)


def _vm_plan(params):
    """A job on one VM: the VM exclusive, its compute shared."""
    instance = _instance(params["instance"])
    return Plan(
        claims={claims.compute_key(instance.compute_id): Claim.SHARED,
                claims.instance_key(instance.uuid): Claim.EXCLUSIVE},
        resource=claims.instance_key(instance.uuid),
        label=f"{instance.compute.name}/{instance.name}",
        intent={"compute": instance.compute_id},
    )


def _checked_proxy(op):
    """In the worker: the user may still change the VM, and it did not move."""
    instance = _authorized(op.user, op.params["instance"])
    if instance.compute_id != op.intent["compute"]:
        raise util.OperationError("The VM moved to another compute after the job was queued")
    return _proxy(instance)


def _recovery_proxy(op):
    """The VM for recovery; None when it left the compute the job ran on."""
    instance = Instance.objects.select_related("compute").filter(pk=op.params["instance"]).first()
    if instance is None or instance.compute_id != op.intent.get("compute"):
        return None
    try:
        return _proxy(instance)
    except libvirtError as err:
        if err.get_error_code() == VIR_ERR_NO_DOMAIN:
            return None
        raise  # unreachable: recovery waits and retries


def _libvirt_busy(proxy):
    """libvirt still runs a job on the VM that the worker no longer waits for."""
    return proxy.instance.isActive() and proxy.instance.jobInfo()[0] != VIR_DOMAIN_JOB_NONE


def _log(op, state, message):
    compute, _sep, vm = op.resource_label.rpartition("/")  # a VM name has no "/"
    return compute, vm, message if state == Operation.SUCCEEDED else f"{message} ({state})"


def _snapshot_free_space():
    """The free space each storage pool keeps for a snapshot, from the
    Settings page; an invalid value leaves the backend's default."""
    values = {}
    for key, argument in (("SNAPSHOT_FREE_SPACE_PERCENT", "free_percent"), ("SNAPSHOT_FREE_SPACE_MIN_GIB", "free_min_gib")):
        try:
            value = float(getattr(app_settings, key))
        except (AttributeError, TypeError, ValueError):
            continue
        if math.isfinite(value) and value >= 0:
            values[argument] = value
    return values


CLONE_POST_KEY_RE = re.compile(r"^(clone-net-mac-\d+|disk-[a-z0-9]+|meta-[a-z0-9]+)$")
CLONE_FIELDS = ("name", "clone-title", "clone-description")


def clone_data(user, instance, fields):
    """The clone's settings from the form fields and the source as it is now;
    OperationError with the reason when it cannot be made."""
    from instances.views import VOLUME_NAME_RE, quota_refused

    data = dict()
    data["name"] = fields.get("name", "").strip()
    data["clone-title"] = fields.get("clone-title", "").strip()
    data["clone-description"] = fields.get("clone-description", "").strip()

    # the clone copies the disks of the persistent definition
    disk_sum = sum(int(disk.get("size") or 0) >> 30 for disk in instance.config_disks)
    quota_msg = utils.check_user_quota(user, 1, instance.vcpu, instance.memory, disk_sum)

    data["disk_owner_uid"] = int(app_settings.INSTANCE_VOLUME_DEFAULT_OWNER_UID)
    data["disk_owner_gid"] = int(app_settings.INSTANCE_VOLUME_DEFAULT_OWNER_GID)

    # Only superusers may choose disk names and MAC addresses; the form only
    # shows those fields to them.
    if user.is_superuser:
        for key, value in fields.items():
            if CLONE_POST_KEY_RE.fullmatch(key):
                data[key] = value.strip()

    if app_settings.CLONE_INSTANCE_AUTO_NAME == "True" and not data["name"]:
        auto_vname = utils.get_clone_free_names()[0]
        data["name"] = auto_vname
        data["clone-net-mac-0"] = utils.get_dhcp_mac_address(auto_vname)
        for disk in instance.config_disks:
            data[f"disk-{disk['dev']}"] = utils.get_clone_disk_name(disk, instance.name, auto_vname)

    if not user.is_superuser:
        for disk in instance.config_disks:
            data[f"disk-{disk['dev']}"] = utils.get_clone_disk_name(disk, instance.name, data["name"])
        for num in range(max(1, len(instance.config_networks))):
            key = f"clone-net-mac-{num}"
            if not data.get(key):
                data[key] = (utils.get_dhcp_mac_address(data["name"]) if num == 0 else "") or utils.get_random_mac_address()

    check_instance = Instance.objects.filter(name=data["name"])
    invalid_macs = [
        value for key, value in data.items()
        if key.startswith("clone-net-mac-") and not utils.MAC_ADDRESS_RE.fullmatch(value)
    ]
    invalid_disks = [
        value for key, value in data.items()
        # None: a disk without a volume, which clone_instance does not copy.
        if key.startswith("disk-") and value is not None and not VOLUME_NAME_RE.fullmatch(value)
    ]

    if not user.is_superuser and quota_msg:
        raise util.OperationError(quota_refused(quota_msg, _(
            "User '%(quota_msg)s' quota reached, cannot create '%(clone_name)s'!"
        ) % {"quota_msg": quota_msg, "clone_name": data["name"]}))
    if check_instance:
        raise util.OperationError(_("Instance '%(clone_name)s' already exists!") % {"clone_name": data["name"]})
    if not re.match(r"^[a-zA-Z0-9-]+$", data["name"]):
        raise util.OperationError(
            _("Instance name '%(clone_name)s' contains invalid characters!") % {"clone_name": data["name"]})
    if "clone-net-mac-0" not in data or invalid_macs:
        raise util.OperationError(_("Instance MAC '%(clone_mac)s' invalid format!") % {"clone_mac": ", ".join(invalid_macs)})
    if invalid_disks:
        raise util.OperationError(
            _("Disk name '%(disk_name)s' contains invalid characters!") % {"disk_name": ", ".join(str(v) for v in invalid_disks)})
    return data


def vm_claims(compute, name):
    """A job that adds a VM: other VMs stay usable; the VM list reconcile waits."""
    return {claims.compute_key(compute.pk): Claim.SHARED, claims.reconcile_key(compute.pk): Claim.EXCLUSIVE,
            claims.name_key(compute.pk, name): Claim.EXCLUSIVE}


def record_vm(op, owner):
    """The panel's records of the new VM, as the synchronous views wrote them."""
    vm = Instance.objects.get_or_create(compute_id=op.intent["compute"], uuid=op.intent["uuid"],
                                        defaults={"name": op.intent["name"]})[0]
    if owner and op.user_id:
        UserInstance.objects.get_or_create(instance=vm, user_id=op.user_id,
                                           defaults={"is_delete": True, "is_change": True, "is_vnc": True})


def vm_outcome(op, owner, volumes):
    """The VM is defined last: if it exists, the job succeeded. If it does not and
    none of its planned volumes exists either, the job failed and changed nothing."""
    c = Compute.objects.get(pk=op.intent["compute"])
    conn = wvmConnect(c.hostname, c.login, c.password, c.type).wvm
    try:
        conn.lookupByUUIDString(op.intent["uuid"])
    except libvirtError as err:
        if err.get_error_code() != VIR_ERR_NO_DOMAIN:
            raise  # unreachable: recovery waits and retries
        held = set()
        for pool in conn.listAllStoragePools(VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE):
            held.update(pool.listVolumes())
        left = sorted(held & set(volumes))
        if not left:
            return Outcome(Operation.FAILED, error="The VM was not defined and no volume of it was left.")
        return Outcome(Operation.UNCERTAIN,
                       error=f"The VM was not defined; check these volumes, then resolve this job: {', '.join(left)}")
    record_vm(op, owner)
    return Outcome(Operation.SUCCEEDED)


def _clonable(user, pk):
    from instances.views import _clone_source

    if user is None or not user.has_perm("instances.clone_instances"):
        raise PermissionDenied
    return _clone_source(user, pk)


def _merged(jobs):
    """(done, total) bytes of the snapshot's block jobs."""
    return sum(j["cur"] for j in jobs), sum(j["end"] for j in jobs)


@register
class DeleteExternalSnapshot(Kind):
    name = "snapshot.delete"
    label = gettext_noop("Delete snapshot")
    time_limit = 6 * 3600  # a merge cannot be stopped: over time is only noted
    progress_bytes = True  # libvirt block job cur/end

    def normalize(self, raw):
        return {"instance": int(raw["instance"]), "name": str(raw.get("name", ""))}

    def validate(self, user, params):
        instance = _authorized(user, params["instance"])
        if params["name"] not in _proxy(instance).instance.snapshotListNames(VIR_DOMAIN_SNAPSHOT_LIST_EXTERNAL):
            raise util.OperationError(f"The VM has no external snapshot named {params['name']}")

    def plan(self, user, params):
        return _vm_plan(params)

    def precheck(self, op, ctx):
        proxy = _checked_proxy(op)
        proxy._snapshot(op.params["name"], external=True)
        proxy._check_snapshot_state()

    def run(self, op, ctx):
        proxy = _proxy(_instance(op.params["instance"]))
        ctx.record_intent(before=proxy.snapshot_files_state())
        ctx.set_phase(gettext_noop("Merging the snapshot"))
        return {"left_behind": proxy.delete_external_snapshot(op.params["name"])}

    def progress(self, op):
        jobs = _proxy(_instance(op.params["instance"])).get_snapshot_jobs()
        return _merged(jobs) if jobs else None

    def reconcile(self, op):
        instance = Instance.objects.select_related("compute").filter(pk=op.params["instance"]).first()
        if instance is None or instance.compute_id != op.intent.get("compute"):
            return Outcome(Operation.UNCERTAIN, error="The VM is no longer on the compute the job ran on")
        try:
            proxy = _proxy(instance)
            listed = op.params["name"] in proxy.instance.snapshotListNames(VIR_DOMAIN_SNAPSHOT_LIST_EXTERNAL)
            jobs = proxy.get_snapshot_jobs()
        except util.ConnectionFailed:
            raise  # unreachable: recovery waits and retries
        except libvirtError as err:  # OperationError is a libvirtError too
            if err.get_error_code() == VIR_ERR_NO_DOMAIN:
                return Outcome(Operation.UNCERTAIN, error="The VM is no longer on its compute")
            raise
        if jobs:
            return Outcome(Operation.RUNNING, phase=gettext_noop("libvirt is still merging the snapshot"),
                           stalled=all(j["cur"] == j["end"] for j in jobs),
                           progress=_merged(jobs))
        if not listed:
            before = op.intent.get("before")  # recorded with effects_started; absent only if run never began
            left = proxy.left_behind_since(before) if before else []
            return Outcome(Operation.SUCCEEDED, result={"left_behind": left})
        return Outcome(Operation.UNCERTAIN,
                       error="The snapshot is still listed and no merge runs. It cannot be told whether the disks "
                             "changed; check the VM before deleting the snapshot again or reverting to it.")

    def log_entry(self, op, state):
        return _log(op, state, f"Delete external snapshot: {op.params['name']}")


GONE = "The VM is no longer on the compute the job ran on"


@register
class CreateSnapshot(Kind):
    name = "snapshot.create"
    label = gettext_noop("Take snapshot")
    time_limit = 6 * 3600  # a snapshot cannot be stopped: over time is only noted

    def normalize(self, raw):
        return {"instance": int(raw["instance"]), "name": str(raw.get("name", "")),
                "description": str(raw.get("description", "")), "external": bool(raw.get("external")),
                "memory": bool(raw.get("memory")), "quiesce": bool(raw.get("quiesce"))}

    def validate(self, user, params):
        # recovery counts a listed snapshot as taken by the job
        if params["name"] in _proxy(_authorized(user, params["instance"])).instance.snapshotListNames(0):
            raise util.OperationError(f"A snapshot named {params['name']} already exists")

    def plan(self, user, params):
        return _vm_plan(params)

    def precheck(self, op, ctx):
        _checked_proxy(op)

    def run(self, op, ctx):
        p = op.params
        proxy = _proxy(_instance(p["instance"]))
        ctx.record_intent()
        ctx.set_phase(gettext_noop("Taking the snapshot"))
        if p["external"]:
            proxy.create_external_snapshot(p["name"], p["description"], p["memory"], p["quiesce"],
                                           **_snapshot_free_space())
        else:
            proxy.create_snapshot(p["name"], p["description"])
        return {}

    def reconcile(self, op):
        proxy = _recovery_proxy(op)
        if proxy is None:
            return Outcome(Operation.UNCERTAIN, error=GONE)
        if _libvirt_busy(proxy):
            return Outcome(Operation.RUNNING, phase=gettext_noop("libvirt is still taking the snapshot"))
        if op.params["name"] in proxy.instance.snapshotListNames(0):
            return Outcome(Operation.SUCCEEDED)
        return Outcome(Operation.FAILED, error="The snapshot was not taken")

    def log_entry(self, op, state):
        kind = "external snapshot" if op.params["external"] else "snapshot"
        return _log(op, state, f"Create {kind}: {op.params['name']}")


def _current(proxy):
    dom = proxy.instance
    return dom.snapshotCurrent(0).getName() if dom.hasCurrentSnapshot(0) else None


@register
class RevertSnapshot(Kind):
    name = "snapshot.revert"
    label = gettext_noop("Revert snapshot")
    time_limit = 6 * 3600  # a revert cannot be stopped: over time is only noted

    def normalize(self, raw):
        return {"instance": int(raw["instance"]), "name": str(raw.get("name", "")),
                "external": bool(raw.get("external"))}

    def validate(self, user, params):
        _proxy(_authorized(user, params["instance"]))._snapshot(params["name"], params["external"])

    def plan(self, user, params):
        return _vm_plan(params)

    def precheck(self, op, ctx):
        proxy = _checked_proxy(op)
        if op.params["external"]:
            proxy.check_external_revert(op.params["name"])
        else:
            proxy._snapshot(op.params["name"], False)

    def run(self, op, ctx):
        proxy = _proxy(_instance(op.params["instance"]))
        ctx.record_intent(was_current=_current(proxy))
        ctx.set_phase(gettext_noop("Reverting the VM"))
        if op.params["external"]:
            return {"left_behind": proxy.revert_external_snapshot(op.params["name"])}
        proxy.snapshot_revert(op.params["name"])
        return {}

    def reconcile(self, op):
        proxy = _recovery_proxy(op)
        if proxy is None:
            return Outcome(Operation.UNCERTAIN, error=GONE)
        if _libvirt_busy(proxy):
            return Outcome(Operation.RUNNING, phase=gettext_noop("libvirt is still reverting the VM"))
        if _current(proxy) == op.params["name"] != op.intent.get("was_current"):
            return Outcome(Operation.SUCCEEDED)
        return Outcome(Operation.UNCERTAIN, error="It cannot be told whether the VM was reverted; "
                                                  "check it, then revert again or resolve this job.")

    def log_entry(self, op, state):
        kind = "external snapshot" if op.params["external"] else "snapshot"
        return _log(op, state, f"Revert {kind}: {op.params['name']}")


@register
class CloneInstance(Kind):
    name = "instance.clone"
    label = gettext_noop("Clone VM")
    time_limit = 12 * 3600  # a copy cannot be stopped: over time is only noted

    def normalize(self, raw):
        fields = {k: str(v) for k, v in raw.items() if k in CLONE_FIELDS or CLONE_POST_KEY_RE.fullmatch(k)}
        return {"instance": int(raw["instance"]), "fields": fields}

    def plan(self, user, params):
        instance = _clonable(user, params["instance"])
        if instance.proxy.get_status() != 5:
            raise util.OperationError("Shut the VM down before cloning it")
        data = clone_data(user, instance, params["fields"])
        keys = vm_claims(instance.compute, data["name"])
        keys[claims.instance_key(instance.uuid)] = Claim.SHARED
        return Plan(claims=keys, resource=claims.instance_key(instance.uuid),
                    label=f"{instance.compute.name}/{instance.name}",
                    intent={"compute": instance.compute_id, "uuid": util.randomUUID(), "name": data["name"],
                            "clone": data})

    def precheck(self, op, ctx):
        instance = _clonable(op.user, op.params["instance"])
        if instance.compute_id != op.intent["compute"]:
            raise util.OperationError("The VM moved to another compute after the job was queued")

    def run(self, op, ctx):
        instance = _instance(op.params["instance"])
        ctx.record_intent()
        ctx.set_phase(gettext_noop("Copying disks"))
        instance.proxy.clone_instance(op.intent["clone"], op.intent["uuid"])
        record_vm(op, owner=True)
        return {}

    def reconcile(self, op):
        disks = [v for k, v in op.intent["clone"].items() if k.startswith("disk-") and v]
        return vm_outcome(op, owner=True, volumes=disks)

    def log_entry(self, op, state):
        compute, _sep, source = op.resource_label.rpartition("/")
        message = f"Create a clone of '{source}'"
        return compute, op.intent["name"], message if state == Operation.SUCCEEDED else f"{message} ({state})"


@register
class CreateInstance(Kind):
    name = "instance.create"
    label = gettext_noop("Create VM")
    time_limit = 12 * 3600  # a copy cannot be stopped: over time is only noted

    def normalize(self, raw):
        # what the API leaves out means what create_instance did by default
        params = {"template": "", "images": "", "firmware": "", "net_model": "virtio", "add_cdrom": "sata",
                  "add_input": "default", "meta_prealloc": False, "hdd_size": 0}
        params.update({k: v for k, v in raw.items() if k not in ("submission_key", "create", "csrfmiddlewaretoken")})
        params.update(compute=int(raw["compute"]), devices=[list(d) for d in raw.get("devices", [])])
        if "UEFI" in params["firmware"] and "secboot" in params["firmware"].partition(":")[2]:
            params["machine"] = "q35"  # a secure boot loader needs SMM, on q35 only
        return params

    def validate(self, user, params):
        from instances.views import CDROM_BUSES

        if not user.is_superuser:
            raise PermissionDenied
        data = params
        c = Compute.objects.get(pk=data["compute"])
        conn = wvmCreate(c.hostname, c.login, c.password, c.type)
        cdrom_buses = [bus for bus in conn.get_disk_bus_types(data["arch"], data["machine"]) if bus in CDROM_BUSES]
        if data["name"] in conn.get_instances():
            raise util.OperationError(_("A virtual machine with this name already exists"))
        if Instance.objects.filter(name__exact=data["name"]):
            raise util.OperationError(_("There is an instance with same name. Remove it and try again!"))
        if not (data["hdd_size"] or data["template"] or data["images"]):
            raise util.OperationError(_("First you need to create or select an image"))
        if not data["hdd_size"] and data["template"] and conn.get_volume_path(data["name"] + ".img", data["storage"]):
            raise util.OperationError(_("Image has already exist. Please check volumes or change instance name"))
        if data["cache_mode"] not in conn.get_cache_modes():
            raise util.OperationError(_("Invalid cache mode"))
        if data["add_cdrom"] != "None" and data["add_cdrom"] not in cdrom_buses:
            raise util.OperationError(_("A CD-ROM cannot use the %(bus)s bus") % {"bus": data["add_cdrom"]})
        try:
            utils.nic_macs(data["mac"], data["networks"])
        except ValueError as err:
            raise util.OperationError(str(err))
        if "UEFI" in data["firmware"] and ":" not in data["firmware"]:
            raise util.OperationError(_("Invalid firmware"))

    def plan(self, user, params):
        compute = Compute.objects.get(pk=params["compute"])
        return Plan(claims=vm_claims(compute, params["name"]), resource=claims.compute_key(compute.pk),
                    label=f"{compute.name}/{params['name']}",
                    intent={"compute": compute.pk, "uuid": util.randomUUID(), "name": params["name"]})

    def run(self, op, ctx):
        data = op.params
        c = Compute.objects.get(pk=data["compute"])
        conn = wvmCreate(c.hostname, c.login, c.password, c.type)
        meta_prealloc = bool(data["meta_prealloc"])
        default_bus = app_settings.INSTANCE_VOLUME_DEFAULT_BUS
        default_disk_format = app_settings.INSTANCE_VOLUME_DEFAULT_FORMAT
        default_disk_owner_uid = int(app_settings.INSTANCE_VOLUME_DEFAULT_OWNER_UID)
        default_disk_owner_gid = int(app_settings.INSTANCE_VOLUME_DEFAULT_OWNER_GID)

        firmware = dict()
        if "UEFI" in data["firmware"]:
            firmware["loader"] = data["firmware"].split(":")[1].strip()
            firmware["secure"] = "yes" if "secboot" in firmware["loader"] else "no"
            firmware["readonly"] = "yes"
            firmware["type"] = "pflash"

        def volume(path, device="disk", bus=default_bus):
            return {
                "path": path,
                "type": conn.get_volume_format_type(path),
                "device": device,
                "bus": bus,
                "scsi_model": app_settings.INSTANCE_VOLUME_DEFAULT_SCSI_CONTROLLER if bus == "scsi" else None,
                "cache_mode": data["cache_mode"],
                "discard_mode": app_settings.INSTANCE_VOLUME_DEFAULT_DISCARD,
                "detect_zeroes_mode": app_settings.INSTANCE_VOLUME_DEFAULT_DETECT_ZEROES,
                "io_mode": app_settings.INSTANCE_VOLUME_DEFAULT_IO,
            }

        ctx.record_intent()
        ctx.set_phase(gettext_noop("Copying disks"))
        created = []  # volumes this job allocated
        volume_list = []
        try:
            if data["hdd_size"]:
                path = conn.create_volume(data["storage"], data["name"], data["hdd_size"], default_disk_format,
                                          meta_prealloc, default_disk_owner_uid, default_disk_owner_gid)
                created.append(path)
                volume_list.append(volume(path))
            elif data["template"]:
                path = conn.clone_from_template(data["name"], conn.get_volume_path(data["template"]), data["storage"],
                                                meta_prealloc, default_disk_owner_uid, default_disk_owner_gid)
                created.append(path)
                volume_list.append(volume(path))
            else:
                for vol, (device, bus) in zip(data["images"].split(","), data["devices"]):
                    volume_list.append(volume(conn.get_volume_path(vol), device, bus))

            ctx.set_phase(gettext_noop("Defining the VM"))
            conn.create_instance(
                name=data["name"],
                memory=data["memory"],
                vcpu=data["vcpu"],
                vcpu_mode=data["vcpu_mode"],
                uuid=op.intent["uuid"],
                arch=data["arch"],
                machine=data["machine"],
                firmware=firmware,
                volumes=volume_list,
                networks=data["networks"],
                virtio=data["virtio"],
                listener_addr=data["listener_addr"],
                nwfilter=data["nwfilter"],
                net_model="virtio" if data["net_model"] == "default" else data["net_model"],
                video=data["video"],
                console_pass="",
                mac=data["mac"],
                qemu_ga=data["qemu_ga"],
                add_cdrom=data["add_cdrom"],
                add_input=data["add_input"],
            )
        except Exception as err:
            # the VM is not defined: its new volumes go
            leftovers = []
            for path in created:
                try:
                    conn.delete_volume(path)
                except libvirtError:
                    leftovers.append(path)
            if leftovers:
                raise util.OperationError(f"{err}. Remove these volumes by hand: {', '.join(leftovers)}") from err
            raise
        record_vm(op, owner=False)
        return {}

    def reconcile(self, op):
        data = op.params
        names = [data["name"] + ext for ext in ("", ".img", ".qcow", ".qcow2")]
        return vm_outcome(op, owner=False, volumes=names if data["hdd_size"] or data["template"] else [])

    def log_entry(self, op, state):
        compute, _sep, name = op.resource_label.rpartition("/")
        message = "Instance is created"
        return compute, name, message if state == Operation.SUCCEEDED else f"{message} ({state})"