diff --git a/README.md b/README.md
index e9bfa054..1cee0540 100644
--- a/README.md
+++ b/README.md
@@ -772,7 +772,7 @@ What an update from an older version changes for operators and users:
- The panel backs itself up every day ([Backups](#backups)); in Docker into a new volume, `webvirtcloud-backups`.
- In Docker, the TLS client certificates of TLS computes are kept in a new volume, `webvirtcloud-pki` (`/var/www/.pki`), and backed up; put them there once ([Adding a Compute Node](#adding-a-compute-node), step 2).
- The SQLite database must be on a local file system (WAL). Outside compose, stop the Docker container with `docker stop -t 95` so the worker can finish running jobs.
-- Long operations run as jobs in a worker process (`manage.py run_worker`): creating, cloning and migrating VMs, cloning volumes, taking and reverting snapshots and deleting external snapshots. The page returns at once and the job panel and the Jobs page follow them; a migration can be cancelled. The API's VM create and migrate answer 202 with the job and need a new `submission_key` per request. The create form no longer takes a console password; set it on the VM page.
+- Long operations run as jobs in a worker process (`manage.py run_worker`): creating, cloning, migrating and deleting VMs, cloning volumes, taking and reverting snapshots and deleting external snapshots. The page returns at once and the job panel and the Jobs page follow them; a migration can be cancelled. The API's VM create, migrate and delete answer 202 with the job and need a new `submission_key` per request. The create form no longer takes a console password; set it on the VM page.
- An ISO upload is tracked as a job that can be cancelled. The browser sends the file, so stay on the page until it ends; an upload left idle for an hour is ended and its partial file deleted.
- A clone of a running VM (libvirt 11.10 or later, local file disks, no TPM) locks the VM until the copy ends and can be cancelled; cancelling a clone of a shut-off VM has no effect. A job that ends uncertain keeps its VM locked until a superuser resolves it on its job page.
- The job worker runs at most half of its concurrency (Settings, at least one) of one host's jobs at a time, so a host that does not answer cannot take every slot.
diff --git a/instances/api/serializers.py b/instances/api/serializers.py
index 33dca89e..e5b72a47 100644
--- a/instances/api/serializers.py
+++ b/instances/api/serializers.py
@@ -70,6 +70,12 @@ class DestroyOptionsSerializer(serializers.Serializer):
delete_nvram = serializers.BooleanField(default=False)
+class DestroySerializer(DestroyOptionsSerializer):
+ """The API's delete: a job, which a new submission key per request names."""
+
+ submission_key = serializers.CharField(write_only=True)
+
+
class CreateInstanceSerializer(serializers.ModelSerializer):
# BIOS, UEFI or UEFI Secure Boot, as the host offers; left out: the machine's default
firmware = serializers.CharField(required=False)
diff --git a/instances/api/viewsets.py b/instances/api/viewsets.py
index 2b5a6cd5..1022d0c3 100644
--- a/instances/api/viewsets.py
+++ b/instances/api/viewsets.py
@@ -7,7 +7,6 @@ from django.shortcuts import get_object_or_404
from django.urls import reverse
from instances.jobs import MIGRATE_OPTIONS
from instances.models import Flavor, Instance
-from instances.views import destroy as instance_destroy
from instances.views import (
force_off,
get_instance,
@@ -28,7 +27,7 @@ from webvirtcloud.serializers import ActionStatusSerializer
from .serializers import (
CreateInstanceSerializer,
- DestroyOptionsSerializer,
+ DestroySerializer,
FlavorSerializer,
InstanceDetailsSerializer,
InstanceSerializer,
@@ -40,8 +39,8 @@ def instance_action_response(action, request, pk, message):
response = action(request, pk)
if response.status_code >= 400:
return response
- # the page's view reports a partial result (a disk that was kept) as a
- # message for the page it redirects to: the API answers with them
+ # the page's view reports a partial result as a message for the page it
+ # redirects to: the API answers with them
return Response({"status": message, "messages": [str(m) for m in get_messages(request)]})
@@ -124,12 +123,19 @@ class InstanceViewSet(viewsets.ViewSet):
return Response(serializer.data)
- @extend_schema(request=DestroyOptionsSerializer, responses=ActionStatusSerializer)
+ @extend_schema(request=DestroySerializer, responses={202: OperationSerializer, 400: dict})
def destroy(self, request, pk=None, compute_pk=None):
queryset = get_instance(request.user, pk, perm_type="delete")
if compute_pk is not None and str(queryset.compute_id) != str(compute_pk):
raise Http404()
- return instance_action_response(instance_destroy, request, pk, "Instance is destroyed")
+ serializer = DestroySerializer(data=request.data)
+ if not serializer.is_valid():
+ return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+ data = dict(serializer.validated_data)
+ key = data.pop("submission_key")
+ op = jobs.submit("instance.delete", request.user, {"instance": queryset.pk, **data}, key)
+ return Response(OperationSerializer(op, context={"request": request}).data, status=status.HTTP_202_ACCEPTED,
+ headers={"Location": reverse("operation-detail", args=[op.pk])})
@extend_schema(request=None, responses=ActionStatusSerializer)
@action(detail=True, methods=["post"])
diff --git a/instances/jobs.py b/instances/jobs.py
index 04d61708..43536877 100644
--- a/instances/jobs.py
+++ b/instances/jobs.py
@@ -14,7 +14,9 @@ from django.template.loader import render_to_string
from django.utils.translation import gettext_noop
from django.utils.translation import gettext_noop as _
from libvirt import (VIR_CONNECT_LIST_STORAGE_POOLS_ACTIVE, VIR_DOMAIN_JOB_NONE, VIR_DOMAIN_SNAPSHOT_LIST_EXTERNAL,
- VIR_DOMAIN_SHUTOFF, VIR_ERR_NO_DOMAIN, VIR_ERR_NO_STORAGE_VOL, libvirtError)
+ VIR_DOMAIN_SHUTOFF, VIR_DOMAIN_UNDEFINE_KEEP_NVRAM, VIR_DOMAIN_UNDEFINE_MANAGED_SAVE,
+ VIR_DOMAIN_UNDEFINE_NVRAM, VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA, VIR_ERR_NO_DOMAIN,
+ VIR_ERR_NO_STORAGE_VOL, libvirtError)
from lxml import etree
from vrtManager import util
from vrtManager.connection import wvmConnect
@@ -27,6 +29,7 @@ from appsettings.settings import app_settings
from computes.models import Compute
from instances import utils
from instances.models import Instance
+from logs.audit import write
from operations import claims
from operations.utils import safe_text, say
from operations.jobs import Cancelled, Kind, Outcome, Plan, register
@@ -1215,3 +1218,108 @@ class MoveDisk(Kind):
def log_entry(self, op, state):
return _log(op, state, f"move disk {op.params['dev']} to pool {op.params['pool']}")
+
+
+DISK_KEPT_BELOW = _("Disk %(path)s was kept: a disk file that stays needs it")
+DISK_NOT_DELETED = _("The VM is removed, but disk %(path)s was not deleted: %(err)s")
+DISK_SHARED = _("Disk %(path)s is used by another VM and was kept")
+
+
+def _deletable(user, pk):
+ """The delete view's rule: the VM's delete right."""
+ from instances.views import get_instance
+
+ if user is None or not user.is_active:
+ raise PermissionDenied
+ return get_instance(user, pk, perm_type="delete")
+
+
+def _removed(op, recovering=False):
+ """What follows the undefine: the VM's record, then the disk files the job
+ chose before it stopped the VM; the job's result. Each disk has its own
+ Logs row; a file that stays keeps the files below it, which it needs. In
+ recovery a file already gone was deleted before the worker stopped."""
+ Instance.objects.filter(compute_id=op.intent["compute"], uuid=op.intent["uuid"]).delete()
+ compute = Compute.objects.get(pk=op.intent["compute"])
+ conn = wvmConnect(compute.hostname, compute.login, compute.password, compute.type)
+ notes, kept = [], set()
+ for path in op.intent["to_delete"]:
+ if path in kept:
+ notes += say(DISK_KEPT_BELOW, path=path)
+ continue
+ try:
+ conn.get_volume_by_path(path).delete(0)
+ except util.ConnectionFailed:
+ raise # unreachable: recovery waits and retries
+ except libvirtError as err:
+ if recovering and err.get_error_code() == VIR_ERR_NO_STORAGE_VOL:
+ continue
+ notes += say(DISK_NOT_DELETED, path=path, err=err)
+ write(op.username, compute.name, op.intent["name"], f"VM {op.intent['name']}: delete volume {path}: "
+ f"{safe_text(err)}", False, uuid=op.intent["uuid"])
+ try:
+ while (path := conn.backing_path(path)) and path not in kept:
+ kept.add(path)
+ except libvirtError:
+ kept.update(op.intent["to_delete"]) # the files it needs are unknown: none of the rest goes
+ else:
+ write(op.username, compute.name, op.intent["name"], f"VM {op.intent['name']}: delete volume {path}",
+ True, uuid=op.intent["uuid"])
+ for path in op.intent["shared"]:
+ notes += say(DISK_SHARED, path=path)
+ return {"open": reverse("instances:index"), **({"warning": notes} if notes else {})}
+
+
+@register
+class DeleteInstance(Kind):
+ """Undefine the VM, then remove its record and the disk files chosen
+ before it was stopped. Recovery finishes a delete libvirt has done."""
+
+ name = "instance.delete"
+ label = gettext_noop("Delete VM")
+
+ def normalize(self, raw):
+ return {"instance": int(raw["instance"]), "delete_disk": bool(raw.get("delete_disk")),
+ "delete_nvram": bool(raw.get("delete_nvram"))}
+
+ def validate(self, user, params):
+ _deletable(user, params["instance"])
+
+ def plan(self, user, params):
+ instance = _instance(params["instance"])
+ # the whole compute: a file to delete must not be attached by another VM meanwhile
+ return Plan(claims={claims.compute_key(instance.compute_id): Claim.EXCLUSIVE,
+ claims.instance_key(instance.uuid): Claim.EXCLUSIVE},
+ resource=claims.instance_key(instance.uuid), label=f"{instance.compute.name}/{instance.name}",
+ intent={"compute": instance.compute_id, "uuid": instance.uuid, "name": instance.name})
+
+ def precheck(self, op, ctx):
+ if _deletable(op.user, op.params["instance"]).compute_id != op.intent["compute"]:
+ raise util.OperationError(MOVED)
+
+ def run(self, op, ctx):
+ instance = _instance(op.params["instance"])
+ proxy = _proxy(instance)
+ to_delete, shared = [], []
+ if op.params["delete_disk"]:
+ # known before the VM is stopped: a host that cannot be checked stops nothing
+ to_delete, shared = proxy.split_disk_paths_by_use(
+ lambda own: utils.paths_used_on_other_computes(instance.compute, own))
+ ctx.record_intent(to_delete=to_delete, shared=shared)
+ ctx.set_phase(gettext_noop("Deleting the VM"))
+ # paused and other active states stop too, or the VM runs on as a transient domain
+ if proxy.instance.isActive():
+ proxy.force_shutdown()
+ # snapshot metadata goes with the domain; internal snapshot data stays in kept disk images
+ flags = VIR_DOMAIN_UNDEFINE_MANAGED_SAVE | VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA
+ flags |= VIR_DOMAIN_UNDEFINE_NVRAM if op.params["delete_nvram"] else VIR_DOMAIN_UNDEFINE_KEEP_NVRAM
+ proxy.delete(flags)
+ return _removed(op)
+
+ def reconcile(self, op):
+ if _domain(op.intent["compute"], op.intent["uuid"]) is not None:
+ return Outcome(Operation.FAILED, error=say(_("The VM was not deleted; it may have been shut off")))
+ return Outcome(Operation.SUCCEEDED, result=_removed(op, recovering=True))
+
+ def log_entry(self, op, state):
+ return _log(op, state, "delete")
diff --git a/instances/templates/instances/destroy_instance_form.html b/instances/templates/instances/destroy_instance_form.html
index e495faef..252ae1ad 100644
--- a/instances/templates/instances/destroy_instance_form.html
+++ b/instances/templates/instances/destroy_instance_form.html
@@ -1,5 +1,6 @@
{% extends 'base.html' %}
{% load i18n %}
+{% load operations %}
{% block title %}{% blocktrans with name=instance.name %}Destroy {{ name }}{% endblocktrans %}{% endblock title %}
{% block page_heading %}
@@ -8,7 +9,7 @@
{% block content %}
{% trans "The VM's definition, saved state and snapshot records are removed. A running VM is forced off first; what the guest has not written to disk is lost." %}