webvirtcloud/computes/utils.py

from contextlib import contextmanager
import logging
from datetime import timedelta

from accounts.models import UserInstance
from django.conf import settings
from django.contrib.auth import get_user_model
from django.db import connection, transaction
from django.utils import timezone
from computes.models import Compute
from instances.models import Instance, InstanceTombstone
from logs.audit import write
from operations import claims
from operations.models import Claim

logger = logging.getLogger(__name__)


@contextmanager
def libvirt_compute_lock(compute, *, shared=False):
    """The compute for the block: exclusive keeps out every VM operation and
    reconcile on it; shared lets sibling VMs and reconciles go on. Busy if it is taken."""
    compute_pk = getattr(compute, "pk", getattr(compute, "id", None))
    if not compute_pk:
        yield
        return
    with claims.hold({claims.compute_key(compute_pk): Claim.SHARED if shared else Claim.EXCLUSIVE}):
        yield


@contextmanager
def user_quota_lock(user):
    """One user's quota check and the change it allows run one at a time."""
    with claims.hold({claims.quota_key(user.pk): Claim.EXCLUSIVE}):
        yield


@contextmanager
def libvirt_instance_lock(instance):
    """One VM for the block, sharing its compute with sibling VMs."""
    compute = instance.compute
    with claims.hold({
        claims.compute_key(getattr(compute, "pk", getattr(compute, "id", None))): Claim.SHARED,
        claims.instance_key(instance.uuid): Claim.EXCLUSIVE,
    }):
        yield


# pg_advisory_xact_lock key of the ownership step ("wvc_own")
OWNERSHIP_LOCK = 0x7776635F6F776E


def lock_ownership():
    """Hold the ownership step of every compute until this transaction ends:
    a bury and a restore of the same UUID on two computes must not see each
    other's work half done. SQLite's IMMEDIATE transactions already do this."""
    if connection.vendor == "postgresql":
        with connection.cursor() as cursor:
            cursor.execute("SELECT pg_advisory_xact_lock(%s)", [OWNERSHIP_LOCK])


def _log(inst, what):
    write("-", inst.compute.name, inst.name, f"VM {inst.name} ({inst.uuid}): {what}", True, uuid=inst.uuid)


def _move_ownership(source, target):
    """Give target the owners and template flag of source; a user's rights are merged."""
    for ui in UserInstance.objects.filter(instance=source):
        existing = UserInstance.objects.filter(instance=target, user=ui.user).first()
        if existing is None:
            ui.instance = target
            ui.save(update_fields=["instance"])
            continue
        for flag in ("is_change", "is_delete", "is_vnc"):
            if getattr(ui, flag):
                setattr(existing, flag, True)
        existing.save()
    if source.is_template and not target.is_template:
        target.is_template = True
        target.save(update_fields=["is_template"])


def forget(inst):
    """Remove an instance whose VM its compute no longer lists, or whose
    compute is deleted. Its ownership goes to the same UUID on another
    compute (an out-of-band move) or is kept in a tombstone in case the VM
    comes back."""
    elsewhere = Instance.objects.filter(uuid=inst.uuid).exclude(compute_id=inst.compute_id).first()
    if elsewhere:
        _move_ownership(inst, elsewhere)
        _log(inst, f"owners moved to {elsewhere.compute.name}")
    else:
        _bury(inst)
    inst.delete()


def _bury(inst):
    owners = [
        {"user": ui.user_id, "is_change": ui.is_change, "is_delete": ui.is_delete, "is_vnc": ui.is_vnc}
        for ui in UserInstance.objects.filter(instance=inst)
    ]
    if owners or inst.is_template:
        InstanceTombstone.objects.create(uuid=inst.uuid, name=inst.name, host=inst.compute.name,
                                         is_template=inst.is_template, owners=owners)
        _log(inst, "owners kept on the Removed VMs page")
    else:
        _log(inst, "left the panel (no owners)")


def _restore(inst):
    tombstone = InstanceTombstone.objects.filter(uuid=inst.uuid).order_by("-removed").first()
    if tombstone is None:
        return
    users = set(get_user_model().objects.filter(id__in=[o["user"] for o in tombstone.owners]).values_list("id", flat=True))
    for owner in tombstone.owners:
        if owner["user"] in users:
            UserInstance.objects.create(
                instance=inst,
                user_id=owner["user"],
                is_change=owner["is_change"],
                is_delete=owner["is_delete"],
                is_vnc=owner["is_vnc"],
            )
    if tombstone.is_template:
        inst.is_template = True
        inst.save(update_fields=["is_template"])
    InstanceTombstone.objects.filter(uuid=inst.uuid).delete()
    _log(inst, "owners restored")


def refresh_instance_database(compute):
    """
    Synchronizes the WebVirtCloud database with libvirt domain state.
    Libvirt is the single source of truth:
      - VMs removed in virt-manager/virsh are deleted from the database; their
        ownership is moved to the same UUID on another compute or kept in a
        tombstone for INSTANCE_OWNERSHIP_RETENTION_DAYS.
      - VMs created in virt-manager/virsh are added to the database.
      - VMs renamed in virt-manager/virsh have their name updated without losing
        UserInstance permission relationships (matched by UUID).
      - If libvirt is unreachable (connection/network error), the database is left
        untouched to avoid false deletions.
    """
    compute_pk = getattr(compute, "pk", getattr(compute, "id", None))
    if not compute_pk:
        return

    try:
        # The compute shared: VM operations and jobs on it go on; changes of
        # its VM set (create, clone, delete, migration, define) hold it
        # exclusive and stay out until the end. One reconcile of a compute at
        # a time: two would read libvirt at different times and could commit
        # in reverse order, an older listing undoing a newer one.
        with claims.hold({claims.compute_key(compute_pk): Claim.SHARED,
                          claims.reconcile_key(compute_pk): Claim.EXCLUSIVE}):
            # libvirt is asked before the transaction: a silent host can keep a
            # call waiting for 30 s, and on SQLite an open transaction makes every
            # other write of the app fail meanwhile.
            if hasattr(compute, "status") and compute.status is False:
                return

            try:
                domains = compute.proxy.wvm.listAllDomains()
            except Exception as e:
                logger.warning(
                    "Failed to retrieve domains from compute %s (%s): %s",
                    getattr(compute, "name", compute),
                    getattr(compute, "hostname", ""),
                    e,
                )
                return

            # Map libvirt UUIDs to current domain names
            host_domains = {}
            for d in domains:
                try:
                    host_domains[d.UUIDString()] = d.name()
                except Exception as e:
                    logger.warning(
                        "Error reading domain metadata from compute %s: %s",
                        getattr(compute, "name", compute),
                        e,
                    )
                    return

            with transaction.atomic():
                lock_ownership()
                # the compute may have been deleted while libvirt was asked
                comp_locked = Compute.objects.select_for_update().filter(pk=compute_pk).first()
                if not comp_locked:
                    return

                db_instances = list(Instance.objects.filter(compute=compute).select_for_update())
                db_uuids = set(inst.uuid for inst in db_instances)

                # 1. Update name for any VM whose name changed out-of-band in virsh/virt-manager
                for inst in db_instances:
                    if inst.uuid in host_domains:
                        new_name = host_domains[inst.uuid]
                        if inst.name != new_name:
                            inst.name = new_name
                            inst.save(update_fields=["name"])

                # 2. Instances gone from this compute
                for inst in db_instances:
                    if inst.uuid not in host_domains:
                        forget(inst)

                # 3. Instances created out-of-band. A UUID that also exists on another
                #    compute is a copy and gets no owners; otherwise a tombstone's
                #    ownership is restored.
                retention = getattr(settings, "INSTANCE_OWNERSHIP_RETENTION_DAYS", 30)
                expired = InstanceTombstone.objects.filter(removed__lt=timezone.now() - timedelta(days=retention))
                for t in expired:
                    write("-", "-", "", f"VM {t.name} ({t.uuid}): removed record expired", True)
                expired.delete()
                for uuid, name in host_domains.items():
                    if uuid not in db_uuids:
                        inst, created = Instance.objects.get_or_create(
                            compute=compute,
                            uuid=uuid,
                            defaults={"name": name},
                        )
                        if created and not Instance.objects.filter(uuid=uuid).exclude(compute=compute).exists():
                            _restore(inst)
    except claims.Busy:
        logger.debug("Compute %s is busy; not synchronized now", compute_pk)
        return