mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-09 19:56:48 +00:00
f2138e54d8
Request jobs (an upload's chunks) run in gunicorn: their kind is admitted running with the request as its worker, never claimed or recovered by the worker, cancelled in the request, and asked by every housekeeping pass to end its jobs that went idle. The panel can follow a job the page just started. A job a request runs does not wait for the worker, so it does not raise the worker-unavailable warning on the Jobs page or in the panel.
278 lines
10 KiB
Python
278 lines
10 KiB
Python
"""The contract every kind of job follows (spec section 5), the kinds by name
|
|
(filled by each app's jobs.py via register) and the context a running job
|
|
reports through. Submitting, cancelling, finishing and resolving jobs (spec
|
|
sections 2-5)."""
|
|
|
|
import logging
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from datetime import timedelta
|
|
|
|
from django.core.exceptions import PermissionDenied
|
|
from django.db import transaction
|
|
from django.db.models import F, Value
|
|
from django.db.models.functions import Coalesce
|
|
from django.utils import timezone
|
|
|
|
from logs.models import Logs
|
|
from vrtManager.util import OperationError
|
|
|
|
from . import claims, utils
|
|
from .models import Claim, LeaseLost, Operation, fenced
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
KINDS = {}
|
|
|
|
SUBMISSION_KEY_TTL = timedelta(hours=24)
|
|
KEY_RE = re.compile(r"[A-Za-z0-9_-]{8,64}")
|
|
|
|
|
|
@dataclass
|
|
class Plan:
|
|
claims: dict
|
|
resource: str
|
|
label: str
|
|
intent: dict = field(default_factory=dict)
|
|
|
|
|
|
@dataclass
|
|
class Outcome:
|
|
"""What a run or an inspection decided. RUNNING: libvirt still works;
|
|
stalled: its work reports nothing left to do but does not end;
|
|
progress: (done, total) of that work, when libvirt tells."""
|
|
|
|
state: str
|
|
result: dict = field(default_factory=dict)
|
|
error: str = ""
|
|
phase: str = ""
|
|
stalled: bool = False
|
|
progress: tuple | None = None
|
|
|
|
|
|
class Cancelled(Exception):
|
|
"""Raised by a kind's run when a cancel stopped its work."""
|
|
|
|
|
|
def get_kind(name):
|
|
return KINDS[name]
|
|
|
|
|
|
def label_of(name):
|
|
kind = KINDS.get(name)
|
|
return kind.label if kind is not None else name
|
|
|
|
|
|
def register(cls):
|
|
KINDS[cls.name] = cls()
|
|
return cls
|
|
|
|
|
|
class Kind:
|
|
name = ""
|
|
label = ""
|
|
cancellable = False
|
|
time_limit = None
|
|
runs_in_request = False # its requests run it: admitted running, never the worker's
|
|
progress_bytes = False # progress counts bytes: the job page shows sizes
|
|
|
|
def normalize(self, raw):
|
|
"""Plain, allowlisted values from the request, without libvirt."""
|
|
raise NotImplementedError
|
|
|
|
def validate(self, user, params):
|
|
"""In the request: permissions and libvirt state. Raise to refuse."""
|
|
|
|
def plan(self, user, params):
|
|
raise NotImplementedError
|
|
|
|
def precheck(self, op, ctx):
|
|
"""In the worker, before any effect: permission and state again."""
|
|
|
|
def run(self, op, ctx):
|
|
raise NotImplementedError
|
|
|
|
def progress(self, op):
|
|
return None
|
|
|
|
def cancel(self, op):
|
|
raise NotImplementedError
|
|
|
|
def reconcile(self, op):
|
|
raise NotImplementedError
|
|
|
|
def expire(self, op):
|
|
"""Called on every housekeeping pass for an active job a request runs;
|
|
the kind decides whether the job is idle and ends it if so."""
|
|
|
|
def log_entry(self, op, state):
|
|
"""(host, instance, message) of the Logs row written with each end of
|
|
the job (in the same transaction); its host and instance also go on
|
|
the row of a resolution. None: no row."""
|
|
return None
|
|
|
|
|
|
class Context:
|
|
def __init__(self, op, worker):
|
|
self.op = op
|
|
self.worker = worker
|
|
self.effects_started = False
|
|
|
|
def _write(self, **fields):
|
|
with fenced(self.worker.identity):
|
|
if not Operation.objects.filter(pk=self.op.pk, worker=self.worker.identity).update(**fields):
|
|
raise LeaseLost("the job is no longer this worker's")
|
|
for name, value in fields.items():
|
|
setattr(self.op, name, value)
|
|
|
|
def record_intent(self, **values):
|
|
"""Commit what run is about to change; the first libvirt effect may
|
|
follow only after this returns."""
|
|
self._write(intent={**self.op.intent, **values}, effects_started=True)
|
|
self.effects_started = True
|
|
|
|
def set_phase(self, text):
|
|
self._write(phase=utils.safe_text(text)[:255])
|
|
|
|
|
|
class SubmissionConflict(Exception):
|
|
"""A submission key reused for another request."""
|
|
|
|
|
|
def _existing(user, key, kind_name, params):
|
|
op = Operation.objects.filter(user=user, submission_key=key).first()
|
|
if op is None:
|
|
return None
|
|
if op.created_at < timezone.now() - SUBMISSION_KEY_TTL:
|
|
Operation.objects.filter(pk=op.pk).update(submission_key=f"expired:{op.pk.hex}")
|
|
return None
|
|
if op.kind != kind_name or op.params != params:
|
|
raise SubmissionConflict("This form was already submitted for another request; reload the page")
|
|
return op
|
|
|
|
|
|
def submit(kind_name, user, raw_params, submission_key):
|
|
if not KEY_RE.fullmatch(submission_key or ""):
|
|
raise OperationError("The form has no valid submission key; reload the page")
|
|
kind = get_kind(kind_name)
|
|
params = kind.normalize(raw_params)
|
|
utils.check_params(params)
|
|
if (op := _existing(user, submission_key, kind.name, params)) is not None:
|
|
return op
|
|
lock = f"submission:{user.pk}:{submission_key}"
|
|
try:
|
|
kind.validate(user, params)
|
|
plan = kind.plan(user, params)
|
|
except Exception:
|
|
# a concurrent duplicate may have been admitted (and even run) meanwhile
|
|
with transaction.atomic():
|
|
claims.serialize([lock])
|
|
if (op := _existing(user, submission_key, kind.name, params)) is not None:
|
|
return op
|
|
raise
|
|
op = Operation(
|
|
kind=kind.name, user=user, username=user.username, submission_key=submission_key,
|
|
resource=plan.resource, resource_label=plan.label[:255], params=params, intent=plan.intent,
|
|
time_limit=kind.time_limit,
|
|
)
|
|
if kind.runs_in_request:
|
|
now = timezone.now()
|
|
op.state, op.started_at, op.heartbeat_at, op.worker = Operation.RUNNING, now, now, Operation.REQUEST
|
|
return claims.admit_job(
|
|
op, plan.claims,
|
|
existing=lambda: _existing(user, submission_key, kind.name, params),
|
|
extra_locks=[lock],
|
|
)
|
|
|
|
|
|
def cancel(op_id, user):
|
|
with transaction.atomic():
|
|
op = Operation.objects.select_for_update().get(pk=op_id)
|
|
if not utils.may_see(op, user):
|
|
raise PermissionDenied
|
|
# read before the update below; a repeated cancel retries the request job's stop
|
|
request_job = op.in_request and op.state in (Operation.RUNNING, Operation.CANCELLING)
|
|
if op.state == Operation.QUEUED:
|
|
Operation.objects.filter(pk=op.pk).update(state=Operation.CANCELLED, finished_at=timezone.now())
|
|
Claim.objects.filter(operation=op).delete()
|
|
if (entry := get_kind(op.kind).log_entry(op, Operation.CANCELLED)) is not None:
|
|
_log(op.username, *entry)
|
|
elif op.state == Operation.RUNNING:
|
|
if not get_kind(op.kind).cancellable:
|
|
raise OperationError("This job cannot be cancelled once it has started")
|
|
Operation.objects.filter(pk=op.pk).update(state=Operation.CANCELLING, cancel_requested=True)
|
|
elif op.state != Operation.CANCELLING:
|
|
raise OperationError("The job has already ended")
|
|
op.refresh_from_db()
|
|
if request_job: # no worker signals it: the request stops it
|
|
get_kind(op.kind).cancel(op)
|
|
op.refresh_from_db()
|
|
return op
|
|
|
|
|
|
def _log(user, host, instance, message):
|
|
Logs.objects.create(user=str(user)[:50], host=str(host)[:50], instance=str(instance)[:50],
|
|
message=utils.safe_text(message)[:255])
|
|
|
|
|
|
def finish(op, outcome, *, worker=None):
|
|
"""Terminal state, claims and history together, once. With a worker,
|
|
only while it holds the lease and owns the job. Uncertain keeps the claims."""
|
|
kind = get_kind(op.kind)
|
|
error = utils.safe_text(outcome.error)
|
|
with fenced(worker.identity) if worker is not None else transaction.atomic():
|
|
q = Operation.objects.filter(pk=op.pk, state__in=Operation.ACTIVE)
|
|
if worker is not None:
|
|
q = q.filter(worker=worker.identity)
|
|
# success clears the phase and completes the progress; another end keeps the last ones
|
|
done = outcome.state == Operation.SUCCEEDED
|
|
if not q.update(state=outcome.state, result=outcome.result, error=error,
|
|
phase=Value(outcome.phase[:255]) if outcome.phase else ("" if done else F("phase")),
|
|
progress_done=Coalesce(F("progress_total"), F("progress_done")) if done else F("progress_done"),
|
|
finished_at=timezone.now()):
|
|
return False
|
|
if outcome.state != Operation.UNCERTAIN:
|
|
Claim.objects.filter(operation=op).delete()
|
|
if (entry := kind.log_entry(op, outcome.state)) is not None:
|
|
_log(op.username, *entry)
|
|
logger.info("job %s %s: %s", op.pk, op.kind, outcome.state)
|
|
op.refresh_from_db()
|
|
return True
|
|
|
|
|
|
def resolve(op_id, user, outcome_state, note):
|
|
if not user.is_superuser:
|
|
raise PermissionDenied
|
|
if outcome_state not in (Operation.SUCCEEDED, Operation.FAILED):
|
|
raise OperationError("Choose succeeded or failed")
|
|
note = note.strip()
|
|
if not note:
|
|
raise OperationError("Write a note on what was checked")
|
|
with transaction.atomic():
|
|
op = Operation.objects.select_for_update().get(pk=op_id)
|
|
if op.state != Operation.UNCERTAIN:
|
|
raise OperationError("Only an uncertain job can be resolved")
|
|
Operation.objects.filter(pk=op.pk).update(state=outcome_state, resolution=outcome_state,
|
|
resolution_note=note, resolved_by=user, resolved_at=timezone.now())
|
|
Claim.objects.filter(operation=op).delete()
|
|
# the host and VM columns of the kind's own entries, so the VM's log shows it
|
|
entry = get_kind(op.kind).log_entry(op, outcome_state)
|
|
host, instance = entry[:2] if entry is not None else ("-", op.resource_label)
|
|
_log(user.username, host, instance, f"Job {op.pk} resolved as {outcome_state}: {note}")
|
|
op.refresh_from_db()
|
|
return op
|
|
|
|
|
|
def remember_job(request, op):
|
|
"""The resource page's panel shows this job even if it ends before the first poll."""
|
|
entries = request.session.get("operations_shown", [])
|
|
request.session["operations_shown"] = [*entries, {"resource": op.resource, "id": str(op.pk)}][-20:]
|
|
|
|
|
|
def take_remembered(request, resource):
|
|
"""The ids remembered for resource; the other resources' entries stay."""
|
|
entries = request.session.get("operations_shown", [])
|
|
taken = [e["id"] for e in entries if e["resource"] == resource]
|
|
if taken: # untouched otherwise: no session write on every page view
|
|
request.session["operations_shown"] = [e for e in entries if e["resource"] != resource]
|
|
return taken
|