"""Kinds used by the operations tests; registered on import. test.scripted takes its behaviour from params and from barriers that tests install before submitting (keyed by submission key). start_worker and stop_worker run a worker's loop in a thread.""" import threading from datetime import timedelta from django.db import connection from django.utils import timezone from operations.jobs import Cancelled, Kind, Outcome, Plan, register from operations.models import Claim, Operation, acquire BARRIERS = {} # submission key -> threading.Event the run waits on CANCELLED = [] EXPIRED = [] def reset(): BARRIERS.clear() CANCELLED.clear() EXPIRED.clear() STUCK.clear() HOST_CALLS.clear() def start_worker(worker): """The loop of Worker.run in a thread, without its process parts.""" acquire(worker.identity, worker.host) worker.recovery.schedule_orphans() def loop(): try: while not worker.stopping.is_set(): worker.tick_once() worker.stopping.wait(worker.tick) finally: connection.close() thread = threading.Thread(target=loop, daemon=True) thread.start() return thread def stop_worker(worker, thread): worker.stopping.set() thread.join(10) for pool in (worker.pool, worker.control, worker.progress_pool): pool.shutdown(wait=True, cancel_futures=True) @register class NoopKind(Kind): name = "test.noop" label = "No-op" def normalize(self, raw): return {"uuid": str(raw.get("uuid", "u1"))} def plan(self, user, params): key = "instance:" + params["uuid"] return Plan(claims={key: Claim.EXCLUSIVE}, resource=key, label="c/vm") def run(self, op, ctx): ctx.record_intent(step="effect") return {"done": True} def reconcile(self, op): return Outcome(Operation.SUCCEEDED, result={"recovered": True}) def log_entry(self, op, state): return ("c", "vm", op.params["uuid"], f"No-op {state}") @register class ScriptedKind(NoopKind): """params: fail_before, fail_after, outcome (what reconcile returns).""" name = "test.scripted" label = "Scripted" cancellable = True def normalize(self, raw): return {"uuid": str(raw.get("uuid", "u1")), "fail_before": bool(raw.get("fail_before")), "fail_after": bool(raw.get("fail_after")), "outcome": str(raw.get("outcome", "uncertain"))} def run(self, op, ctx): if op.params["fail_before"]: raise RuntimeError("refused before effects") ctx.record_intent(step="effect") if op.params["fail_after"]: raise RuntimeError("failed after effects") barrier = BARRIERS.get(op.submission_key) if barrier is not None: barrier.wait(30) if str(op.id) in CANCELLED: raise Cancelled() return {"ran": True} def cancel(self, op): CANCELLED.append(str(op.id)) barrier = BARRIERS.get(op.submission_key) if barrier is not None: barrier.set() def reconcile(self, op): return Outcome(op.params["outcome"], error="scripted") @register class RequestKind(NoopKind): """A job a request runs, idle after 60 s; params: fail_expire.""" name = "test.request" label = "Request" cancellable = True runs_in_request = True def normalize(self, raw): return {"uuid": str(raw.get("uuid", "u1")), "fail_expire": bool(raw.get("fail_expire"))} def cancel(self, op): CANCELLED.append(str(op.id)) def expire(self, op): if op.heartbeat_at > timezone.now() - timedelta(seconds=60): return if op.params["fail_expire"]: raise RuntimeError("expire failed") EXPIRED.append(str(op.id)) STUCK = {} # compute id -> threading.Event: test.host's libvirt calls on it wait until it is set HOST_CALLS = [] # (call, compute id) as test.host's progress and cancel begin @register class HostKind(NoopKind): """A job on compute params["host"], whose calls wait while STUCK holds it, as calls to a host that is up but does not answer do.""" name = "test.host" label = "Host" cancellable = True def normalize(self, raw): return {"host": int(raw["host"]), "n": int(raw["n"])} def plan(self, user, params): key = f"instance:host-{params['host']}-{params['n']}" return Plan(claims={f"compute:{params['host']}": Claim.SHARED, key: Claim.EXCLUSIVE}, resource=key, label="c/vm") def _call(self, name, op): HOST_CALLS.append((name, op.params["host"])) if (stuck := STUCK.get(op.params["host"])) is not None: stuck.wait(30) def run(self, op, ctx): ctx.record_intent(step="effect") self._call("run", op) if (release := BARRIERS.get(op.submission_key)) is not None: release.wait(30) return {} def progress(self, op): self._call("progress", op) return (1, 2) def cancel(self, op): self._call("cancel", op) if (release := BARRIERS.get(op.submission_key)) is not None: release.set()