feat(operations): kind contract, job context and worker lease

operations/tests/fakes.py

@@ -0,0 +1,72 @@
+"""Kinds used by the operations tests; registered on import. test.scripted
+takes its behaviour from params and from barriers that tests install before
+submitting (keyed by submission key)."""
+
+from operations.jobs import Cancelled, Kind, Outcome, Plan, register
+from operations.models import Claim, Operation
+
+BARRIERS = {}  # submission key -> threading.Event the run waits on
+CANCELLED = []
+
+
+def reset():
+    BARRIERS.clear()
+    CANCELLED.clear()
+
+
+@register
+class NoopKind(Kind):
+    name = "test.noop"
+    label = "No-op"
+
+    def normalize(self, raw):
+        return {"uuid": str(raw.get("uuid", "u1"))}
+
+    def plan(self, user, params):
+        key = "instance:" + params["uuid"]
+        return Plan(claims={key: Claim.EXCLUSIVE}, resource=key, label="c/vm")
+
+    def run(self, op, ctx):
+        ctx.record_intent(step="effect")
+        return {"done": True}
+
+    def reconcile(self, op):
+        return Outcome(Operation.SUCCEEDED, result={"recovered": True})
+
+    def log_entry(self, op, state):
+        return ("c", "vm", f"No-op {state}")
+
+
+@register
+class ScriptedKind(NoopKind):
+    """params: fail_before, fail_after, outcome (what reconcile returns)."""
+
+    name = "test.scripted"
+    label = "Scripted"
+    cancellable = True
+
+    def normalize(self, raw):
+        return {"uuid": str(raw.get("uuid", "u1")), "fail_before": bool(raw.get("fail_before")),
+                "fail_after": bool(raw.get("fail_after")), "outcome": str(raw.get("outcome", "uncertain"))}
+
+    def run(self, op, ctx):
+        if op.params["fail_before"]:
+            raise RuntimeError("refused before effects")
+        ctx.record_intent(step="effect")
+        if op.params["fail_after"]:
+            raise RuntimeError("failed after effects")
+        barrier = BARRIERS.get(op.submission_key)
+        if barrier is not None:
+            barrier.wait(30)
+            if str(op.id) in CANCELLED:
+                raise Cancelled()
+        return {"ran": True}
+
+    def cancel(self, op):
+        CANCELLED.append(str(op.id))
+        barrier = BARRIERS.get(op.submission_key)
+        if barrier is not None:
+            barrier.set()
+
+    def reconcile(self, op):
+        return Outcome(op.params["outcome"], error="scripted")

operations/tests/test_kinds.py

@@ -0,0 +1,69 @@
+from datetime import timedelta
+from types import SimpleNamespace
+
+from django.contrib.auth import get_user_model
+from django.test import TestCase
+from django.utils import timezone
+
+from operations.jobs import KINDS, Context, get_kind, label_of
+from operations.models import TAKEOVER_AFTER, LeaseLost, Operation, WorkerLease, acquire, fenced, renew
+from operations.tests import fakes  # noqa: F401
+
+
+class RegistryTestCase(TestCase):
+    def test_kinds_register_by_name(self):
+        self.assertIs(get_kind("test.noop"), KINDS["test.noop"])
+        self.assertEqual((label_of("test.noop"), label_of("unknown.kind")), ("No-op", "unknown.kind"))
+
+
+class LeaseTestCase(TestCase):
+    def test_the_first_worker_takes_the_lease_and_renews_it(self):
+        self.assertTrue(acquire("h1:1:a", "h1"))
+        self.assertTrue(renew("h1:1:a"))
+        self.assertFalse(renew("h1:2:b"))
+
+    def test_the_same_host_takes_over_at_once(self):
+        acquire("h1:1:a", "h1")
+        self.assertTrue(acquire("h1:2:b", "h1"))  # the caller holds the host's flock
+        self.assertFalse(renew("h1:1:a"))
+
+    def test_another_host_waits_until_the_lease_is_stale(self):
+        acquire("h1:1:a", "h1")
+        self.assertFalse(acquire("h2:1:c", "h2"))
+        WorkerLease.objects.update(heartbeat_at=timezone.now() - TAKEOVER_AFTER - timedelta(seconds=1))
+        self.assertTrue(acquire("h2:1:c", "h2"))
+
+    def test_fenced_refuses_a_worker_without_the_lease(self):
+        acquire("h1:1:a", "h1")
+        with fenced("h1:1:a"):
+            pass
+        with self.assertRaises(LeaseLost):
+            with fenced("h9:9:z"):
+                pass
+
+
+class ContextTestCase(TestCase):
+    def setUp(self):
+        user = get_user_model().objects.create_user("ctx_user", "x@example.com", "pw")
+        acquire("h1:1:a", "h1")
+        self.worker = SimpleNamespace(identity="h1:1:a")
+        self.op = Operation.objects.create(kind="test.noop", user=user, username="ctx_user", submission_key="k",
+                                           resource="r", resource_label="r", state=Operation.RUNNING,
+                                           worker="h1:1:a")
+
+    def test_intent_is_committed_at_once(self):
+        ctx = Context(self.op, self.worker)
+        self.assertFalse(ctx.effects_started)
+        ctx.record_intent(path="/pool/x.qcow2")
+        self.op.refresh_from_db()
+        self.assertTrue(ctx.effects_started and self.op.effects_started)
+        self.assertEqual(self.op.intent, {"path": "/pool/x.qcow2"})
+
+    def test_a_worker_without_the_lease_records_nothing(self):
+        WorkerLease.objects.update(holder="h2:1:b", host="h2")
+        ctx = Context(self.op, self.worker)
+        with self.assertRaises(LeaseLost):
+            ctx.record_intent(path="/pool/x.qcow2")
+        self.op.refresh_from_db()
+        self.assertEqual((self.op.intent, self.op.effects_started), ({}, False))
+        self.assertFalse(ctx.effects_started)