refactor(computes): the lock helpers take claims instead of file locks

operations/claims.py

@@ -5,6 +5,7 @@ claim conflicts; otherwise it is refused at once (Busy)."""
 import os
 import socket
 import threading
+from contextlib import contextmanager
 from datetime import timedelta
 
 from django.db import connection, transaction
@@ -31,6 +32,10 @@ def compute_key(compute_id):
     return f"compute:{compute_id}"
 
 
+def reconcile_key(compute_id):
+    return f"reconcile:{compute_id}"
+
+
 def instance_key(uuid):
     return f"instance:{uuid}"
 
@@ -90,3 +95,51 @@ def admit_job(op, wanted, existing, extra_locks=()):
         op.save()
         Claim.objects.bulk_create([Claim(operation=op, resource=k, mode=m) for k, m in sorted(wanted.items())])
         return op
+
+
+_local = threading.local()
+
+
+def _held():
+    if not hasattr(_local, "held"):
+        _local.held = {}
+    return _local.held
+
+
+@contextmanager
+def hold(wanted):
+    """Transient claims for the block. Keys this thread holds are reused;
+    exclusive covers shared; shared cannot become exclusive inside its block."""
+    held = _held()
+    new = {}
+    for key, mode in wanted.items():
+        if key in held:
+            if held[key] == Claim.SHARED and mode == Claim.EXCLUSIVE:
+                raise RuntimeError(f"Cannot upgrade a shared claim on {key}")
+            continue
+        new[key] = mode
+    if not new:
+        yield
+        return
+    ids = admit(new, holder=holder_id())
+    held.update(new)
+    try:
+        yield
+    finally:
+        for key in new:
+            held.pop(key, None)
+        Claim.objects.filter(pk__in=ids).delete()
+
+
+@contextmanager
+def adopted(op):
+    """The job thread holds its job's claims, so code it calls that takes a
+    lock helper does not refuse its own job."""
+    held = _held()
+    own = dict(Claim.objects.filter(operation=op).values_list("resource", "mode"))
+    held.update(own)
+    try:
+        yield
+    finally:
+        for key in own:
+            held.pop(key, None)

operations/jobs.py

@@ -0,0 +1,12 @@
+"""Kinds of jobs by name (filled by each app's jobs.py via register)."""
+
+KINDS = {}
+
+
+def get_kind(name):
+    return KINDS[name]
+
+
+def label_of(name):
+    kind = KINDS.get(name)
+    return kind.label if kind is not None else name

operations/tests/test_hold.py

@@ -0,0 +1,68 @@
+"""Short operations hold transient claims for their duration; a thread may
+nest them; a job thread holds its job's claims."""
+
+import threading
+
+from django.contrib.auth import get_user_model
+from django.db import connection
+from django.test import TransactionTestCase
+
+from operations import claims
+from operations.models import Claim, Operation
+
+S, X = Claim.SHARED, Claim.EXCLUSIVE
+
+
+class HoldTestCase(TransactionTestCase):
+    def test_claims_live_for_the_block(self):
+        with claims.hold({"instance:u1": X}):
+            self.assertTrue(Claim.objects.filter(resource="instance:u1").exists())
+        self.assertFalse(Claim.objects.exists())
+
+    def test_another_thread_is_refused_at_once(self):
+        inside, release, seen = threading.Event(), threading.Event(), []
+
+        def holder():
+            try:
+                with claims.hold({"instance:u1": X}):
+                    inside.set()
+                    release.wait(10)
+            finally:
+                connection.close()
+
+        t = threading.Thread(target=holder)
+        t.start()
+        self.assertTrue(inside.wait(10))
+        try:
+            with claims.hold({"instance:u1": X}):
+                pass
+        except claims.Busy as busy:
+            seen.append(busy.blocker)
+        finally:
+            release.set()
+            t.join(10)
+        self.assertEqual(seen, [None])
+
+    def test_nesting_in_one_thread_reuses_held_claims(self):
+        with claims.hold({"compute:1": X}):
+            with claims.hold({"compute:1": S, "instance:u1": X}):
+                self.assertEqual(Claim.objects.count(), 2)
+            self.assertEqual(Claim.objects.count(), 1)
+
+    def test_a_shared_claim_cannot_be_upgraded(self):
+        with claims.hold({"compute:1": S}):
+            with self.assertRaisesRegex(RuntimeError, "upgrade"):
+                with claims.hold({"compute:1": X}):
+                    pass
+
+    def test_a_job_thread_holds_its_own_claims(self):
+        user = get_user_model().objects.create_user("hold_user", "h@example.com", "pw")
+        op = Operation(kind="test.noop", user=user, username="hold_user", submission_key="k",
+                       resource="instance:u1", resource_label="c/vm")
+        claims.admit_job(op, {"compute:1": S, "instance:u1": X}, existing=lambda: None)
+        with claims.adopted(op):
+            with claims.hold({"compute:1": S, "instance:u1": X}):
+                pass
+        with self.assertRaises(claims.Busy):
+            with claims.hold({"instance:u1": X}):
+                pass

operations/utils.py

@@ -1,7 +1,15 @@
-"""Secrets never reach a job record, a response or a log line."""
+"""Secrets never reach a job record, a response or a log line. What a refused
+request says about the job holding the resource: details for its owner and
+superusers, a minimal summary for everyone else."""
 
 import re
 
+from django.utils import timezone
+from django.utils.timesince import timesince
+from django.utils.translation import gettext as _
+
+from . import jobs
+
 _NAMES = r"(?:pass|secret|token|key)\w*"
 _PATTERNS = [
     (re.compile(r"(://[^:/@\s]+:)[^@\s]+@"), r"\1***@"),
@@ -30,3 +38,25 @@ def check_params(params):
     elif isinstance(params, (list, tuple)):
         for value in params:
             check_params(value)
+
+
+def may_see(op, user):
+    return user.is_superuser or (op.user_id is not None and op.user_id == user.pk)
+
+
+def summary(op, user):
+    base = {"kind": op.kind, "label": str(_(jobs.label_of(op.kind))), "state": op.state, "since": op.created_at}
+    if may_see(op, user):
+        base.update({"id": str(op.id), "user": op.username})
+    return base
+
+
+def busy_message(busy, user):
+    op = busy.blocker
+    if op is None:
+        return _("A short operation is in progress on this resource; try again in a moment.")
+    values = {"label": _(jobs.label_of(op.kind)), "state": op.state, "since": timesince(op.created_at, timezone.now())}
+    if may_see(op, user):
+        return _("%(label)s is %(state)s on this resource (started by %(user)s, %(since)s ago); "
+                 "see the Jobs page.") % {**values, "user": op.username}
+    return _("%(label)s is %(state)s on this resource (since %(since)s).") % values