mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-08 11:16:47 +00:00
f2138e54d8
Request jobs (an upload's chunks) run in gunicorn: their kind is admitted running with the request as its worker, never claimed or recovered by the worker, cancelled in the request, and asked by every housekeeping pass to end its jobs that went idle. The panel can follow a job the page just started. A job a request runs does not wait for the worker, so it does not raise the worker-unavailable warning on the Jobs page or in the panel.
105 lines
4.7 KiB
Python
105 lines
4.7 KiB
Python
from datetime import timedelta
|
|
from types import SimpleNamespace
|
|
|
|
from django.contrib.auth import get_user_model
|
|
from django.test import TestCase
|
|
from django.utils import timezone
|
|
|
|
from operations import jobs
|
|
from operations.jobs import KINDS, Context, get_kind, label_of
|
|
from operations.models import TAKEOVER_AFTER, Claim, LeaseLost, Operation, WorkerLease, acquire, fenced, renew
|
|
from operations.tests import fakes
|
|
|
|
|
|
class RegistryTestCase(TestCase):
|
|
def test_kinds_register_by_name(self):
|
|
self.assertIs(get_kind("test.noop"), KINDS["test.noop"])
|
|
self.assertEqual((label_of("test.noop"), label_of("unknown.kind")), ("No-op", "unknown.kind"))
|
|
|
|
|
|
class LeaseTestCase(TestCase):
|
|
def test_the_first_worker_takes_the_lease_and_renews_it(self):
|
|
self.assertTrue(acquire("h1:1:a", "h1"))
|
|
self.assertTrue(renew("h1:1:a"))
|
|
self.assertFalse(renew("h1:2:b"))
|
|
|
|
def test_the_same_host_takes_over_at_once(self):
|
|
acquire("h1:1:a", "h1")
|
|
self.assertTrue(acquire("h1:2:b", "h1")) # the caller holds the host's flock
|
|
self.assertFalse(renew("h1:1:a"))
|
|
|
|
def test_another_host_waits_until_the_lease_is_stale(self):
|
|
acquire("h1:1:a", "h1")
|
|
self.assertFalse(acquire("h2:1:c", "h2"))
|
|
WorkerLease.objects.update(heartbeat_at=timezone.now() - TAKEOVER_AFTER - timedelta(seconds=1))
|
|
self.assertTrue(acquire("h2:1:c", "h2"))
|
|
|
|
def test_fenced_refuses_a_worker_without_the_lease(self):
|
|
acquire("h1:1:a", "h1")
|
|
with fenced("h1:1:a"):
|
|
pass
|
|
with self.assertRaises(LeaseLost):
|
|
with fenced("h9:9:z"):
|
|
pass
|
|
|
|
|
|
class ContextTestCase(TestCase):
|
|
def setUp(self):
|
|
user = get_user_model().objects.create_user("ctx_user", "x@example.com", "pw")
|
|
acquire("h1:1:a", "h1")
|
|
self.worker = SimpleNamespace(identity="h1:1:a")
|
|
self.op = Operation.objects.create(kind="test.noop", user=user, username="ctx_user", submission_key="k",
|
|
resource="r", resource_label="r", state=Operation.RUNNING,
|
|
worker="h1:1:a")
|
|
|
|
def test_intent_is_committed_at_once(self):
|
|
ctx = Context(self.op, self.worker)
|
|
self.assertFalse(ctx.effects_started)
|
|
ctx.record_intent(path="/pool/x.qcow2")
|
|
self.op.refresh_from_db()
|
|
self.assertTrue(ctx.effects_started and self.op.effects_started)
|
|
self.assertEqual(self.op.intent, {"path": "/pool/x.qcow2"})
|
|
|
|
def test_a_worker_without_the_lease_records_nothing(self):
|
|
WorkerLease.objects.update(holder="h2:1:b", host="h2")
|
|
ctx = Context(self.op, self.worker)
|
|
with self.assertRaises(LeaseLost):
|
|
ctx.record_intent(path="/pool/x.qcow2")
|
|
self.op.refresh_from_db()
|
|
self.assertEqual((self.op.intent, self.op.effects_started), ({}, False))
|
|
self.assertFalse(ctx.effects_started)
|
|
|
|
|
|
class RequestJobTestCase(TestCase):
|
|
def setUp(self):
|
|
fakes.reset()
|
|
self.user = get_user_model().objects.create_user("req_user", "q@example.com", "pw")
|
|
|
|
def test_a_request_job_is_admitted_running_in_the_request(self):
|
|
op = jobs.submit("test.request", self.user, {}, "key-00000001")
|
|
op.refresh_from_db()
|
|
self.assertEqual((op.state, op.worker, op.in_request), (Operation.RUNNING, Operation.REQUEST, True))
|
|
self.assertIsNotNone(op.started_at)
|
|
self.assertEqual(op.heartbeat_at, op.started_at)
|
|
self.assertEqual(list(op.claims.values_list("resource", "mode")), [("instance:u1", Claim.EXCLUSIVE)])
|
|
|
|
def test_the_same_submission_returns_the_running_request_job(self):
|
|
op = jobs.submit("test.request", self.user, {}, "key-00000001")
|
|
again = jobs.submit("test.request", self.user, {}, "key-00000001")
|
|
self.assertEqual((again.pk, Operation.objects.count()), (op.pk, 1))
|
|
|
|
def test_cancelling_a_request_job_runs_its_cancel_in_the_request(self):
|
|
op = jobs.submit("test.request", self.user, {}, "key-00000001")
|
|
op = jobs.cancel(op.pk, self.user)
|
|
self.assertEqual((op.state, op.cancel_requested), (Operation.CANCELLING, True))
|
|
self.assertEqual(fakes.CANCELLED, [str(op.pk)])
|
|
jobs.cancel(op.pk, self.user) # its stop did not end it: a repeated cancel retries
|
|
self.assertEqual(fakes.CANCELLED, [str(op.pk)] * 2)
|
|
|
|
def test_cancelling_a_workers_job_leaves_its_cancel_to_the_worker(self):
|
|
op = jobs.submit("test.scripted", self.user, {}, "key-00000001")
|
|
Operation.objects.filter(pk=op.pk).update(state=Operation.RUNNING, worker="h1:1:a")
|
|
op = jobs.cancel(op.pk, self.user)
|
|
self.assertEqual(op.state, Operation.CANCELLING)
|
|
jobs.cancel(op.pk, self.user)
|
|
self.assertEqual(fakes.CANCELLED, [])
|