1
0
Fork 0
mirror of https://github.com/retspen/webvirtcloud synced 2026-11-11 12:46:49 +00:00

fix(instances): a migration that does not finish in time is cancelled, not cut off

A VM that changes its memory faster than the network carries it never
finishes a live migration without auto converge: measured with a guest
rewriting about 1.2 GB/s over an 8.5 Gbit/s link. The migration then
ran until gunicorn ended the request after 600 s. libvirt cancelled it
when the request went away and the VM stayed on the source, but the
user got an error page and nothing was logged.

The app now cancels a migration that has not finished after 540 s
(MIGRATION_TIME_LIMIT, below the 600 s of gunicorn.conf.py; a test
keeps it below). The failure is reported and logged like any other:
the migration was cancelled, the VM stays on the source host, and a
VM like that needs auto converge. A timer that fires as the migration
ends finishes before the compute locks are released, and an error
the cancel did not cause keeps its own message.
This commit is contained in:
catborise 2026-10-06 16:56:22 +03:00
parent 1016f745c2
commit 043c24a208
4 changed files with 120 additions and 5 deletions

View file

@ -2,17 +2,20 @@
destination with the app's connection, re-reads the VM under the compute
locks, and records a finished migration even when autostart fails."""
import re
from pathlib import Path
from unittest.mock import MagicMock, patch
from accounts.models import UserInstance
from computes.models import Compute
from django.conf import settings
from django.contrib.auth import get_user_model
from django.db import DatabaseError
from django.test import TestCase
from logs.models import Logs
from instances.models import Instance
from instances.utils import migrate_instance
from instances.utils import MIGRATION_TIME_LIMIT, migrate_instance
from vrtManager import util
@ -85,3 +88,10 @@ class MigrationBookkeepingTests(TestCase):
found.refresh_from_db()
self.assertTrue(found.is_template)
self.assertEqual(UserInstance.objects.get(user=owner).instance_id, found.id)
def test_a_migration_is_cancelled_before_the_web_server_ends_the_request(self):
self.migrate()
self.assertEqual(self.moveto.call_args.kwargs["timeout"], MIGRATION_TIME_LIMIT)
gunicorn = (Path(settings.BASE_DIR) / "gunicorn.conf.py").read_text()
request_timeout = int(re.search(r"^timeout = (\d+)", gunicorn, re.M).group(1))
self.assertLess(MIGRATION_TIME_LIMIT, request_timeout - 30) # time to answer and log

View file

@ -150,6 +150,11 @@ def get_network_tuple(network_source_str):
return network_source_pack[0], "net"
# A migration that has not finished by then is cancelled: below the 600 s
# after which gunicorn (gunicorn.conf.py) ends the request.
MIGRATION_TIME_LIMIT = 540
def migrate_instance(
new_compute,
instance,
@ -194,6 +199,7 @@ def migrate_instance(
autoconverge,
compress,
uri=new_compute.migration_uri,
timeout=MIGRATION_TIME_LIMIT,
)
except Exception as error:
msg = _("Instance migration to %(target)s failed: %(error)s") % {