1
0
Fork 0
mirror of https://github.com/retspen/webvirtcloud synced 2026-11-11 12:46:49 +00:00

fix(gstfsd): find the guest OS by inspection, answer once, and stop a change at its deadline

gstfsd mounted every partition it listed: a root on LVM was never found,
a disk with two OS roots got the change twice and two concatenated
replies, and no root, an error or a request longer than 1024 bytes ended
the connection without a reply. It now lets libguestfs inspect the guest,
mounts its one OS the way the guest does (a broken filesystem off the
changed path is left out), and sends exactly one reply, keeping the first
error. The root entry is found on any line of /etc/shadow and must be
unique; a key needs no shadow file. A request is at most 64 KiB. Each
change runs in a worker process with a directly started appliance; both
are killed when the request is not done 100 s after gstfsd accepted it
(GSTFSD_DEADLINE may only lower that), before the panel's 120 s timeout
releases the VM, or when the panel's connection ends, and the reply waits
until they are gone, so nothing touches the disk once the panel has given
up. A worker that dies says why. The panel
connects over IPv4 or IPv6, sends the whole request and reads the reply
until gstfsd closes.
This commit is contained in:
catborise 2026-10-07 22:53:48 +03:00
parent bb6ad06697
commit 5ee13f305d
5 changed files with 610 additions and 92 deletions

View file

@ -10,7 +10,8 @@ from pathlib import Path
GSTFSD = Path(__file__).resolve().parent.parent / "conf" / "daemon" / "gstfsd"
# A stand-in for libguestfs: one partition whose files live in a JSON file.
# A stand-in for libguestfs: the guest's operating systems ("roots", each with
# its mountpoints), its files, and what is mounted live in a JSON file.
FAKE_GUESTFS = '''
import json, os
STATE = os.environ["FAKE_GUESTFS_STATE"]
@ -21,22 +22,61 @@ def _save(state):
with open(STATE, "w") as f:
json.dump(state, f)
class GuestFS:
def __init__(self, **kwargs): pass
def add_domain(self, name): pass
def launch(self): pass
def list_partitions(self): return ["/dev/sda1"]
def mount(self, part, mountpoint): pass
def umount(self, part): pass
def shutdown(self): pass
def close(self): pass
def is_file(self, path): return path in _load()["files"]
def is_dir(self, path): return path in _load()["dirs"]
def cat(self, path): return _load()["files"][path]
def __init__(self, **kwargs):
if _load().get("crash"):
import sys
sys.stderr.write("guestfs: the appliance crashed\\n")
os._exit(3)
self.launched = False
def add_domain(self, name, **kwargs):
if _load().get("fail_add"):
raise RuntimeError("no domain named " + name)
def launch(self):
import time
if _load().get("appliance"):
# a stand-in for the appliance: a child process, like qemu
import subprocess
child = subprocess.Popen(["sleep", "60"])
state = _load(); state["appliance_pid"] = child.pid; _save(state)
time.sleep(_load().get("launch_seconds", 0))
state = _load(); state["backend"] = os.environ.get("LIBGUESTFS_BACKEND"); _save(state)
self.launched = True
def inspect_os(self): return list(_load()["roots"])
def inspect_get_mountpoints(self, root): return _load()["roots"][root]
def mount(self, device, mountpoint):
state = _load()
if device in state.get("broken", []):
raise RuntimeError("mount: wrong fs type on " + device)
# a mountpoint needs its parent filesystem mounted first
if mountpoint != "/" and "/" not in state["mounted"]:
raise RuntimeError("mount: " + mountpoint + " does not exist")
state["mounted"].append(mountpoint); state["history"].append([device, mountpoint]); _save(state)
def umount_all(self):
if not self.launched:
raise RuntimeError("umount_all: call launch before using this function")
state = _load(); state["mounted"] = []; _save(state)
def shutdown(self):
if not self.launched:
raise RuntimeError("shutdown: call launch before using this function")
def close(self):
state = _load(); state["closed"] = True; _save(state)
def _guest(self):
if "/" not in _load()["mounted"]:
raise RuntimeError("the guest root is not mounted")
def is_file(self, path): self._guest(); return path in _load()["files"]
def is_dir(self, path): self._guest(); return path in _load()["dirs"]
def cat(self, path): self._guest(); return _load()["files"][path]
def write(self, path, content):
state = _load(); state["files"][path] = content; _save(state)
self._guest()
import time
time.sleep(_load().get("write_seconds", 0))
state = _load()
if state.get("fail_write"):
raise RuntimeError("write: read-only file system")
state["files"][path] = content; state["writes"] += 1; _save(state)
def mkdir(self, path):
state = _load(); state["dirs"].append(path); _save(state)
def chmod(self, mode, path): pass
self._guest(); state = _load(); state["dirs"].append(path); _save(state)
def chmod(self, mode, path): self._guest()
'''
@ -55,8 +95,13 @@ class GstfsdTestCase(unittest.TestCase):
Path(self.tmp.name, "guestfs.py").write_text(FAKE_GUESTFS)
self.state = Path(self.tmp.name, "state.json")
self.state.write_text(json.dumps({
# one Linux guest with its root on LVM and a separate /boot
"roots": {"/dev/vg/root": {"/": "/dev/vg/root", "/boot": "/dev/sda1"}},
"files": {"/etc/shadow": "root:x:1::::::\n", "/root/.ssh/authorized_keys": "ssh-rsa OLD old@host\n"},
"dirs": ["/root/.ssh"],
"mounted": [],
"history": [],
"writes": 0,
}))
def _env(self, **extra):
@ -67,10 +112,10 @@ class GstfsdTestCase(unittest.TestCase):
**extra,
)
def _start(self, host, port, family=socket.AF_INET):
def _start(self, host, port, family=socket.AF_INET, **env):
proc = subprocess.Popen(
[sys.executable, str(GSTFSD)],
env=self._env(GSTFSD_BIND_HOST=host, GSTFSD_PORT=str(port)),
env=self._env(GSTFSD_BIND_HOST=host, GSTFSD_PORT=str(port), **env),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
@ -93,7 +138,8 @@ class GstfsdTestCase(unittest.TestCase):
s.settimeout(10)
s.connect((host, port))
s.sendall(json.dumps(data).encode())
return json.loads(s.recv(1024))
# everything until gstfsd closes: more than one reply fails to parse
return json.loads(b"".join(iter(lambda: s.recv(4096), b"")))
def test_refuses_non_loopback_bind_without_opt_in(self):
result = subprocess.run(
@ -117,6 +163,16 @@ class GstfsdTestCase(unittest.TestCase):
self.assertEqual(keys.splitlines(), ["ssh-rsa OLD old@host", "ssh-ed25519 NEW new@host"])
@unittest.skipUnless(socket.has_ipv6, "no IPv6")
def test_refuses_a_deadline_the_panel_does_not_wait_for(self):
# the panel gives up after 120 s and releases the VM
proc = subprocess.run(
[sys.executable, str(GSTFSD)],
env=self._env(GSTFSD_PORT=str(free_port()), GSTFSD_DEADLINE="180"),
capture_output=True, text=True, timeout=10,
)
self.assertNotEqual(proc.returncode, 0)
self.assertIn("GSTFSD_DEADLINE", proc.stderr)
def test_listens_on_ipv6_loopback(self):
try:
port = free_port(socket.AF_INET6, "::1")
@ -128,3 +184,262 @@ class GstfsdTestCase(unittest.TestCase):
socket.AF_INET6,
)
self.assertEqual(reply["return"], "success")
def _update_state(self, **values):
state = json.loads(self.state.read_text())
state.update(values)
self.state.write_text(json.dumps(state))
def _serve(self):
port = free_port()
self._start("127.0.0.1", port)
return port
def test_root_password_is_set_on_the_guest_root(self):
reply = self._request("127.0.0.1", self._serve(), {"action": "password", "vname": "vm", "passwd": "$6$new"})
self.assertEqual(reply, {"return": "success"})
state = json.loads(self.state.read_text())
self.assertEqual(state["files"]["/etc/shadow"], "root:$6$new:1::::::\n")
self.assertEqual(state["mounted"], [])
self.assertTrue(state["closed"])
def test_no_or_several_operating_systems_give_one_error(self):
for roots in ({}, {"/dev/sda1": {"/": "/dev/sda1"}, "/dev/sdb1": {"/": "/dev/sdb1"}}):
with self.subTest(roots=list(roots)):
self._update_state(roots=roots)
reply = self._request("127.0.0.1", self._serve(), {"action": "password", "vname": "vm", "passwd": "$6$x"})
self.assertEqual(reply["return"], "error")
self.assertEqual(json.loads(self.state.read_text())["files"]["/etc/shadow"], "root:x:1::::::\n")
def test_a_failed_change_gives_one_error_and_cleans_up(self):
self._update_state(fail_write=True)
reply = self._request("127.0.0.1", self._serve(), {"action": "password", "vname": "vm", "passwd": "$6$x"})
self.assertEqual(reply["return"], "error")
self.assertIn("read-only", reply["message"])
state = json.loads(self.state.read_text())
self.assertEqual(state["mounted"], [])
self.assertTrue(state["closed"])
def test_a_long_key_arrives_whole(self):
# longer than one read of the daemon (4096 bytes)
key = "ssh-rsa " + "A" * 6000 + " long@host"
reply = self._request("127.0.0.1", self._serve(), {"action": "publickey", "vname": "vm", "key": key})
self.assertEqual(reply, {"return": "success"})
keys = json.loads(self.state.read_text())["files"]["/root/.ssh/authorized_keys"]
self.assertEqual(keys.splitlines()[-1], key)
def test_the_root_is_mounted_before_the_filesystems_under_it(self):
self._request("127.0.0.1", self._serve(), {"action": "password", "vname": "vm", "passwd": "$6$x"})
self.assertEqual(
json.loads(self.state.read_text())["history"], [["/dev/vg/root", "/"], ["/dev/sda1", "/boot"]]
)
def test_a_broken_unrelated_filesystem_does_not_block_the_change(self):
self._update_state(broken=["/dev/sda1"]) # /boot
reply = self._request("127.0.0.1", self._serve(), {"action": "password", "vname": "vm", "passwd": "$6$y"})
self.assertEqual(reply, {"return": "success"})
self.assertEqual(json.loads(self.state.read_text())["files"]["/etc/shadow"], "root:$6$y:1::::::\n")
def test_a_broken_filesystem_on_the_path_blocks_the_change(self):
state = json.loads(self.state.read_text())
state["roots"]["/dev/vg/root"]["/root"] = "/dev/vg/home"
state["broken"] = ["/dev/vg/home"]
self.state.write_text(json.dumps(state))
reply = self._request("127.0.0.1", self._serve(), {"action": "publickey", "vname": "vm", "key": "ssh-ed25519 K k@host"})
self.assertEqual(reply["return"], "error")
self.assertEqual(json.loads(self.state.read_text())["writes"], 0)
def test_root_is_found_anywhere_in_shadow(self):
self._update_state(files={"/etc/shadow": "daemon:*:1::::::\nroot:x:1::::::\n"})
reply = self._request("127.0.0.1", self._serve(), {"action": "password", "vname": "vm", "passwd": "$6$z"})
self.assertEqual(reply, {"return": "success"})
self.assertEqual(
json.loads(self.state.read_text())["files"]["/etc/shadow"], "daemon:*:1::::::\nroot:$6$z:1::::::\n"
)
def test_a_shadow_without_root_is_an_error(self):
self._update_state(files={"/etc/shadow": "daemon:*:1::::::\n"})
reply = self._request("127.0.0.1", self._serve(), {"action": "password", "vname": "vm", "passwd": "$6$z"})
self.assertEqual(reply["return"], "error")
self.assertEqual(json.loads(self.state.read_text())["writes"], 0)
def test_an_early_failure_keeps_its_own_message(self):
self._update_state(fail_add=True)
reply = self._request("127.0.0.1", self._serve(), {"action": "password", "vname": "vm", "passwd": "$6$x"})
self.assertEqual(reply["return"], "error")
self.assertIn("no domain named vm", reply["message"])
def test_a_request_in_two_pieces(self):
port = self._serve()
body = json.dumps({"action": "password", "vname": "vm", "passwd": "$6$parts"}).encode()
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.settimeout(10)
s.connect(("127.0.0.1", port))
s.sendall(body[:10])
time.sleep(0.3)
s.sendall(body[10:])
reply = json.loads(b"".join(iter(lambda: s.recv(4096), b"")))
self.assertEqual(reply, {"return": "success"})
def test_a_request_over_the_limit_is_refused(self):
port = self._serve()
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.settimeout(10)
s.connect(("127.0.0.1", port))
s.sendall(b'{"action": "publickey", "vname": "vm", "key": "' + b"A" * 70000 + b'"}')
reply = json.loads(b"".join(iter(lambda: s.recv(4096), b"")))
self.assertEqual(reply["return"], "error")
self.assertEqual(json.loads(self.state.read_text())["writes"], 0)
def test_a_trickling_client_is_cut_off(self):
port = free_port()
self._start("127.0.0.1", port, GSTFSD_DEADLINE="2")
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect(("127.0.0.1", port))
s.settimeout(0.25)
start = time.monotonic()
reply = b""
# a space every quarter second: each read is quick, the request never completes
while not reply and time.monotonic() - start < 8:
try:
s.sendall(b" ")
reply = s.recv(4096)
except socket.timeout:
pass
except OSError:
break
self.assertTrue(reply, "gstfsd kept reading a request that never completes")
self.assertEqual(json.loads(reply)["return"], "error")
def test_a_slow_write_is_stopped_at_the_deadline(self):
# the whole change is bounded, not only its start: a write still
# running at the deadline is killed with the appliance
port = free_port()
self._start("127.0.0.1", port, GSTFSD_DEADLINE="1")
self._update_state(write_seconds=3)
start = time.monotonic()
reply = self._request("127.0.0.1", port, {"action": "password", "vname": "vm", "passwd": "$6$late"})
self.assertLess(time.monotonic() - start, 2.5)
self.assertEqual(reply["return"], "error")
time.sleep(3) # longer than the write would have taken
self.assertEqual(json.loads(self.state.read_text())["writes"], 0)
def test_a_key_needs_no_shadow_file(self):
self._update_state(files={"/root/.ssh/authorized_keys": ""})
reply = self._request("127.0.0.1", self._serve(), {"action": "publickey", "vname": "vm", "key": "ssh-ed25519 K k@host"})
self.assertEqual(reply, {"return": "success"})
def test_a_client_that_leaves_does_not_break_the_service(self):
port = free_port()
proc = self._start("127.0.0.1", port, GSTFSD_DEADLINE="1")
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect(("127.0.0.1", port))
s.sendall(b'{"action": ')
time.sleep(1.5) # gstfsd times the request out and answers a closed socket
reply = self._request("127.0.0.1", port, {"action": "password", "vname": "vm", "passwd": "$6$after"})
self.assertEqual(reply, {"return": "success"})
proc.kill()
self.assertNotIn("Traceback", proc.stderr.read())
def test_nothing_is_written_after_the_deadline(self):
# the panel gives up after its timeout; gstfsd must not write later
port = free_port()
self._start("127.0.0.1", port, GSTFSD_DEADLINE="1")
self._update_state(launch_seconds=2)
reply = self._request("127.0.0.1", port, {"action": "password", "vname": "vm", "passwd": "$6$late"})
self.assertEqual(reply["return"], "error")
state = json.loads(self.state.read_text())
self.assertEqual(state["writes"], 0)
self.assertEqual(state["mounted"], [])
def test_the_appliance_runs_where_the_deadline_can_kill_it(self):
# a libvirt-started appliance is no child of the worker: killing the
# worker's process group would leave it writing to the disks
port = free_port()
self._start("127.0.0.1", port, LIBGUESTFS_BACKEND="libvirt")
reply = self._request("127.0.0.1", port, {"action": "password", "vname": "vm", "passwd": "$6$d"})
self.assertEqual(reply, {"return": "success"})
self.assertEqual(json.loads(self.state.read_text())["backend"], "direct")
def test_reading_the_request_counts_toward_the_deadline(self):
# the panel waits a fixed time from connecting; a slow request leaves
# less time for the change, not more
port = free_port()
self._start("127.0.0.1", port, GSTFSD_DEADLINE="2")
self._update_state(launch_seconds=3)
request = json.dumps({"action": "password", "vname": "vm", "passwd": "$6$slow"}).encode()
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.settimeout(10)
s.connect(("127.0.0.1", port))
start = time.monotonic()
s.sendall(request[:10])
time.sleep(1.5)
s.sendall(request[10:])
reply = json.loads(b"".join(iter(lambda: s.recv(4096), b"")))
self.assertEqual(reply["return"], "error")
self.assertLess(time.monotonic() - start, 2.8)
def test_a_crashed_worker_tells_why(self):
self._update_state(crash=True)
reply = self._request("127.0.0.1", self._serve(), {"action": "password", "vname": "vm", "passwd": "$6$c"})
self.assertEqual(reply["return"], "error")
self.assertIn("the appliance crashed", reply["message"])
def test_a_client_that_leaves_stops_the_change(self):
# the panel releases the VM when its connection breaks: nothing may
# change the disks after that
port = free_port()
self._start("127.0.0.1", port)
self._update_state(launch_seconds=2)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect(("127.0.0.1", port))
s.sendall(json.dumps({"action": "password", "vname": "vm", "passwd": "$6$gone"}).encode())
time.sleep(0.5)
time.sleep(3) # longer than the change would have taken
self.assertEqual(json.loads(self.state.read_text())["writes"], 0)
def test_the_reply_waits_until_the_appliance_is_gone(self):
port = free_port()
self._start("127.0.0.1", port, GSTFSD_DEADLINE="1")
self._update_state(appliance=True, launch_seconds=3)
reply = self._request("127.0.0.1", port, {"action": "password", "vname": "vm", "passwd": "$6$late"})
self.assertEqual(reply["return"], "error")
pid = json.loads(self.state.read_text())["appliance_pid"]
try:
with open("/proc/%d/stat" % pid) as f:
state = f.read().rsplit(")", 1)[1].split()[0]
except FileNotFoundError:
state = "gone"
self.assertIn(state, ("gone", "Z"))
class GstfsdClientTestCase(unittest.TestCase):
"""The panel side sends the whole request and reads the whole reply."""
def test_a_reply_in_pieces_is_read_whole(self):
import threading
from unittest.mock import patch
from instances import views
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(("127.0.0.1", 0))
server.listen(1)
self.addCleanup(server.close)
received = []
def serve():
conn, _ = server.accept()
with conn:
received.append(conn.recv(65536))
conn.sendall(b'{"return": ')
time.sleep(0.2)
conn.sendall(b'"success"}')
thread = threading.Thread(target=serve)
thread.start()
with patch.object(views, "GSTFSD_PORT", server.getsockname()[1]):
reply = views.gstfsd_request("127.0.0.1", {"action": "publickey", "vname": "vm", "key": "k"})
thread.join(5)
self.assertEqual(reply, {"return": "success"})
self.assertEqual(json.loads(received[0])["vname"], "vm")

View file

@ -28,10 +28,12 @@ class SetRootPassTestCase(TestCase):
def test_set_root_pass_post(self, mock_proxy, mock_socket_cls):
# Renamed on the host: gstfsd must get the name of this UUID's domain
mock_proxy.instance.name.return_value = "renamed-on-host"
mock_proxy.get_status.return_value = 5 # status: running
mock_proxy.get_status.return_value = 5 # shut off
mock_sock = MagicMock()
mock_socket_cls.return_value = mock_sock
mock_sock.recv.return_value = json.dumps({"return": "success"}).encode()
replies = iter([json.dumps({"return": "success"}).encode(), b""] * 2)
# one reply, then gstfsd closes the connection
mock_sock.recv.side_effect = lambda size: next(replies)
response = self.client.post(
reverse("instances:rootpasswd", args=[self.instance.id]),
@ -41,8 +43,8 @@ class SetRootPassTestCase(TestCase):
self.assertEqual(response.status_code, 302)
# Check socket sent the command with hashed password
self.assertTrue(mock_sock.send.called)
sent_bytes = mock_sock.send.call_args[0][0]
self.assertTrue(mock_sock.sendall.called)
sent_bytes = mock_sock.sendall.call_args[0][0]
data = json.loads(sent_bytes.decode())
self.assertEqual(data["action"], "password")
self.assertEqual(data["vname"], "renamed-on-host")
@ -65,13 +67,15 @@ class SetRootPassTestCase(TestCase):
mock_proxy.get_status.return_value = 5
mock_sock = MagicMock()
mock_socket_cls.return_value = mock_sock
mock_sock.recv.return_value = json.dumps({"return": "success"}).encode()
replies = iter([json.dumps({"return": "success"}).encode(), b""] * 2)
# one reply, then gstfsd closes the connection
mock_sock.recv.side_effect = lambda size: next(replies)
url = reverse("instances:rootpasswd", args=[self.instance.id])
self.client.post(url, {"passwd": "samePassword1"})
self.client.post(url, {"passwd": "samePassword1"})
hashes = [json.loads(c[0][0].decode())["passwd"] for c in mock_sock.send.call_args_list]
hashes = [json.loads(c[0][0].decode())["passwd"] for c in mock_sock.sendall.call_args_list]
self.assertEqual(len(hashes), 2)
self.assertNotEqual(hashes[0], hashes[1])

View file

@ -707,17 +707,20 @@ def migrate(request, pk):
GSTFSD_PORT = 16510
GSTFSD_TIMEOUT = 120 # gstfsd starts a guestfs appliance, which takes a while
GSTFSD_TIMEOUT = 120 # gstfsd answers within its 100 s limit for a request (GSTFSD_DEADLINE)
def gstfsd_request(hostname, data):
"""Send one request to gstfsd and return its JSON reply, or an error reply."""
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(GSTFSD_TIMEOUT)
try:
s.connect((hostname, GSTFSD_PORT))
s.send(json.dumps(data).encode())
return json.loads(s.recv(1024).strip())
# the host name may resolve to IPv4 or IPv6
s = socket.create_connection((hostname, GSTFSD_PORT), timeout=GSTFSD_TIMEOUT)
except OSError as err:
return {"return": "error", "message": _("gstfsd error: %(err)s") % {"err": err}}
try:
s.sendall(json.dumps(data).encode())
# gstfsd sends one reply and closes the connection
return json.loads(b"".join(iter(lambda: s.recv(4096), b"")))
except (OSError, ValueError) as err:
return {"return": "error", "message": _("gstfsd error: %(err)s") % {"err": err}}
finally: