mirror of
https://github.com/retspen/webvirtcloud
synced 2026-11-08 11:16:47 +00:00
fix(console): serial sessions start under a lock and say why they are refused
- socketiod's handlers run on their own threads. A connection is now admitted under a lock, so two at once can no longer both open a VM's console or pass the session cap. A session's terminal is used and closed only under its own lock while it is open: a closed descriptor's number, taken again by a new session, never receives another session's input or gives it the other's output. - A refused connection says why: serial console off, the link expired, the VM's console open in another window, too many consoles. The page shows it instead of "refused". - The page no longer retries on its own with a token that has expired after two minutes; Reconnect loads a new page. Input the browser cannot encode no longer raises in the daemon.
This commit is contained in:
catborise
parent
294b008811
commit
1fac3de23c
3 changed files with 108 additions and 45 deletions
|
|
@ -14,6 +14,7 @@ import signal
|
|||
import struct
|
||||
import sys
|
||||
import termios
|
||||
import threading
|
||||
|
||||
import django
|
||||
|
||||
|
|
@ -34,44 +35,61 @@ CONSOLECALLBACK = os.path.join(ROOT_PATH, "conf", "daemon", "consolecallback")
|
|||
MAX_SESSIONS = 20
|
||||
|
||||
sio = socketio.Server(async_mode="threading") # the same origin only
|
||||
sessions = {} # sid -> {"pid", "fd", "vm"}
|
||||
# Handlers run on their own threads. sessions (sid -> {"pid", "fd", "vm",
|
||||
# "lock", "open"}) changes only under sessions_lock; a session's fd is used
|
||||
# and closed only under its own lock, while it is open, so a closed fd's
|
||||
# number, reused by a new session, never reaches another console.
|
||||
sessions = {}
|
||||
sessions_lock = threading.Lock()
|
||||
|
||||
|
||||
@sio.event
|
||||
def connect(sid, environ, auth):
|
||||
# the page shows the reason
|
||||
if not settings.SERIAL_CONSOLE_ENABLED:
|
||||
return False
|
||||
raise socketio.exceptions.ConnectionRefusedError("The serial console is turned off on this server")
|
||||
try:
|
||||
instance = authorize((auth or {}).get("token"))
|
||||
except PermissionError:
|
||||
return False
|
||||
raise socketio.exceptions.ConnectionRefusedError("The console link has expired: reload the page")
|
||||
finally:
|
||||
db.close_old_connections()
|
||||
if len(sessions) >= MAX_SESSIONS or any(s["vm"] == instance.pk for s in sessions.values()):
|
||||
return False
|
||||
pid, fd = pty.fork()
|
||||
if pid == 0:
|
||||
try:
|
||||
# pty.fork's master is inheritable: without this each worker would
|
||||
# hold the other sessions' consoles open
|
||||
os.closerange(3, os.sysconf("SC_OPEN_MAX"))
|
||||
os.execv(sys.executable, [sys.executable, CONSOLECALLBACK, str(instance.pk)])
|
||||
finally:
|
||||
os._exit(1)
|
||||
sessions[sid] = session = {"pid": pid, "fd": fd, "vm": instance.pk}
|
||||
with sessions_lock:
|
||||
if any(s["vm"] == instance.pk for s in sessions.values()):
|
||||
raise socketio.exceptions.ConnectionRefusedError("The console of this VM is open in another window")
|
||||
if len(sessions) >= MAX_SESSIONS:
|
||||
raise socketio.exceptions.ConnectionRefusedError("Too many consoles are open on this server: try again later")
|
||||
pid, fd = pty.fork()
|
||||
if pid == 0:
|
||||
try:
|
||||
# pty.fork's master is inheritable: without this each worker would
|
||||
# hold the other sessions' consoles open
|
||||
os.closerange(3, os.sysconf("SC_OPEN_MAX"))
|
||||
os.execv(sys.executable, [sys.executable, CONSOLECALLBACK, str(instance.pk)])
|
||||
finally:
|
||||
os._exit(1)
|
||||
sessions[sid] = session = {"pid": pid, "fd": fd, "vm": instance.pk, "lock": threading.Lock(), "open": True}
|
||||
sio.start_background_task(forward, sid, session)
|
||||
|
||||
|
||||
def forward(sid, session):
|
||||
"""The worker's output to its client, until the worker ends."""
|
||||
decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") # a character may be split
|
||||
while sessions.get(sid) is session:
|
||||
if not select.select([session["fd"]], [], [], 0.1)[0]:
|
||||
continue
|
||||
while session["open"]:
|
||||
try:
|
||||
output = os.read(session["fd"], 20480)
|
||||
except OSError: # EIO: the worker ended
|
||||
output = b""
|
||||
# a fd closed meanwhile may wake this up: the session is closed then
|
||||
ready = select.select([session["fd"]], [], [], 0.1)[0]
|
||||
except (OSError, ValueError):
|
||||
ready = True
|
||||
if not ready:
|
||||
continue
|
||||
with session["lock"]:
|
||||
if not session["open"]:
|
||||
return
|
||||
try:
|
||||
output = os.read(session["fd"], 20480)
|
||||
except OSError: # EIO: the worker ended
|
||||
output = b""
|
||||
if not output:
|
||||
sio.emit("pty_output", {"output": decoder.decode(b"", final=True) + "\r\n[console closed]\r\n"}, to=sid)
|
||||
stop(sid)
|
||||
|
|
@ -82,17 +100,20 @@ def forward(sid, session):
|
|||
|
||||
def stop(sid):
|
||||
"""End the sid's worker; once, whoever calls it first."""
|
||||
session = sessions.pop(sid, None)
|
||||
with sessions_lock:
|
||||
session = sessions.pop(sid, None)
|
||||
if session is None:
|
||||
return
|
||||
try:
|
||||
os.close(session["fd"])
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.kill(session["pid"], signal.SIGKILL)
|
||||
os.kill(session["pid"], signal.SIGKILL) # first: a write blocked on its terminal ends
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
with session["lock"]:
|
||||
session["open"] = False
|
||||
try:
|
||||
os.close(session["fd"])
|
||||
except OSError:
|
||||
pass
|
||||
sio.start_background_task(os.waitpid, session["pid"], 0)
|
||||
|
||||
|
||||
|
|
@ -100,10 +121,13 @@ def stop(sid):
|
|||
def pty_input(sid, message):
|
||||
session = sessions.get(sid)
|
||||
if session and isinstance(message, dict) and isinstance(message.get("input"), str):
|
||||
try:
|
||||
os.write(session["fd"], message["input"].encode())
|
||||
except OSError:
|
||||
pass
|
||||
data = message["input"].encode(errors="replace") # a lone surrogate from the browser
|
||||
with session["lock"]:
|
||||
if session["open"]:
|
||||
try:
|
||||
os.write(session["fd"], data)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
@sio.event
|
||||
|
|
@ -113,7 +137,9 @@ def resize(sid, message):
|
|||
return
|
||||
rows, cols = message.get("rows"), message.get("cols")
|
||||
if all(isinstance(v, int) and 1 <= v <= 500 for v in (rows, cols)):
|
||||
fcntl.ioctl(session["fd"], termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0))
|
||||
with session["lock"]:
|
||||
if session["open"]:
|
||||
fcntl.ioctl(session["fd"], termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0))
|
||||
|
||||
|
||||
@sio.event
|
||||
|
|
|
|||
|
|
@ -23,8 +23,9 @@
|
|||
<div id="terminal"></div>
|
||||
<script>
|
||||
// socketiod is behind this page's own /socket.io/ location; the page's
|
||||
// signed token is the credential (the cookie holds the last console's)
|
||||
var socket = io({transports: ["websocket", "polling"], auth: {token: "{{ token|escapejs }}"}});
|
||||
// signed token is the credential. It lasts two minutes: a new
|
||||
// connection takes a new page (Reconnect), not an automatic retry.
|
||||
var socket = io({transports: ["websocket", "polling"], auth: {token: "{{ token|escapejs }}"}, reconnection: false});
|
||||
|
||||
const status = document.getElementById("status");
|
||||
const button = document.getElementById("button");
|
||||
|
|
@ -49,12 +50,12 @@
|
|||
function showStatus(text, color) {
|
||||
status.textContent = text;
|
||||
status.style.backgroundColor = color;
|
||||
button.textContent = socket.connected ? "Disconnect" : "Connect";
|
||||
button.textContent = socket.connected ? "{% trans 'Disconnect'|escapejs %}" : "{% trans 'Reconnect'|escapejs %}";
|
||||
}
|
||||
socket.on("connect", () => showStatus("connected", "lightgreen"));
|
||||
socket.on("disconnect", () => showStatus("disconnected", "#ff8383"));
|
||||
// refused: serial console off, an expired token, or the VM's console already open
|
||||
socket.on("connect_error", () => showStatus("refused", "#ff8383"));
|
||||
socket.on("connect", () => showStatus("{% trans 'connected'|escapejs %}", "lightgreen"));
|
||||
socket.on("disconnect", () => showStatus("{% trans 'disconnected'|escapejs %}", "#ff8383"));
|
||||
// the server says why: serial console off, an expired link, the console open elsewhere
|
||||
socket.on("connect_error", (err) => showStatus(err.message, "#ff8383"));
|
||||
|
||||
function myFunction(){
|
||||
if (socket.connected) {
|
||||
|
|
|
|||
|
|
@ -90,19 +90,24 @@ class SocketiodTestCase(Base):
|
|||
self.assertEqual(self.d.sessions["sid-1"]["vm"], self.vm.pk)
|
||||
self.assertEqual(self.started[0][0], self.d.forward)
|
||||
|
||||
def test_refused_connections_start_nothing(self):
|
||||
def refused(self, sid, auth):
|
||||
with self.assertRaises(self.d.socketio.exceptions.ConnectionRefusedError) as refusal:
|
||||
self.d.connect(sid, {}, auth)
|
||||
return refusal.exception.error_args["message"]
|
||||
|
||||
def test_refused_connections_start_nothing_and_say_why(self):
|
||||
other = Instance.objects.create(compute=self.compute, name="vm-b", uuid="11111111-0000-0000-0000-000000000002")
|
||||
UserInstance.objects.create(user=self.user, instance=other)
|
||||
with override_settings(SERIAL_CONSOLE_ENABLED=False):
|
||||
self.assertIs(self.d.connect("off", {}, {"token": self.token()}), False)
|
||||
self.assertIn("turned off", self.refused("off", {"token": self.token()}))
|
||||
with override_settings(SERIAL_CONSOLE_ENABLED=True):
|
||||
self.assertIs(self.d.connect("no-auth", {}, None), False)
|
||||
self.assertIs(self.d.connect("bad", {}, {"token": "x"}), False)
|
||||
self.assertIn("expired", self.refused("no-auth", None))
|
||||
self.assertIn("expired", self.refused("bad", {"token": "x"}))
|
||||
self.assertIsNone(self.d.connect("first", {}, {"token": self.token()}))
|
||||
# one session per VM
|
||||
self.assertIs(self.d.connect("second", {}, {"token": self.token()}), False)
|
||||
self.assertIn("another window", self.refused("second", {"token": self.token()}))
|
||||
with patch.object(self.d, "MAX_SESSIONS", 1):
|
||||
self.assertIs(self.d.connect("over", {}, {"token": self.token(vm=other)}), False)
|
||||
self.assertIn("Too many", self.refused("over", {"token": self.token(vm=other)}))
|
||||
self.assertEqual(list(self.d.sessions), ["first"])
|
||||
|
||||
@override_settings(SERIAL_CONSOLE_ENABLED=True)
|
||||
|
|
@ -124,10 +129,41 @@ class SocketiodTestCase(Base):
|
|||
def test_a_session_is_stopped_once(self):
|
||||
self.d.connect("sid-1", {}, {"token": self.token()})
|
||||
pid = self.d.sessions["sid-1"]["pid"]
|
||||
session = self.d.sessions["sid-1"]
|
||||
self.d.disconnect("sid-1")
|
||||
self.d.stop("sid-1")
|
||||
self.assertNotIn("sid-1", self.d.sessions)
|
||||
self.d.os.kill.assert_called_once_with(pid, self.d.signal.SIGKILL)
|
||||
# a closed session's fd number may belong to a new session: nothing uses it
|
||||
self.assertFalse(session["open"])
|
||||
with patch.object(self.d.os, "write") as write:
|
||||
self.d.pty_input("sid-1", {"input": "x"})
|
||||
session_input = self.d.sessions.get("sid-1")
|
||||
write.assert_not_called()
|
||||
self.assertIsNone(session_input)
|
||||
|
||||
@override_settings(SERIAL_CONSOLE_ENABLED=True)
|
||||
def test_two_connections_at_once_start_one_console(self):
|
||||
import threading
|
||||
|
||||
barrier, results = threading.Barrier(8), []
|
||||
|
||||
def connect(n):
|
||||
barrier.wait()
|
||||
try:
|
||||
results.append(self.d.connect(f"sid-{n}", {}, {"token": self.token()}))
|
||||
except self.d.socketio.exceptions.ConnectionRefusedError:
|
||||
results.append("refused")
|
||||
|
||||
threads = [threading.Thread(target=connect, args=(n,)) for n in range(8)]
|
||||
# the threads have their own database connections, outside this test's transaction
|
||||
with patch.object(self.d, "authorize", return_value=self.vm):
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
self.assertEqual(sorted(results, key=str), [None] + ["refused"] * 7)
|
||||
self.assertEqual(len(self.d.sessions), 1)
|
||||
|
||||
|
||||
class WorkerIsolationTestCase(Base):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue