feat(console): typed text follows the VM's keyboard layout

"Type it" sent each character's keysym and let QEMU pick the key from
its own US map, with Shift added for the US shifted characters. A VM
on another layout got other characters: on Turkish Q, '"' typed 'İ',
'@' typed "'", and ğ, ü, ş, ı, ö, ç typed nothing.

The Clipboard dialog now asks for the VM's keyboard layout and
remembers it per browser. Each character is typed as the key that makes
it on that layout (its scancode), with Shift, AltGr or both (AltGr
first: pressed after Shift, the right Alt is Alt on the Linux console),
so the VM's own layout makes the character. The layout is fixed when
typing starts. Text with a character no key types on the layout is not
typed at all, and the dialog lists those characters: a password or a
command with a character left out would be a wrong one.

A layout is one file in console/layouts/: its name and, for each key,
the characters it types plainly, with Shift, with AltGr and with
Shift+AltGr. The view offers every file, US first. dev/console_layout.py
writes a file from the system's XKB data as libxkbcommon computes it
(through ctypes, no new dependency), mapping keys to DOM codes with
noVNC's table from QEMU's keycodemapdb; it lists the dead keys it leaves
out, and drops the keys the keyboard model lacks (the key left of Z on
US pc104 keyboards). English (US) and Turkish Q come with it; the
README says how to add one.

README.md

@@ -698,6 +698,18 @@ Run every step as your normal user, not with `sudo`: if `manage.py` ever runs as
 
    Open `http://127.0.0.1:8000`. The browser consoles connect to `/novncd/` and `/socket.io/` on the panel's own address, which the dev server does not proxy: try them in the Docker image, or behind nginx with `conf/nginx/webvirtcloud.conf`.
 
+### Keyboard layouts
+
+The VNC console's "Type it" sends the keys that type each character on the VM's keyboard layout. A layout is one file in `console/layouts/`, written from the XKB layout of the same name (needs `libxkbcommon` and `xkeyboard-config`, present on Linux desktops):
+
+```bash
+python3 dev/console_layout.py de "German"                  # XKB layout, the name the console shows
+python3 dev/console_layout.py tr "Turkish F" --variant f   # a variant
+python3 dev/console_layout.py us "English (US)" --model pc104  # a keyboard without the key left of Z
+```
+
+It lists the dead keys it leaves out: a character only a dead key types cannot be typed. Type a sample with every character group of the layout into a VM set to that layout before committing the file.
+
 ### Running tests
 
 ```bash

console/layouts/tr.json

@@ -0,0 +1,54 @@
+{
+    "name": "Turkish Q",
+    "source": {"xkb_layout": "tr", "xkb_variant": "", "xkb_model": "pc105"},
+    "keys": {
+        "Digit1": ["1", "!", ">", "¡"],
+        "Digit2": ["2", "'", "£", "²"],
+        "Digit3": ["3", "^", "#", "³"],
+        "Digit4": ["4", "+", "$", "¼"],
+        "Digit5": ["5", "%", "½", "⅜"],
+        "Digit6": ["6", "&", "¾", null],
+        "Digit7": ["7", "/", "{", null],
+        "Digit8": ["8", "(", "[", null],
+        "Digit9": ["9", ")", "]", "±"],
+        "Digit0": ["0", "=", "}", "°"],
+        "Minus": ["*", "?", "\\", "¿"],
+        "Equal": ["-", "_", "|", null],
+        "KeyQ": ["q", "Q", "@", "Ω"],
+        "KeyW": ["w", "W", null, null],
+        "KeyE": ["e", "E", "€", null],
+        "KeyR": ["r", "R", "¶", "®"],
+        "KeyT": ["t", "T", "₺", null],
+        "KeyY": ["y", "Y", "←", "¥"],
+        "KeyU": ["u", "U", "û", "Û"],
+        "KeyI": ["ı", "I", "î", "Î"],
+        "KeyO": ["o", "O", "ô", "Ô"],
+        "KeyP": ["p", "P", null, null],
+        "BracketLeft": ["ğ", "Ğ", null, null],
+        "BracketRight": ["ü", "Ü", "~", null],
+        "KeyA": ["a", "A", "â", "Â"],
+        "KeyS": ["s", "S", "ß", null],
+        "KeyD": ["d", "D", null, null],
+        "KeyF": ["f", "F", "ª", null],
+        "KeyG": ["g", "G", null, null],
+        "KeyH": ["h", "H", null, null],
+        "KeyJ": ["j", "J", null, null],
+        "KeyK": ["k", "K", null, null],
+        "KeyL": ["l", "L", null, null],
+        "Semicolon": ["ş", "Ş", "´", null],
+        "Quote": ["i", "İ", "'", null],
+        "Backquote": ["\"", "é", "<", "°"],
+        "Backslash": [",", ";", "`", null],
+        "KeyZ": ["z", "Z", "«", "<"],
+        "KeyX": ["x", "X", "»", ">"],
+        "KeyC": ["c", "C", "¢", "©"],
+        "KeyV": ["v", "V", "„", "‚"],
+        "KeyB": ["b", "B", "“", "‘"],
+        "KeyN": ["n", "N", "”", "’"],
+        "KeyM": ["m", "M", "µ", "º"],
+        "Comma": ["ö", "Ö", "×", null],
+        "Period": ["ç", "Ç", "·", "÷"],
+        "Slash": [".", ":", null, null],
+        "IntlBackslash": ["<", ">", "|", "¦"]
+    }
+}

console/layouts/us.json

@@ -0,0 +1,53 @@
+{
+    "name": "English (US)",
+    "source": {"xkb_layout": "us", "xkb_variant": "", "xkb_model": "pc104"},
+    "keys": {
+        "Digit1": ["1", "!", null, null],
+        "Digit2": ["2", "@", null, null],
+        "Digit3": ["3", "#", null, null],
+        "Digit4": ["4", "$", null, null],
+        "Digit5": ["5", "%", null, null],
+        "Digit6": ["6", "^", null, null],
+        "Digit7": ["7", "&", null, null],
+        "Digit8": ["8", "*", null, null],
+        "Digit9": ["9", "(", null, null],
+        "Digit0": ["0", ")", null, null],
+        "Minus": ["-", "_", null, null],
+        "Equal": ["=", "+", null, null],
+        "KeyQ": ["q", "Q", null, null],
+        "KeyW": ["w", "W", null, null],
+        "KeyE": ["e", "E", null, null],
+        "KeyR": ["r", "R", null, null],
+        "KeyT": ["t", "T", null, null],
+        "KeyY": ["y", "Y", null, null],
+        "KeyU": ["u", "U", null, null],
+        "KeyI": ["i", "I", null, null],
+        "KeyO": ["o", "O", null, null],
+        "KeyP": ["p", "P", null, null],
+        "BracketLeft": ["[", "{", null, null],
+        "BracketRight": ["]", "}", null, null],
+        "KeyA": ["a", "A", null, null],
+        "KeyS": ["s", "S", null, null],
+        "KeyD": ["d", "D", null, null],
+        "KeyF": ["f", "F", null, null],
+        "KeyG": ["g", "G", null, null],
+        "KeyH": ["h", "H", null, null],
+        "KeyJ": ["j", "J", null, null],
+        "KeyK": ["k", "K", null, null],
+        "KeyL": ["l", "L", null, null],
+        "Semicolon": [";", ":", null, null],
+        "Quote": ["'", "\"", null, null],
+        "Backquote": ["`", "~", null, null],
+        "Backslash": ["\\", "|", null, null],
+        "KeyZ": ["z", "Z", null, null],
+        "KeyX": ["x", "X", null, null],
+        "KeyC": ["c", "C", null, null],
+        "KeyV": ["v", "V", null, null],
+        "KeyB": ["b", "B", null, null],
+        "KeyN": ["n", "N", null, null],
+        "KeyM": ["m", "M", null, null],
+        "Comma": [",", "<", null, null],
+        "Period": [".", ">", null, null],
+        "Slash": ["/", "?", null, null]
+    }
+}

console/templates/console-vnc.html

@@ -7,6 +7,7 @@
     #screen { flex: 1 1 auto; min-height: 0; overflow: hidden; }
 </style>
 {{ vnc|json_script:"vnc-config" }}
+{{ layouts|json_script:"keyboard-layouts" }}
 <script type="module" src="{% static "js/console_vnc.js" %}"></script>
 {% endblock %}
 
@@ -103,8 +104,16 @@
                 <label class="form-label" for="clipboard_text">{% trans "Text" %}</label>
                 <textarea class="form-control font-monospace" id="clipboard_text" rows="6"></textarea>
                 <div class="form-text">
-                    {% trans "Type it: the VM gets it key by key, also in its firmware, an installer or a login prompt. The VM's clipboard takes it only with clipboard support in the VM; what the VM copies shows here." %}
+                    {% trans "Type it: the VM gets it key by key, as the keys of its keyboard layout, also in its firmware, an installer or a login prompt. The VM's clipboard takes it only with clipboard support in the VM; what the VM copies shows here." %}
                 </div>
+                <label class="form-label mt-3" for="clipboard_layout">{% trans "The VM's keyboard layout" %}</label>
+                <select class="form-select" id="clipboard_layout">
+                    {% for id, layout in layouts.items %}
+                    <option value="{{ id }}">{{ layout.name }}</option>
+                    {% endfor %}
+                </select>
+                <div class="alert alert-warning mt-3 mb-0" id="clipboard_missing" role="alert" hidden
+                    data-text="{% trans "Nothing was typed: no key types these characters on this layout:" %}"></div>
             </div>
             <div class="modal-footer">
                 <button type="button" class="btn btn-secondary" id="clipboard_send">{% trans "Send to the VM's clipboard" %}</button>

console/tests/test_layouts.py

@@ -0,0 +1,47 @@
+"""The keyboard layout files the VNC console types text with."""
+
+import json
+from pathlib import Path
+
+from django.conf import settings
+from django.test import SimpleTestCase
+
+from console.views import LAYOUTS_DIR
+
+SCANCODES = Path(settings.BASE_DIR) / "static/js/novnc/core/input/xtscancodes.js"
+
+
+def unique_keys(pairs):
+    keys = [key for key, _ in pairs]
+    if len(keys) != len(set(keys)):
+        raise ValueError(f"a key twice: {keys}")
+    return dict(pairs)
+
+
+class LayoutFilesTestCase(SimpleTestCase):
+    def test_each_layout_file_is_a_layout(self):
+        # the key codes noVNC sends as scancodes
+        codes = {line.split('"')[1] for line in SCANCODES.read_text().splitlines() if line.startswith('  "')}
+        paths = sorted(LAYOUTS_DIR.glob("*.json"))
+        self.assertIn("us", [path.stem for path in paths])
+        for path in paths:
+            with self.subTest(path.name):
+                layout = json.loads(path.read_text(), object_pairs_hook=unique_keys)
+                self.assertRegex(path.stem, r"^[a-z]+(-[a-z0-9_]+)?$")
+                self.assertEqual(set(layout), {"name", "source", "keys"})
+                self.assertTrue(layout["name"])
+                for code, characters in layout["keys"].items():
+                    self.assertIn(code, codes)
+                    self.assertEqual(len(characters), 4, code)
+                    for character in characters:
+                        # one printable character, or none
+                        self.assertTrue(character is None or (len(character) == 1 and character.isprintable()),
+                                        f"{code}: {character!r}")
+
+    def test_us_and_turkish_q_type_every_printable_ascii_character(self):
+        ascii_characters = {chr(c) for c in range(0x21, 0x7F)}
+        for layout_id in ("us", "tr"):
+            with self.subTest(layout_id):
+                layout = json.loads((LAYOUTS_DIR / f"{layout_id}.json").read_text())
+                typed = {c for characters in layout["keys"].values() for c in characters if c}
+                self.assertEqual(ascii_characters - typed, set())

console/tests/test_views.py

@@ -8,6 +8,7 @@ from unittest.mock import MagicMock, patch
 from accounts.models import UserInstance
 from appsettings.models import AppSettings
 from computes.models import Compute
+from console.views import LAYOUTS_DIR
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import Permission
 from django.core.signing import TimestampSigner
@@ -250,6 +251,19 @@ class ConsoleViewsTestCase(TestCase):
         script = (Path(settings.BASE_DIR) / "static/js/console_vnc.js").read_text()
         self.assertIn("scheme + window.location.host + '/novncd/?token='", script)
 
+    @patch("console.views.wvmInstance")
+    def test_the_vnc_page_offers_each_keyboard_layout_file(self, mock_wvm):
+        mock_wvm.return_value.get_console_type.return_value = "vnc"
+        mock_wvm.return_value.get_console_passwd.return_value = None
+        self.client.force_login(self.alice)
+        response = self.client.get(self.url)
+        layouts = json.loads(html.fromstring(response.content).get_element_by_id("keyboard-layouts").text)
+        files = sorted(path.stem for path in LAYOUTS_DIR.glob("*.json"))
+        self.assertEqual(sorted(layouts), files)
+        self.assertEqual(next(iter(layouts)), "us")
+        for layout_id, layout in layouts.items():
+            self.assertContains(response, f'<option value="{layout_id}">{layout["name"]}</option>', html=True)
+
     @patch("console.views.wvmInstance")
     def test_the_vnc_page_starts_as_the_console_settings_say(self, mock_wvm):
         mock_wvm.return_value.get_console_type.return_value = "vnc"

console/views.py

@@ -1,5 +1,9 @@
 # pylint: disable=no-name-in-module,no-member
 
+import functools
+import json
+from pathlib import Path
+
 from appsettings.settings import app_settings
 from console import tokens
 from django.conf import settings
@@ -12,6 +16,22 @@ from vrtManager.connection import CONN_TCP, CONN_TLS
 from vrtManager.instance import wvmInstance
 from vrtManager.util import is_loopback
 
+# one file a layout, written by dev/console_layout.py
+LAYOUTS_DIR = Path(__file__).resolve().parent / "layouts"
+
+
+@functools.cache
+def keyboard_layouts():
+    """The keyboard layouts the VNC console types text with, US first, then
+    by name: {id: {"name": ..., "keys": {DOM code: [plain, Shift, AltGr,
+    Shift+AltGr]}}}."""
+    layouts = {}
+    for path in LAYOUTS_DIR.glob("*.json"):
+        layout = json.loads(path.read_text())
+        layouts[path.stem] = {"name": layout["name"], "keys": layout["keys"]}
+    us = layouts.pop("us")
+    return {"us": us, **dict(sorted(layouts.items(), key=lambda item: item[1]["name"]))}
+
 
 def console(request, compute_id, uuid):
     """The console page of the VM with this UUID on this compute. A console
@@ -89,4 +109,4 @@ def console(request, compute_id, uuid):
         "clip": app_settings.CONSOLE_CLIP_VIEWPORT == "True",
         "viewOnly": app_settings.CONSOLE_VIEW_ONLY == "True",
     }
-    return render(request, "console-vnc.html", {"instance": instance, "vnc": vnc})
+    return render(request, "console-vnc.html", {"instance": instance, "vnc": vnc, "layouts": keyboard_layouts()})

dev/console_layout.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Write console/layouts/<id>.json, a keyboard layout the VNC console's
+"Type it" can type with, from the system's XKB data (xkeyboard-config),
+as libxkbcommon computes it.
+
+    python3 dev/console_layout.py tr "Turkish Q"
+    python3 dev/console_layout.py tr "Turkish F" --variant f
+    python3 dev/console_layout.py us "English (US)" --model pc104
+
+The model is the keyboard's: pc105 (the default) has a key left of Z
+(IntlBackslash) that US keyboards (pc104) lack.
+
+Each key the layout writes with gets the characters it types plainly, with
+Shift, with AltGr and with Shift+AltGr (null: none, or the same as without
+AltGr). Dead keys are left out and listed.
+"""
+
+import argparse
+import ctypes
+import ctypes.util
+import json
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+SCANCODES = ROOT / "static/js/novnc/core/input/xtscancodes.js"
+LAYOUTS = ROOT / "console/layouts"
+
+KEY_DOWN = 1
+SHIFT = "ShiftLeft"
+ALTGR = "AltRight"
+MODIFIERS = [[], [SHIFT], [ALTGR], [ALTGR, SHIFT]]  # pressed in this order, as the console page does
+# the writing keys a keyboard model lacks (XKB defines them all the same);
+# other models, such as jp106 and abnt2, have them
+MISSING_KEYS = {"pc104": {"IntlBackslash", "IntlRo", "IntlYen"}, "pc105": {"IntlRo", "IntlYen"}}
+
+
+class RuleNames(ctypes.Structure):
+    _fields_ = [(name, ctypes.c_char_p) for name in ("rules", "model", "layout", "variant", "options")]
+
+
+def load_xkbcommon():
+    xkb = ctypes.CDLL(ctypes.util.find_library("xkbcommon") or "libxkbcommon.so.0")
+    for name, restype, argtypes in (
+        ("xkb_context_new", ctypes.c_void_p, [ctypes.c_int]),
+        ("xkb_keymap_new_from_names", ctypes.c_void_p, [ctypes.c_void_p, ctypes.POINTER(RuleNames), ctypes.c_int]),
+        ("xkb_state_new", ctypes.c_void_p, [ctypes.c_void_p]),
+        ("xkb_state_unref", None, [ctypes.c_void_p]),
+        ("xkb_state_update_key", ctypes.c_int, [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_int]),
+        ("xkb_state_key_get_utf32", ctypes.c_uint32, [ctypes.c_void_p, ctypes.c_uint32]),
+        ("xkb_state_key_get_one_sym", ctypes.c_uint32, [ctypes.c_void_p, ctypes.c_uint32]),
+        ("xkb_keysym_get_name", ctypes.c_int, [ctypes.c_uint32, ctypes.c_char_p, ctypes.c_size_t]),
+    ):
+        function = getattr(xkb, name)
+        function.restype, function.argtypes = restype, argtypes
+    return xkb
+
+
+def linux_keycodes():
+    """{DOM code: Linux keycode}, from noVNC's table (QEMU's keycodemapdb)."""
+    pattern = re.compile(r'^  "(\w+)": 0x\w+, /\* html:\w+ \(\w+\) -> linux:(\d+) ', re.M)
+    return {code: int(linux) for code, linux in pattern.findall(SCANCODES.read_text())}
+
+
+def writing_keys(codes, model):
+    """The keys of the model's keyboard that write characters: not the keypad,
+    not Space (typed on its own)."""
+    missing = MISSING_KEYS.get(model, set()) | {"Space"}
+    return {code: linux for code, linux in codes.items() if not code.startswith("Numpad") and code not in missing}
+
+
+def press(xkb, keymap, keycodes, keys):
+    """What the last of the keys gives with the others held: (character or None, keysym)."""
+    state = xkb.xkb_state_new(keymap)
+    for key in keys[:-1]:
+        xkb.xkb_state_update_key(state, keycodes[key], KEY_DOWN)
+    utf32 = xkb.xkb_state_key_get_utf32(state, keycodes[keys[-1]])
+    keysym = xkb.xkb_state_key_get_one_sym(state, keycodes[keys[-1]])
+    xkb.xkb_state_unref(state)
+    return (chr(utf32) if utf32 >= 0x20 and utf32 != 0x7F else None), keysym
+
+
+def keysym_name(xkb, keysym):
+    name = ctypes.create_string_buffer(64)
+    xkb.xkb_keysym_get_name(keysym, name, len(name))
+    return name.value.decode()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("layout", help='XKB layout, e.g. "tr", "de", "fr"')
+    parser.add_argument("name", help='the name the console shows, e.g. "Turkish Q"')
+    parser.add_argument("--variant", default="", help='XKB variant, e.g. "f" for Turkish F')
+    parser.add_argument("--model", default="pc105", help="XKB keyboard model: pc105, or pc104 for US keyboards")
+    args = parser.parse_args()
+
+    xkb = load_xkbcommon()
+    names = RuleNames(b"evdev", args.model.encode(), args.layout.encode(), args.variant.encode(), b"")
+    keymap = xkb.xkb_keymap_new_from_names(xkb.xkb_context_new(0), ctypes.byref(names), 0)
+    if not keymap:
+        parser.error(f"XKB has no layout {args.layout!r} with variant {args.variant!r}")
+    # XKB keycodes are Linux keycodes + 8
+    keycodes = {code: linux + 8 for code, linux in linux_keycodes().items()}
+
+    keys, dead = {}, []
+    for code in sorted(writing_keys(keycodes, args.model), key=keycodes.get):
+        characters = []
+        for slot, modifiers in enumerate(MODIFIERS):
+            character, keysym = press(xkb, keymap, keycodes, [*modifiers, code])
+            if keysym_name(xkb, keysym).startswith("dead_"):
+                dead.append(f"{'+'.join([*modifiers, code])}: {keysym_name(xkb, keysym)}")
+            # AltGr that changes nothing is no AltGr (the US layout has none)
+            if slot >= 2 and character == characters[slot - 2]:
+                character = None
+            characters.append(character)
+        if any(characters):
+            keys[code] = characters
+
+    layout_id = args.layout + (f"-{args.variant}" if args.variant else "")
+    lines = ",\n".join(f"        {json.dumps(code)}: {json.dumps(chars, ensure_ascii=False)}" for code, chars in keys.items())
+    source = json.dumps({"xkb_layout": args.layout, "xkb_variant": args.variant, "xkb_model": args.model})
+    path = LAYOUTS / f"{layout_id}.json"
+    path.write_text(f'{{\n    "name": {json.dumps(args.name, ensure_ascii=False)},\n    "source": {source},\n'
+                    f'    "keys": {{\n{lines}\n    }}\n}}\n')
+    print(f"wrote {path.relative_to(ROOT)}: {len(keys)} keys")
+    if dead:
+        print("dead keys, left out:\n  " + "\n  ".join(dead))
+
+
+if __name__ == "__main__":
+    main()

static/js/console_layouts.js

@@ -0,0 +1,29 @@
+// The keys that type a character on the VM's keyboard layout: the VM gets
+// keys, not characters, and its own layout makes characters of them. The
+// layouts are console/layouts/*.json, put in the page by the view.
+
+// the modifiers of a layout key's four characters, pressed in this order:
+// AltGr before Shift, as the right Alt pressed after Shift is Alt on the
+// Linux console
+const MODIFIERS = [[], ['ShiftLeft'], ['AltRight'], ['AltRight', 'ShiftLeft']];
+
+const layouts = JSON.parse(document.getElementById('keyboard-layouts').textContent);
+
+// layout id: {character: [key code, modifier codes]}; of the keys that type
+// a character, the first with the fewest modifiers
+const keys = {};
+for (const [id, layout] of Object.entries(layouts)) {
+    keys[id] = {};
+    MODIFIERS.forEach((modifiers, slot) => {
+        for (const [code, characters] of Object.entries(layout.keys)) {
+            const character = characters[slot];
+            if (character !== null && !(character in keys[id])) {
+                keys[id][character] = [code, modifiers];
+            }
+        }
+    });
+}
+
+export function layoutKeys(layout, character) {
+    return keys[layout][character];
+}

static/js/console_vnc.js

@@ -6,6 +6,7 @@ import KeyTable from './novnc/core/input/keysym.js';
 import keysyms from './novnc/core/input/keysymdef.js';
 import { isTouchDevice } from './novnc/core/util/browser.js';
 import { bindFullscreen, bindKeyAction, bindSendKeys, releaseHeld } from './console_keys.js';
+import { layoutKeys } from './console_layouts.js';
 
 const config = JSON.parse(document.getElementById('vnc-config').textContent);
 const status = document.getElementById('status');
@@ -99,50 +100,88 @@ bindKeyAction('credentials_cancel', () => {
 bindSendKeys(rfb);
 bindFullscreen();
 
-// the characters a US keyboard types with Shift: the VM gets keys, and its
-// key for "A" is the key for "a"
-const SHIFTED = /[A-Z~!@#$%^&*()_+{}|:"<>?]/;
+// the VM's keyboard layout, the viewer's last choice
+const layout = document.getElementById('clipboard_layout');
+try {
+    layout.value = localStorage.getItem('console_layout') || layout.value;
+} catch {}
+layout.selectedIndex = Math.max(layout.selectedIndex, 0);  // a layout no longer offered
+layout.addEventListener('change', () => {
+    try {
+        localStorage.setItem('console_layout', layout.value);
+    } catch {}
+});
 
 const typedKeys = {
     '\n': [KeyTable.XK_Return, 'Enter'],
     '\b': [KeyTable.XK_BackSpace, 'Backspace'],
     '\t': [KeyTable.XK_Tab, 'Tab'],
+    ' ': [KeyTable.XK_space, 'Space'],
 };
+const modifierKeysyms = { ShiftLeft: KeyTable.XK_Shift_L, AltRight: KeyTable.XK_ISO_Level3_Shift };
 
-async function typeCharacter(character) {
+// [keysym, key code, modifier codes], or nothing when no key types it: the
+// VM makes the character of the key (its scancode), the keysym only names it
+function keysFor(layoutId, character) {
+    if (character in typedKeys) {
+        return [...typedKeys[character], []];
+    }
+    const keys = layoutKeys(layoutId, character);
+    return keys && [keysyms.lookup(character.codePointAt(0)), ...keys];
+}
+
+async function typeKeys([keysym, code, modifiers]) {
     // a typist's pace: firmware drops keys that come faster
     await new Promise((resolve) => setTimeout(resolve, 25));
-    const [keysym, code] = typedKeys[character] || [keysyms.lookup(character.codePointAt(0)), null];
-    if (SHIFTED.test(character)) {
-        rfb.sendKey(KeyTable.XK_Shift_L, 'ShiftLeft', true);
-        rfb.sendKey(keysym, code);
-        rfb.sendKey(KeyTable.XK_Shift_L, 'ShiftLeft', false);
-    } else {
-        rfb.sendKey(keysym, code);
-    }
+    modifiers.forEach((modifier) => rfb.sendKey(modifierKeysyms[modifier], modifier, true));
+    rfb.sendKey(keysym, code);
+    [...modifiers].reverse().forEach((modifier) => rfb.sendKey(modifierKeysyms[modifier], modifier, false));
 }
 
 // key by key, after what is still being typed: works where no clipboard does
-// (firmware, installers, login prompts); "\b" is a Backspace
+// (firmware, installers, login prompts); "\b" is a Backspace. All or nothing,
+// a password or a command with a character left out is a wrong one: returns
+// the characters no key types on the layout.
 let typing = Promise.resolve();
 function typeText(text) {
-    for (const character of text.replace(/\r\n?/g, '\n')) {
-        typing = typing.then(() => typeCharacter(character));
+    const characters = [...text.replace(/\r\n?/g, '\n')];
+    const keys = characters.map((character) => keysFor(layout.value, character));
+    const missing = characters.filter((character, i) => !keys[i]);
+    if (missing.length === 0) {
+        for (const characterKeys of keys) {
+            typing = typing.then(() => typeKeys(characterKeys));
+        }
     }
+    return missing;
+}
+
+// a character as the user can see it: an invisible one by its code point
+function shown(character) {
+    const invisible = /[\s\p{C}]/u.test(character);
+    return invisible ? 'U+' + character.codePointAt(0).toString(16).toUpperCase().padStart(4, '0') : character;
 }
 
 const clipboardText = document.getElementById('clipboard_text');
+const clipboardMissing = document.getElementById('clipboard_missing');
 const clipboardDialog = bootstrap.Modal.getOrCreateInstance('#clipboard_dialog');
 rfb.addEventListener('clipboard', (e) => {
     clipboardText.value = e.detail.text;
 });
+document.getElementById('clipboard_dialog').addEventListener('show.bs.modal', () => {
+    clipboardMissing.hidden = true;
+});
 bindKeyAction('clipboard_send', () => {
     rfb.clipboardPasteFrom(clipboardText.value);
     clipboardDialog.hide();
 });
 bindKeyAction('clipboard_type', () => {
-    clipboardDialog.hide();
-    typeText(clipboardText.value);
+    const missing = typeText(clipboardText.value);
+    if (missing.length === 0) {
+        clipboardDialog.hide();
+    } else {
+        clipboardMissing.textContent = clipboardMissing.dataset.text + ' ' + [...new Set(missing)].map(shown).join(' ');
+        clipboardMissing.hidden = false;
+    }
 });
 
 const bell = document.getElementById('bell');
@@ -212,8 +251,10 @@ if (isTouchDevice) {
         while (same < Math.min(text.length, lastText.length) && text[same] === lastText[same]) {
             same++;
         }
-        typeText('\b'.repeat(lastText.length - same) + text.slice(same));
-        if (text.length === 0 || text.length > 200) {
+        // a character the layout has no key for: the field stays as the VM has it
+        if (typeText('\b'.repeat(lastText.length - same) + text.slice(same)).length > 0) {
+            keyboardInput.value = lastText;
+        } else if (text.length === 0 || text.length > 200) {
             resetInput();
         } else {
             lastText = text;