From a50e9793ea61c6f249be7281f913bcd23d2554c9 Mon Sep 17 00:00:00 2001 From: Bernardo Magri Date: Fri, 12 Jun 2026 08:05:13 +0100 Subject: [PATCH] feat(tools): commit the VM test harness + one-command installer regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Promotes the throwaway harness that verified the installer into tools/: - tools/vm/qmp.py QMP keystroke injection / typing / quit - tools/vm/vncshot.py GL-safe screenshots via VNC readback (QMP screendump shows "no surface" with virtio-vga-gl) - tools/vm/gap-analysis.py drv-graph diff that converged the offline pin set; run it when offline installs build from source - tools/test-install.sh the full offline regression: build ISO, boot offline, unattended LUKS+swap install via a config disk (typed long commands drop keys), wait for poweroff, boot the installed disk, screenshot the first boot for visual verdict docs/TESTING.md §4 now points at the script. Co-Authored-By: Claude Fable 5 --- README.md | 5 +- docs/TESTING.md | 34 +++++---- tools/test-install.sh | 151 +++++++++++++++++++++++++++++++++++++++ tools/vm/gap-analysis.py | 74 +++++++++++++++++++ tools/vm/qmp.py | 105 +++++++++++++++++++++++++++ tools/vm/vncshot.py | 104 +++++++++++++++++++++++++++ 6 files changed, 457 insertions(+), 16 deletions(-) create mode 100755 tools/test-install.sh create mode 100755 tools/vm/gap-analysis.py create mode 100755 tools/vm/qmp.py create mode 100755 tools/vm/vncshot.py diff --git a/README.md b/README.md index b47da81..48c115d 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,10 @@ Flat on purpose. Two module trees, one options file each, no hidden layers. ├── docs/TESTING.md # how to verify changes (incl. AI-agent rules) └── tools/ # maintainer-only ├── import-palettes.py # converts old-distro themes → JSON + assets - └── test-live-iso.sh # build the ISO + boot it in QEMU + ├── test-live-iso.sh # build the ISO + boot it in QEMU + ├── test-install.sh # full offline-install regression in QEMU + └── vm/ # headless-VM helpers (QMP keys, VNC shots, + # offline-pin gap analysis) ``` **Rule of thumb:** `modules/` is the distro (reusable, no machine specifics), diff --git a/docs/TESTING.md b/docs/TESTING.md index 34bb755..b9bc829 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -69,25 +69,29 @@ model end to end. ## 4. Testing the installer -The installer has an unattended mode for exactly this. Boot the live ISO in -QEMU **with a second, blank disk attached** (e.g. `-drive -file=target.img,if=virtio,format=raw` on a 20 G image), then in the live -terminal: +One command runs the whole offline regression: ```sh -sudo NOMARCHY_UNATTENDED=1 NOMARCHY_DISK=/dev/vda \ - NOMARCHY_USERNAME=me NOMARCHY_PASSWORD=test \ - NOMARCHY_LUKS_PASSPHRASE=testtest1 \ - NOMARCHY_FINISH=poweroff nomarchy-install +tools/test-install.sh ``` -Then boot the VM again **from the target disk** (drop `-cdrom`). After -typing the LUKS passphrase at the initrd prompt, the machine must reach -the themed desktop **without a login prompt** (LUKS implies auto-login) — -wallpaper, Waybar — and `nomarchy-theme-sync apply ` must work. For the -offline claim, add `restrict=on` to the `-netdev` and re-run the whole -flow: it must behave identically (requires an ISO built from a clean git -tree, which is what bakes the rev into the installer). +It builds the ISO, boots it in QEMU **with networking disabled**, runs an +unattended LUKS+swap install onto a blank disk via the installer's +`NOMARCHY_UNATTENDED=1` mode (config delivered on a small vfat disk — +typing long commands into a guest drops keystrokes), waits for the +installer's poweroff, then boots the installed disk, enters the +passphrase, and screenshots the first boot. The machine-checkable parts +(install completes, disk boots) fail the script; the visual verdict — +**themed desktop, no autogenerated-config banner, no login prompt** — is +yours, from `first-boot.png`. The helpers it builds on live in +`tools/vm/` (QMP key injection, GL-safe VNC screenshots) and work for any +manual VM poking; `tools/vm/gap-analysis.py` is the maintainer tool for +diagnosing offline installs that try to build from source. + +To test by hand instead, replicate what `tools/test-install.sh` does: the +unattended env it uses is in the script, and the same flow works +interactively from the live terminal. If the desktop comes up unthemed, +read `/var/log/nomarchy-hm-preactivate.log` on the installed system. ## 5. VM-specific gotchas diff --git a/tools/test-install.sh b/tools/test-install.sh new file mode 100755 index 0000000..48cd1a2 --- /dev/null +++ b/tools/test-install.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# End-to-end regression test for nomarchy-install: build the live ISO, +# run a fully OFFLINE unattended install into a fresh VM disk, then boot +# the installed system and capture what the first boot looks like. +# +# This automates the verification protocol from docs/TESTING.md §4 that +# debugged the installer in the first place. It asserts the machine- +# checkable parts (installer reaches poweroff; installed disk boots); +# the visual part — a fully themed desktop, no autogenerated-config +# banner — lands as screenshots for a human (or agent) to inspect: +# +# $NOMARCHY_VM_DIR/install-typed.png the typed command, pre-enter +# $NOMARCHY_VM_DIR/luks-prompt.png initrd passphrase prompt +# $NOMARCHY_VM_DIR/first-boot.png THE result: themed desktop +# +# Requirements: KVM, qemu, OVMF, dosfstools (mkfs.vfat), mtools (mcopy), +# python3. Tunables (env): +# NOMARCHY_VM_DIR scratch + screenshots (default /tmp/nomarchy-vm) +# NOMARCHY_TARGET_DIR where the 25G target.img lives — must NOT be +# tmpfs (default ~/.cache/nomarchy-vm) +# NOMARCHY_BOOT_WAIT seconds to wait for the live desktop (default 480) +# NOMARCHY_INSTALL_TIMEOUT seconds before declaring a hang (default 2700) + +set -euo pipefail +cd "$(dirname "$0")/.." + +VM_DIR="${NOMARCHY_VM_DIR:-/tmp/nomarchy-vm}" +TARGET_DIR="${NOMARCHY_TARGET_DIR:-$HOME/.cache/nomarchy-vm}" +BOOT_WAIT="${NOMARCHY_BOOT_WAIT:-480}" +INSTALL_TIMEOUT="${NOMARCHY_INSTALL_TIMEOUT:-2700}" +export NOMARCHY_QMP_SOCK="$VM_DIR/qmp.sock" +QMP=tools/vm/qmp.py +SHOT=tools/vm/vncshot.py +VNC_DISPLAY=17 # port 5917 + +for tool in qemu-system-x86_64 mkfs.vfat mcopy python3; do + command -v "$tool" >/dev/null || { + echo "missing $tool — try: nix shell nixpkgs#qemu nixpkgs#dosfstools nixpkgs#mtools" >&2 + exit 1 + } +done +[ -r /dev/kvm ] || { echo "no KVM — this test is too slow without it" >&2; exit 1; } + +mkdir -p "$VM_DIR" "$TARGET_DIR" +if df --output=fstype "$TARGET_DIR" | tail -1 | grep -q tmpfs; then + echo "NOMARCHY_TARGET_DIR=$TARGET_DIR is tmpfs — the install writes ~10G; pick a real disk" >&2 + exit 1 +fi + +OVMF="" +for c in /run/current-system/sw/share/qemu/edk2-x86_64-code.fd \ + /run/current-system/sw/share/OVMF/OVMF_CODE.fd \ + /usr/share/OVMF/OVMF_CODE.fd; do + [ -f "$c" ] && { OVMF="$c"; break; } +done +[ -n "$OVMF" ] || { echo "no OVMF firmware found" >&2; exit 1; } + +echo "==> Building the live ISO..." +nix build .#nixosConfigurations.nomarchy-live.config.system.build.isoImage +ISO=$(ls -1 result/iso/*.iso | head -n1) + +echo "==> Preparing disks..." +# Unattended config rides in on a tiny vfat disk: the installer test +# can't reliably TYPE a 200-char env line into the guest (keystrokes +# get dropped), but one short mount-and-run command works. +cat > "$VM_DIR/go.sh" <<'EOF' +#!/usr/bin/env bash +export NOMARCHY_UNATTENDED=1 +export NOMARCHY_DISK=/dev/vda +export NOMARCHY_USERNAME=ada +export NOMARCHY_PASSWORD=nomarchy +export NOMARCHY_HOSTNAME=testbox +export NOMARCHY_LUKS_PASSPHRASE=testtest1 +export NOMARCHY_SWAP_GB=2 +export NOMARCHY_FINISH=poweroff +nomarchy-install 2>&1 | tee /tmp/install.log +EOF +rm -f "$VM_DIR/config.img" +truncate -s 8M "$VM_DIR/config.img" +mkfs.vfat "$VM_DIR/config.img" >/dev/null +mcopy -i "$VM_DIR/config.img" "$VM_DIR/go.sh" ::go.sh + +rm -f "$TARGET_DIR/target.img" "$NOMARCHY_QMP_SOCK" +qemu-img create -f raw "$TARGET_DIR/target.img" 25G >/dev/null + +qemu_base=( + qemu-system-x86_64 + -enable-kvm -cpu host -m 6144 -smp 2 + -drive "if=pflash,format=raw,readonly=on,file=$OVMF" + -device virtio-vga-gl -display egl-headless + -vnc "127.0.0.1:$VNC_DISPLAY" + -device virtio-net-pci,netdev=n0 -netdev user,id=n0,restrict=on # OFFLINE + -qmp "unix:$NOMARCHY_QMP_SOCK,server,nowait" +) + +echo "==> Phase 1: installing (offline) — this takes ~20-30 min..." +"${qemu_base[@]}" \ + -drive "file=$TARGET_DIR/target.img,if=virtio,format=raw" \ + -drive "file=$VM_DIR/config.img,if=virtio,format=raw,readonly=on" \ + -cdrom "$ISO" -boot d \ + >"$VM_DIR/qemu-install.log" 2>&1 & +QPID=$! +trap 'kill $QPID 2>/dev/null || true' EXIT + +until [ -S "$NOMARCHY_QMP_SOCK" ]; do sleep 2; done +echo " waiting ${BOOT_WAIT}s for the live desktop..." +sleep "$BOOT_WAIT" +python3 "$QMP" key meta_l-ret +sleep 8 +python3 "$QMP" type 'sudo bash -c "mkdir -p /m; mount /dev/vdb /m; bash /m/go.sh"' +sleep 2 +python3 "$SHOT" "$VM_DIR/install-typed.png" 127.0.0.1 "59$VNC_DISPLAY" +python3 "$QMP" key ret +echo " install running (typed command captured in install-typed.png)..." + +waited=0 +while kill -0 $QPID 2>/dev/null; do + if (( waited >= INSTALL_TIMEOUT )); then + python3 "$SHOT" "$VM_DIR/install-hung.png" 127.0.0.1 "59$VNC_DISPLAY" || true + echo "FAIL: installer did not power off within ${INSTALL_TIMEOUT}s (see install-hung.png)" >&2 + exit 1 + fi + sleep 20; waited=$((waited + 20)) +done +trap - EXIT +echo " VM powered off after ~${waited}s — install completed." + +echo "==> Phase 2: first boot of the installed system..." +rm -f "$NOMARCHY_QMP_SOCK" +"${qemu_base[@]}" \ + -drive "file=$TARGET_DIR/target.img,if=virtio,format=raw" \ + >"$VM_DIR/qemu-boot.log" 2>&1 & +QPID=$! +trap 'kill $QPID 2>/dev/null || true' EXIT + +until [ -S "$NOMARCHY_QMP_SOCK" ]; do sleep 2; done +sleep 40 +python3 "$SHOT" "$VM_DIR/luks-prompt.png" 127.0.0.1 "59$VNC_DISPLAY" +python3 "$QMP" type 'testtest1' +python3 "$QMP" key ret +sleep 150 +python3 "$SHOT" "$VM_DIR/first-boot.png" 127.0.0.1 "59$VNC_DISPLAY" +python3 "$QMP" quit || true +trap - EXIT + +echo +echo "PASS (machine-checkable part): install powered off, installed disk boots." +echo "Now INSPECT $VM_DIR/first-boot.png:" +echo " expected: themed desktop (wallpaper + waybar), NO red banner, no login prompt" +echo " if it shows the Hyprland water-drop or an 'autogenerated config' banner," +echo " log in on tty2 (ada/nomarchy) and read /var/log/nomarchy-hm-preactivate.log" diff --git a/tools/vm/gap-analysis.py b/tools/vm/gap-analysis.py new file mode 100755 index 0000000..4688db3 --- /dev/null +++ b/tools/vm/gap-analysis.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Which store paths would an offline nixos-install need that the ISO lacks? + +Walk the target drv graph: an output that's PRESENT needs nothing; a +missing output forces building its drv, which needs all its input +sources present and all referenced input outputs (recursively). + +This is the maintainer tool that converged the offline pin set in +flake.nix (extraDependencies): when an offline install builds from +source, run this instead of bisecting the VM. Healthy output is a few +dozen per-machine config drvs (etc, initrd-*, unit-*, …) and zero +[FETCH/NETWORK] leaves; any real package name here needs pinning. + +Usage: + # 1. the present set = everything the ISO carries: + live=$(nix build --no-link --print-out-paths \ + .#nixosConfigurations.nomarchy-live.config.system.build.toplevel) + nix path-info -r "$live" | sort -u > /tmp/present.txt + # 2. instantiate an install-shaped toplevel for a FOREIGN + # username/hostname/uuids (see docs/TESTING.md) and: + gap-analysis.py /tmp/present.txt +""" +import json +import subprocess +import sys + +target_drv, present_file = sys.argv[1], sys.argv[2] +present = set(open(present_file).read().split()) + +graph = json.loads(subprocess.check_output( + ["nix", "derivation", "show", "-r", target_drv + "^*"])) + +must_build = set() +missing_srcs = set() +visited = set() + + +def outputs_of(drv_name): + return [o["path"] for o in graph[drv_name]["outputs"].values()] + + +def need_drv(drv_name): + if drv_name in visited: + return + visited.add(drv_name) + must_build.add(drv_name) + node = graph[drv_name] + for src in node.get("inputSrcs", []): + if src not in present: + missing_srcs.add(src) + for dep_drv, want in node.get("inputDrvs", {}).items(): + dep_outputs = graph.get(dep_drv) + if dep_outputs is None: + continue + wanted = want.get("outputs", []) if isinstance(want, dict) else want + for out_name in wanted: + path = graph[dep_drv]["outputs"][out_name]["path"] + if path not in present: + need_drv(dep_drv) + break + + +# Roots: the target's own outputs are by definition "to build". +need_drv(target_drv) + +print(f"drvs that must be built offline: {len(must_build)}") +for d in sorted(must_build): + name = d.split("-", 1)[1] if "-" in d else d + builder = graph[d].get("builder", "") + fetch = " [FETCH/NETWORK]" if graph[d].get("env", {}).get("urls") or graph[d].get("env", {}).get("url") else "" + print(f" {name}{fetch}") +print(f"missing input sources: {len(missing_srcs)}") +for s in sorted(missing_srcs): + print(f" SRC {s}") diff --git a/tools/vm/qmp.py b/tools/vm/qmp.py new file mode 100755 index 0000000..644ce7a --- /dev/null +++ b/tools/vm/qmp.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Minimal QMP driver for the Nomarchy test VMs. + +Drives a QEMU instance through its QMP socket: inject keystrokes, type +text, take screendumps, power off. Used by tools/test-install.sh; also +handy interactively while debugging a VM that has no SSH. + +Usage: + qmp.py key # e.g. meta_l-ret, ctrl-alt-f2, ret + qmp.py type # types literal text (US layout) + qmp.py screenshot # NOTE: with virtio-vga-gl this fails + # ("no surface") once the guest uses GL — + # use vncshot.py instead for desktops + qmp.py quit # hard power-off + +The socket path comes from $NOMARCHY_QMP_SOCK +(default /tmp/nomarchy-vm/qmp.sock). + +Keystroke pacing: the guest drops keys when busy (especially during +boot). $NOMARCHY_TYPE_DELAY (seconds between keys, default 0.08) trades +speed for reliability; verify long typed lines with a screenshot before +pressing enter. +""" +import json +import os +import socket +import sys +import time + +SOCK = os.environ.get("NOMARCHY_QMP_SOCK", "/tmp/nomarchy-vm/qmp.sock") +TYPE_DELAY = float(os.environ.get("NOMARCHY_TYPE_DELAY", "0.08")) + +CHAR_KEYS = { + " ": "spc", "-": "minus", "=": "equal", "[": "bracket_left", + "]": "bracket_right", ";": "semicolon", "'": "apostrophe", + "`": "grave_accent", "\\": "backslash", ",": "comma", ".": "dot", + "/": "slash", "\n": "ret", "\t": "tab", +} +SHIFT_CHARS = { + "!": "1", "@": "2", "#": "3", "$": "4", "%": "5", "^": "6", + "&": "7", "*": "8", "(": "9", ")": "0", "_": "minus", "+": "equal", + "{": "bracket_left", "}": "bracket_right", ":": "semicolon", + '"': "apostrophe", "~": "grave_accent", "|": "backslash", + "<": "comma", ">": "dot", "?": "slash", +} + + +class QMP: + def __init__(self): + self.s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.s.connect(SOCK) + self.f = self.s.makefile("rw") + self._read() # greeting + self.cmd("qmp_capabilities") + + def _read(self): + while True: + msg = json.loads(self.f.readline()) + if "event" in msg: + continue + return msg + + def cmd(self, name, **args): + self.f.write(json.dumps({"execute": name, "arguments": args}) + "\n") + self.f.flush() + resp = self._read() + if "error" in resp: + raise RuntimeError(f"{name}: {resp['error']}") + return resp + + def send_keys(self, names, hold=40): + keys = [{"type": "qcode", "data": n} for n in names] + self.cmd("send-key", keys=keys, **{"hold-time": hold}) + + def type_text(self, text): + for ch in text: + if ch.isalpha() and ch.lower() != ch: + self.send_keys(["shift", ch.lower()]) + elif ch in SHIFT_CHARS: + self.send_keys(["shift", SHIFT_CHARS[ch]]) + elif ch in CHAR_KEYS: + self.send_keys([CHAR_KEYS[ch]]) + else: + self.send_keys([ch]) + time.sleep(TYPE_DELAY) + + +def main(): + q = QMP() + cmd = sys.argv[1] + if cmd == "screenshot": + q.cmd("screendump", filename=sys.argv[2], format="png") + elif cmd == "key": + q.send_keys(sys.argv[2].split("-")) + elif cmd == "type": + q.type_text(" ".join(sys.argv[2:])) + elif cmd == "quit": + q.f.write(json.dumps({"execute": "quit"}) + "\n") + q.f.flush() + else: + sys.exit(f"unknown command {cmd}") + + +if __name__ == "__main__": + main() diff --git a/tools/vm/vncshot.py b/tools/vm/vncshot.py new file mode 100755 index 0000000..d9efd9d --- /dev/null +++ b/tools/vm/vncshot.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Grab one full framebuffer from a no-auth VNC server and save as PNG. + +Why VNC and not QMP screendump: with virtio-vga-gl the guest renders +into a GL scanout and screendump fails with "no surface"; QEMU's VNC +server reads the GL framebuffer back, so this always works. Pair the VM +with `-display egl-headless -vnc 127.0.0.1:17`. + +Stdlib only (manual RFB handshake + raw-encoding framebuffer + PNG +writer) so it runs anywhere python3 exists. + +Usage: vncshot.py [host] [port] # default 127.0.0.1:5917 +""" +import socket, struct, sys, zlib + +def recv_exact(s, n): + buf = b"" + while len(buf) < n: + chunk = s.recv(n - len(buf)) + if not chunk: + raise EOFError("server closed") + buf += chunk + return buf + +def png_write(path, w, h, rgb): + def chunk(tag, data): + c = struct.pack(">I", len(data)) + tag + data + return c + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + raw = b"".join(b"\x00" + rgb[y * w * 3:(y + 1) * w * 3] for y in range(h)) + out = (b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 6)) + + chunk(b"IEND", b"")) + with open(path, "wb") as f: + f.write(out) + +def main(): + out = sys.argv[1] + host = sys.argv[2] if len(sys.argv) > 2 else "127.0.0.1" + port = int(sys.argv[3]) if len(sys.argv) > 3 else 5917 + s = socket.create_connection((host, port), timeout=30) + s.settimeout(30) + + ver = recv_exact(s, 12) # "RFB 003.008\n" + s.sendall(b"RFB 003.008\n") + ntypes = recv_exact(s, 1)[0] + types = recv_exact(s, ntypes) + if 1 not in types: + sys.exit(f"no 'None' security type offered: {list(types)}") + s.sendall(b"\x01") + res = struct.unpack(">I", recv_exact(s, 4))[0] + if res != 0: + sys.exit("security handshake failed") + s.sendall(b"\x01") # ClientInit: shared + w, h = struct.unpack(">HH", recv_exact(s, 4)) + recv_exact(s, 16) # server pixel format (ignored) + namelen = struct.unpack(">I", recv_exact(s, 4))[0] + recv_exact(s, namelen) + + # SetPixelFormat: 32bpp truecolor, shifts r=16 g=8 b=0 (little-endian BGRX) + pf = struct.pack(">BBBBHHHBBBxxx", 32, 24, 0, 1, 255, 255, 255, 16, 8, 0) + s.sendall(b"\x00\x00\x00\x00" + pf) + s.sendall(struct.pack(">BxH i", 2, 1, 0)) # SetEncodings: Raw only + s.sendall(struct.pack(">BBHHHH", 3, 0, 0, 0, w, h)) # full update request + + fb = bytearray(w * h * 4) + got = False + while not got: + mtype = recv_exact(s, 1)[0] + if mtype == 0: # FramebufferUpdate + recv_exact(s, 1) + nrects = struct.unpack(">H", recv_exact(s, 2))[0] + for _ in range(nrects): + x, y, rw, rh = struct.unpack(">HHHH", recv_exact(s, 8)) + enc = struct.unpack(">i", recv_exact(s, 4))[0] + if enc == 0: + data = recv_exact(s, rw * rh * 4) + for row in range(rh): + off = ((y + row) * w + x) * 4 + fb[off:off + rw * 4] = data[row * rw * 4:(row + 1) * rw * 4] + elif enc == -224: # LastRect + break + else: + sys.exit(f"unsupported encoding {enc}") + got = True + elif mtype == 2: # Bell + pass + elif mtype == 3: # ServerCutText + recv_exact(s, 3) + ln = struct.unpack(">I", recv_exact(s, 4))[0] + recv_exact(s, ln) + else: + sys.exit(f"unexpected server message {mtype}") + + rgb = bytearray(w * h * 3) + for i in range(w * h): + rgb[i * 3 + 0] = fb[i * 4 + 2] + rgb[i * 3 + 1] = fb[i * 4 + 1] + rgb[i * 3 + 2] = fb[i * 4 + 0] + png_write(out, w, h, bytes(rgb)) + print(f"saved {out} ({w}x{h})") + +if __name__ == "__main__": + main()