Nomarchy/tools/test-install-safety.sh

#!/usr/bin/env bash
# V2 harness for BACKLOG #54 installer safety guards.
# Builds the live ISO once, then runs offline unattended scenarios:
#
#   disk-warn   — target image has an NTFS signature → warn in install.log
#   compose-fail — NOMARCHY_TEST_FORCE_COMPOSE_FAIL=1 → offline fail-closed
#   hm-hint     — NOMARCHY_TEST_FORCE_HM_FAIL=1 → hint file on target home
#
# Usage:
#   tools/test-install-safety.sh              # all three
#   tools/test-install-safety.sh disk-warn    # one scenario
#
# Env (same as test-install.sh where applicable):
#   NOMARCHY_VM_DIR  NOMARCHY_TARGET_DIR  NOMARCHY_BOOT_WAIT  NOMARCHY_INSTALL_TIMEOUT
#   NOMARCHY_SCENARIOS  space-separated override (default: all three)

set -euo pipefail
cd "$(dirname "$0")/.."

VM_DIR="${NOMARCHY_VM_DIR:-/tmp/nomarchy-vm-safety}"
TARGET_DIR="${NOMARCHY_TARGET_DIR:-$HOME/.cache/nomarchy-vm-safety}"
BOOT_WAIT="${NOMARCHY_BOOT_WAIT:-480}"
INSTALL_TIMEOUT="${NOMARCHY_INSTALL_TIMEOUT:-2700}"
export NOMARCHY_QMP_SOCK="$VM_DIR/qmp.sock"
QMP=tools/vm/qmp.py
SHOT=tools/vm/vncshot.py
# Prefer a free VNC display (vm leftovers often hold :17/:18).
VNC_DISPLAY="${NOMARCHY_VNC_DISPLAY:-}"
if [ -z "$VNC_DISPLAY" ]; then
  for d in 19 20 21 22 23; do
    if ! ss -ltn 2>/dev/null | grep -q ":59${d} "; then
      VNC_DISPLAY=$d
      break
    fi
  done
fi
VNC_DISPLAY="${VNC_DISPLAY:-19}"

SCENARIOS=("disk-warn" "compose-fail" "hm-hint")
if [[ $# -gt 0 ]]; then
  SCENARIOS=("$@")
elif [[ -n "${NOMARCHY_SCENARIOS:-}" ]]; then
  # shellcheck disable=SC2206
  SCENARIOS=($NOMARCHY_SCENARIOS)
fi

for tool in qemu-system-x86_64 mkfs.vfat mcopy python3; do
  command -v "$tool" >/dev/null || {
    echo "missing $tool" >&2
    exit 1
  }
done
[ -r /dev/kvm ] || { echo "no KVM" >&2; exit 1; }

mkdir -p "$VM_DIR" "$TARGET_DIR"
if df --output=fstype "$TARGET_DIR" | tail -1 | grep -q tmpfs; then
  echo "NOMARCHY_TARGET_DIR=$TARGET_DIR is tmpfs" >&2
  exit 1
fi

OVMF="${NOMARCHY_OVMF:-}"
[ -n "$OVMF" ] || for c in /run/current-system/sw/share/qemu/edk2-x86_64-code.fd \
  /run/current-system/sw/share/OVMF/OVMF_CODE.fd \
  /usr/share/OVMF/OVMF_CODE.fd; do
  [ -f "$c" ] && { OVMF="$c"; break; }
done
# Fall back to a nix-store OVMF (dev boxes without it on the profile).
if [ -z "$OVMF" ]; then
  OVMF=$(ls -1 /nix/store/*/FV/OVMF_CODE.fd 2>/dev/null | head -1 || true)
fi
[ -n "$OVMF" ] && [ -f "$OVMF" ] || { echo "no OVMF" >&2; exit 1; }

echo "==> Building live ISO..."
nix build .#nixosConfigurations.nomarchy-live.config.system.build.isoImage
ISO=$(ls -1 result/iso/*.iso | head -n1)
echo "    ISO=$ISO"

qemu_base=(
  qemu-system-x86_64
  -enable-kvm -cpu host -m 6144 -smp 2
  -drive "if=pflash,format=raw,readonly=on,file=$OVMF"
  -device virtio-vga-gl -display egl-headless
  -vnc "127.0.0.1:$VNC_DISPLAY"
  -device virtio-net-pci,netdev=n0 -netdev user,id=n0,restrict=on
  -qmp "unix:$NOMARCHY_QMP_SOCK,server,nowait"
)

prepare_signature_target() {
  local img=$1
  rm -f "$img"
  # Full-size image (install continues after the warn). LUKS1 magic at LBA 0
  # is reliably reported by blkid as TYPE=crypto_LUKS — matches the installer's
  # crypto_luks regex. No root/cryptsetup needed.
  qemu-img create -f raw "$img" 25G >/dev/null
  python3 - "$img" <<'PY'
import sys
path = sys.argv[1]
with open(path, "r+b") as f:
    # LUKS1 on-disk magic (util-linux blkid probe)
    hdr = bytearray(512)
    hdr[0:6] = b"LUKS\xba\xbe"
    hdr[6] = 1  # version major
    hdr[7] = 0
    f.write(hdr)
print(f"    wrote LUKS magic at LBA 0 on {path}")
PY
}

write_go_sh() {
  local scenario=$1
  local out=$2
  local force_compose=0 force_hm=0 finish=poweroff
  case "$scenario" in
    disk-warn) finish=poweroff ;;
    compose-fail) force_compose=1; finish=none ;; # fail before poweroff path
    hm-hint) force_hm=1; finish=poweroff ;;
    *) echo "unknown scenario $scenario" >&2; exit 1 ;;
  esac
  cat >"$out" <<EOF
#!/usr/bin/env bash
set -uo pipefail
export NOMARCHY_UNATTENDED=1
export NOMARCHY_DISK=/dev/vda
export NOMARCHY_USERNAME=ada
export NOMARCHY_PASSWORD=nomarchy
export NOMARCHY_HOSTNAME=testbox
export NOMARCHY_LUKS_PASSPHRASE=testtest1
export NOMARCHY_SWAP_GB=2
export NOMARCHY_FINISH=$finish
export NOMARCHY_TEST_FORCE_COMPOSE_FAIL=$force_compose
export NOMARCHY_TEST_FORCE_HM_FAIL=$force_hm
# Do not set -e: compose-fail exits 1 and we still need the log on vfat.
nomarchy-install 2>&1 | tee /tmp/install.log
rc=\${PIPESTATUS[0]}
cp -f /tmp/install.log /m/install.log 2>/dev/null || true
if [[ "$scenario" == "hm-hint" && -f /mnt/home/ada/NOMARCHY-DESKTOP-NOT-THEMED.txt ]]; then
  cp -f /mnt/home/ada/NOMARCHY-DESKTOP-NOT-THEMED.txt /m/hint.txt || true
fi
echo "INSTALL_RC=\$rc" | tee -a /tmp/install.log | tee -a /m/install.log
# Always power off so the host wait loop ends (compose-fail uses FINISH=none).
systemctl poweroff 2>/dev/null || poweroff -f 2>/dev/null || true
exit \$rc
EOF
}

run_scenario() {
  local scenario=$1
  local sdir="$VM_DIR/$scenario"
  mkdir -p "$sdir"
  local target="$TARGET_DIR/$scenario-target.img"
  local cfgimg="$sdir/config.img"
  local log="$sdir/install.log"
  echo
  echo "========== scenario: $scenario =========="

  if [[ "$scenario" == "disk-warn" ]]; then
    prepare_signature_target "$target"
  else
    rm -f "$target"
    qemu-img create -f raw "$target" 25G >/dev/null
  fi

  write_go_sh "$scenario" "$sdir/go.sh"
  chmod +x "$sdir/go.sh"
  rm -f "$cfgimg"
  truncate -s 16M "$cfgimg"
  mkfs.vfat "$cfgimg" >/dev/null
  mcopy -i "$cfgimg" "$sdir/go.sh" ::go.sh

  rm -f "$NOMARCHY_QMP_SOCK"
  "${qemu_base[@]}" \
    -drive "file=$target,if=virtio,format=raw" \
    -drive "file=$cfgimg,if=virtio,format=raw" \
    -cdrom "$ISO" -boot d \
    >"$sdir/qemu-install.log" 2>&1 &
  local qpid=$!
  # shellcheck disable=SC2064
  trap "kill $qpid 2>/dev/null || true" RETURN

  waited_sock=0
  until [ -S "$NOMARCHY_QMP_SOCK" ]; do
    if ! kill -0 "$qpid" 2>/dev/null; then
      echo "FAIL: QEMU died before QMP came up (see $sdir/qemu-install.log)" >&2
      tail -20 "$sdir/qemu-install.log" >&2 || true
      return 1
    fi
    if (( waited_sock > 120 )); then
      echo "FAIL: no QMP socket after 120s" >&2
      return 1
    fi
    sleep 2
    waited_sock=$((waited_sock + 2))
  done
  echo "    waiting ${BOOT_WAIT}s for live desktop (VNC :$VNC_DISPLAY)..."
  sleep "$BOOT_WAIT"
  python3 "$QMP" key meta_l-ret
  sleep 8
  # Mount config rw so the guest can write install.log back.
  python3 "$QMP" type 'sudo bash -c "mkdir -p /m; mount /dev/vdb /m; bash /m/go.sh; umount /m || true"'
  sleep 2
  python3 "$SHOT" "$sdir/install-typed.png" 127.0.0.1 "59$VNC_DISPLAY" || true
  python3 "$QMP" key ret
  echo "    install running..."

  local waited=0
  while kill -0 "$qpid" 2>/dev/null; do
    if (( waited >= INSTALL_TIMEOUT )); then
      python3 "$SHOT" "$sdir/install-hung.png" 127.0.0.1 "59$VNC_DISPLAY" || true
      echo "FAIL: $scenario hung after ${INSTALL_TIMEOUT}s" >&2
      return 1
    fi
    sleep 20
    waited=$((waited + 20))
  done
  trap - RETURN
  echo "    VM exited after ~${waited}s"

  # Pull install.log from the config vfat (guest wrote it under /m).
  mcopy -n -i "$cfgimg" ::install.log "$log" 2>/dev/null || true
  if [[ ! -s "$log" ]]; then
    echo "FAIL: $scenario — no install.log recovered from config disk" >&2
    return 1
  fi
  echo "    log: $log ($(wc -l <"$log") lines)"

  case "$scenario" in
    disk-warn)
      if ! grep -qiE 'Windows/BitLocker/NTFS or LUKS' "$log"; then
        echo "FAIL: disk-warn — expected signature warn missing" >&2
        tail -40 "$log" >&2 || true
        return 1
      fi
      echo "PASS disk-warn: signature warn present"
      ;;
    compose-fail)
      if ! grep -q 'Offline lock composition failed and there is no network to fall back to' "$log"; then
        echo "FAIL: compose-fail — offline fail-closed message missing" >&2
        tail -60 "$log" >&2 || true
        return 1
      fi
      if grep -qE 'resolving over the network|flake lock' "$log"; then
        # "flake lock" only as a warning path — must not succeed online.
        if grep -q 'Offline lock composition failed — resolving over the network' "$log"; then
          echo "FAIL: compose-fail — took the ONLINE fallback while offline" >&2
          return 1
        fi
      fi
      if ! grep -q 'INSTALL_RC=1' "$log"; then
        echo "FAIL: compose-fail — expected INSTALL_RC=1" >&2
        grep INSTALL_RC "$log" >&2 || true
        return 1
      fi
      echo "PASS compose-fail: offline fail-closed (rc=1, no network fallback)"
      ;;
    hm-hint)
      mcopy -n -i "$cfgimg" ::hint.txt "$sdir/hint.txt" 2>/dev/null || true
      if [[ ! -f "$sdir/hint.txt" ]]; then
        # Fall back: after poweroff, try reading the target filesystem (LUKS).
        echo "FAIL: hm-hint — hint.txt not copied from /mnt during install" >&2
        tail -40 "$log" >&2 || true
        return 1
      fi
      if ! grep -q 'home-manager switch --flake ~/.nomarchy -b bak' "$sdir/hint.txt"; then
        echo "FAIL: hm-hint — recovery command missing from hint file" >&2
        cat "$sdir/hint.txt" >&2
        return 1
      fi
      if ! grep -q 'Desktop pre-activation failed' "$sdir/hint.txt"; then
        echo "FAIL: hm-hint — unexpected hint contents" >&2
        cat "$sdir/hint.txt" >&2
        return 1
      fi
      echo "PASS hm-hint: NOMARCHY-DESKTOP-NOT-THEMED.txt written on target home"
      ;;
  esac
  return 0
}

failed=0
for s in "${SCENARIOS[@]}"; do
  if ! run_scenario "$s"; then
    failed=1
  fi
done

echo
if (( failed )); then
  echo "FAIL: one or more #54 safety scenarios failed (logs under $VM_DIR)" >&2
  exit 1
fi
echo "PASS: all requested #54 safety scenarios"
echo "Artifacts: $VM_DIR"