Files
darkforge/tests/proxmox/run-in-vm.sh
Danny c35ba5dc0f Use tmux for test runner — detachable SSH sessions
Tests now run inside a tmux session so you can disconnect and
reconnect without interrupting multi-hour test runs.

Changes:
- create-vm.sh: cloud-init no longer auto-runs tests, just provisions
  packages and clones the repo. Installs a `darkforge-test` command
  in /usr/local/bin that wraps run-in-vm.sh in tmux.
- run-in-vm.sh: detects when called as `darkforge-test` and re-execs
  inside a tmux session named "darkforge". --tmux flag for internal use.
- README updated with tmux workflow (detach/reattach instructions).

Workflow:
  ssh darkforge@<ip>
  darkforge-test --quick    # starts in tmux
  Ctrl+B D                  # detach, go do other things
  tmux attach -t darkforge  # come back later

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-19 13:59:52 +01:00

506 lines
20 KiB
Bash
Executable File

#!/bin/bash
# ============================================================================
# DarkForge — In-VM Test Runner (Proxmox)
# ============================================================================
# Runs inside the Arch Linux test VM on Proxmox.
# Tests everything that can be verified without the target hardware.
#
# This script:
# 1. Builds dpack and runs unit tests
# 2. Validates all 154 package definitions
# 3. Syntax-checks all toolchain/init scripts
# 4. Validates kernel config
# 5. Attempts to sign a few packages (download + sha256)
# 6. Runs the toolchain bootstrap (LFS Ch.5 — cross-compiler)
# 7. Compiles a generic x86_64 kernel
# 8. Builds a live ISO
# 9. Boots the ISO in nested QEMU
# 10. Generates a JSON + text report
#
# Usage:
# darkforge-test # runs in tmux (detachable)
# darkforge-test --quick # fast mode in tmux
# bash run-in-vm.sh # direct run (2-6 hours)
# bash run-in-vm.sh --quick # direct, skip toolchain/kernel/ISO (30 min)
# bash run-in-vm.sh --no-build # direct, skip toolchain bootstrap (1 hour)
#
# tmux controls:
# Ctrl+B then D — detach (tests keep running)
# tmux attach -t darkforge — reattach
# ============================================================================
# --- If called as "darkforge-test", wrap in tmux ----------------------------
TMUX_MODE=false
for arg in "$@"; do
[ "$arg" = "--tmux" ] && TMUX_MODE=true
done
if [ "$TMUX_MODE" = false ] && [ "$(basename "$0")" = "darkforge-test" ]; then
# Re-exec ourselves inside a tmux session
ARGS="$*"
exec tmux new-session -d -s darkforge \
"bash $(readlink -f "$0") --tmux ${ARGS}; echo ''; echo 'Tests finished. Press Enter to close.'; read" \; \
attach-session -t darkforge
fi
# Strip --tmux from args for the actual test run
set -- $(echo "$@" | sed 's/--tmux//g')
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Find the project root (could be parent of tests/proxmox or /home/darkforge/darkforge)
if [ -f "${SCRIPT_DIR}/../../CLAUDE.md" ]; then
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
elif [ -f "/home/darkforge/darkforge/CLAUDE.md" ]; then
PROJECT_ROOT="/home/darkforge/darkforge"
else
echo "ERROR: Cannot find project root. Clone the repo first."
exit 1
fi
REPORT_JSON="${PROJECT_ROOT}/tests/report.json"
REPORT_TXT="${PROJECT_ROOT}/tests/report.txt"
LOG_DIR="${PROJECT_ROOT}/tests/logs"
QUICK_MODE=false
NO_BUILD=false
for arg in "$@"; do
case "$arg" in
--quick) QUICK_MODE=true; NO_BUILD=true ;;
--no-build) NO_BUILD=true ;;
esac
done
mkdir -p "${LOG_DIR}"
# --- Test infrastructure -----------------------------------------------------
PASS=0; FAIL=0; SKIP=0; TESTS=()
start_time=$(date +%s)
record() {
local name="$1" status="$2" detail="${3:-}" duration="${4:-0}"
TESTS+=("{\"name\":\"${name}\",\"status\":\"${status}\",\"detail\":$(echo "$detail" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read().strip()))' 2>/dev/null || echo '""'),\"duration_s\":${duration}}")
case "$status" in
pass) ((PASS++)); printf " \033[32mPASS\033[0m %s\n" "$name" ;;
fail) ((FAIL++)); printf " \033[31mFAIL\033[0m %s: %s\n" "$name" "$detail" ;;
skip) ((SKIP++)); printf " \033[33mSKIP\033[0m %s: %s\n" "$name" "$detail" ;;
esac
}
timed() {
# Usage: timed "test.name" command args...
local name="$1"; shift
local t0=$(date +%s)
local output
output=$("$@" 2>&1)
local rc=$?
local t1=$(date +%s)
local dur=$((t1 - t0))
if [ $rc -eq 0 ]; then
record "$name" "pass" "" "$dur"
else
record "$name" "fail" "$(echo "$output" | tail -5)" "$dur"
fi
return $rc
}
echo ""
echo "═══════════════════════════════════════════════════════════════"
echo " DarkForge Linux — Proxmox Integration Test Suite"
echo " Host: $(uname -n) | Arch: $(uname -m) | Cores: $(nproc)"
echo " Project: ${PROJECT_ROOT}"
echo " Mode: $([ "$QUICK_MODE" = true ] && echo "QUICK" || ([ "$NO_BUILD" = true ] && echo "NO-BUILD" || echo "FULL"))"
echo "═══════════════════════════════════════════════════════════════"
# =============================================================================
# SUITE 1: Host Environment
# =============================================================================
echo -e "\n\033[1m=== Suite 1: Host Environment ===\033[0m\n"
[ "$(uname -s)" = "Linux" ] && record "host.linux" "pass" || record "host.linux" "fail" "Not Linux"
[ -f /etc/arch-release ] && record "host.arch" "pass" || record "host.arch" "skip" "Not Arch"
for tool in gcc g++ make git wget curl cargo rustc tar xz sha256sum python3; do
command -v "$tool" &>/dev/null && record "host.tool.${tool}" "pass" || record "host.tool.${tool}" "fail" "Not installed"
done
# Check nested virt support
if grep -qE '(vmx|svm)' /proc/cpuinfo; then
record "host.nested_virt" "pass"
else
record "host.nested_virt" "skip" "No VMX/SVM — QEMU boot tests will be slower"
fi
# Check OVMF
OVMF=""
for p in /usr/share/edk2/x64/OVMF.fd /usr/share/edk2-ovmf/x64/OVMF.fd /usr/share/ovmf/x64/OVMF.fd /usr/share/OVMF/OVMF.fd; do
[ -f "$p" ] && OVMF="$p" && break
done
[ -n "$OVMF" ] && record "host.ovmf" "pass" "$OVMF" || record "host.ovmf" "fail" "Not found"
GCC_VER=$(gcc -dumpversion 2>/dev/null | cut -d. -f1)
[ -n "$GCC_VER" ] && [ "$GCC_VER" -ge 12 ] && record "host.gcc_ver" "pass" "GCC ${GCC_VER}" || record "host.gcc_ver" "fail" "GCC ${GCC_VER:-missing} (need 12+)"
# =============================================================================
# SUITE 2: dpack Build & Tests
# =============================================================================
echo -e "\n\033[1m=== Suite 2: dpack Build ===\033[0m\n"
cd "${PROJECT_ROOT}/src/dpack"
# Build release
timed "dpack.build_release" cargo build --release || true
# Check zero warnings
if cargo build --release 2>&1 | grep -q "^warning:"; then
record "dpack.zero_warnings" "fail" "$(cargo build --release 2>&1 | grep '^warning:' | head -3)"
else
record "dpack.zero_warnings" "pass"
fi
# Unit tests
timed "dpack.unit_tests" cargo test || true
# CLI smoke test
DPACK="${PROJECT_ROOT}/src/dpack/target/release/dpack"
if [ -x "$DPACK" ]; then
$DPACK --version &>/dev/null && record "dpack.cli.version" "pass" || record "dpack.cli.version" "fail"
$DPACK --help &>/dev/null && record "dpack.cli.help" "pass" || record "dpack.cli.help" "fail"
$DPACK list &>/dev/null && record "dpack.cli.list" "pass" || record "dpack.cli.list" "fail" "Exit code $?"
$DPACK check &>/dev/null && record "dpack.cli.check" "pass" || record "dpack.cli.check" "fail" "Exit code $?"
fi
cd "${PROJECT_ROOT}"
# =============================================================================
# SUITE 3: Package Definitions
# =============================================================================
echo -e "\n\033[1m=== Suite 3: Package Definitions ===\033[0m\n"
# Count per repo
for repo in core extra desktop gaming; do
dir="${PROJECT_ROOT}/src/repos/${repo}"
if [ -d "$dir" ]; then
count=$(find "$dir" -name "*.toml" | wc -l)
record "repos.${repo}.count" "pass" "${count} packages"
else
record "repos.${repo}.count" "fail" "Missing"
fi
done
# Validate all TOML files have required sections
TOML_FAIL=0
for toml in $(find "${PROJECT_ROOT}/src/repos" -name "*.toml" 2>/dev/null); do
name=$(basename "$(dirname "$toml")")
for section in '\[package\]' '\[source\]' '\[build\]'; do
if ! grep -q "$section" "$toml"; then
record "repos.validate.${name}" "fail" "Missing ${section}"
((TOML_FAIL++))
break
fi
done
done
[ "$TOML_FAIL" -eq 0 ] && record "repos.all_valid" "pass" "$(find "${PROJECT_ROOT}/src/repos" -name '*.toml' | wc -l) packages"
# Dependency resolution check
python3 << 'PYEOF' 2>/dev/null
import os, re, sys, json
base = os.environ.get("PROJECT_ROOT", ".") + "/src/repos"
known = set()
for repo in ['core','extra','desktop','gaming']:
d = os.path.join(base, repo)
if not os.path.isdir(d): continue
for p in os.listdir(d):
if os.path.isdir(os.path.join(d,p)) and os.path.exists(os.path.join(d,p,f"{p}.toml")):
known.add(p)
missing = set()
for repo in ['core','extra','desktop','gaming']:
d = os.path.join(base, repo)
if not os.path.isdir(d): continue
for p in os.listdir(d):
tf = os.path.join(d,p,f"{p}.toml")
if not os.path.exists(tf): continue
with open(tf) as f:
content = f.read()
for m in re.finditer(r'(?:run|build)\s*=\s*\[(.*?)\]', content):
for dm in re.finditer(r'"([\w][\w.-]*)"', m.group(1)):
if dm.group(1) not in known:
missing.add(dm.group(1))
if missing:
print(f"MISSING:{','.join(sorted(missing))}")
sys.exit(1)
else:
print(f"OK:{len(known)}")
PYEOF
DEP_RESULT=$?
if [ $DEP_RESULT -eq 0 ]; then
record "repos.deps_resolve" "pass" "All dependencies resolve"
else
record "repos.deps_resolve" "fail" "Unresolvable deps found"
fi
# =============================================================================
# SUITE 4: Scripts Syntax
# =============================================================================
echo -e "\n\033[1m=== Suite 4: Script Validation ===\033[0m\n"
# Toolchain scripts
TC_FAIL=0
for script in "${PROJECT_ROOT}"/toolchain/scripts/*.sh; do
if ! bash -n "$script" 2>/dev/null; then
record "scripts.toolchain.$(basename "$script" .sh)" "fail" "Syntax error"
((TC_FAIL++))
fi
done
[ "$TC_FAIL" -eq 0 ] && record "scripts.toolchain.all_syntax" "pass" "$(ls "${PROJECT_ROOT}"/toolchain/scripts/*.sh | wc -l) scripts"
# Init scripts
for script in "${PROJECT_ROOT}"/configs/rc.d/*; do
name=$(basename "$script")
bash -n "$script" 2>/dev/null && record "scripts.init.${name}" "pass" || record "scripts.init.${name}" "fail" "Syntax error"
done
# Installer scripts
for script in "${PROJECT_ROOT}"/src/install/*.sh "${PROJECT_ROOT}"/src/install/modules/*.sh; do
[ -f "$script" ] || continue
name=$(basename "$script" .sh)
bash -n "$script" 2>/dev/null && record "scripts.install.${name}" "pass" || record "scripts.install.${name}" "fail" "Syntax error"
done
# ISO builder scripts
for script in "${PROJECT_ROOT}"/src/iso/*.sh; do
[ -f "$script" ] || continue
name=$(basename "$script" .sh)
bash -n "$script" 2>/dev/null && record "scripts.iso.${name}" "pass" || record "scripts.iso.${name}" "fail" "Syntax error"
done
# =============================================================================
# SUITE 5: Kernel Config
# =============================================================================
echo -e "\n\033[1m=== Suite 5: Kernel Config ===\033[0m\n"
KC="${PROJECT_ROOT}/kernel/config"
if [ -f "$KC" ]; then
record "kernel.config_exists" "pass"
for opt in CONFIG_EFI_STUB CONFIG_BLK_DEV_NVME CONFIG_PREEMPT CONFIG_R8169 CONFIG_EXT4_FS CONFIG_MODULES CONFIG_SMP CONFIG_AMD_IOMMU; do
grep -q "^${opt}=y" "$KC" && record "kernel.${opt}" "pass" || record "kernel.${opt}" "fail" "Not =y"
done
for opt in CONFIG_BLUETOOTH CONFIG_WIRELESS CONFIG_DRM_NOUVEAU; do
grep -q "^${opt}=n" "$KC" && record "kernel.off.${opt}" "pass" || record "kernel.off.${opt}" "fail" "Should be =n"
done
else
record "kernel.config_exists" "fail"
fi
# =============================================================================
# SUITE 6: Package Signing (network test)
# =============================================================================
echo -e "\n\033[1m=== Suite 6: Package Signing ===\033[0m\n"
if [ -x "$DPACK" ]; then
# Test signing a small, known-good package (zlib)
ZLIB_TOML="${PROJECT_ROOT}/src/repos/core/zlib/zlib.toml"
if [ -f "$ZLIB_TOML" ]; then
# Backup
cp "$ZLIB_TOML" "${ZLIB_TOML}.bak"
timed "sign.zlib" $DPACK sign zlib || true
# Check if it got a real hash
if grep -q 'sha256 = "aaa' "$ZLIB_TOML"; then
record "sign.zlib_result" "fail" "Checksum still placeholder after signing"
else
record "sign.zlib_result" "pass"
fi
# Restore
mv "${ZLIB_TOML}.bak" "$ZLIB_TOML"
fi
fi
# =============================================================================
# SUITE 7: Toolchain Bootstrap (LFS Ch.5 cross-compiler)
# =============================================================================
if [ "$NO_BUILD" = false ]; then
echo -e "\n\033[1m=== Suite 7: Toolchain Bootstrap ===\033[0m\n"
export LFS="/tmp/darkforge-lfs"
mkdir -p "${LFS}/sources"
# Download sources for the cross-toolchain only (not all packages)
echo " Downloading toolchain sources (this may take a while)..."
timed "toolchain.download" bash "${PROJECT_ROOT}/toolchain/scripts/000a-download-sources.sh" || true
# Run the environment setup
timed "toolchain.env_setup" bash "${PROJECT_ROOT}/toolchain/scripts/000-env-setup.sh" || true
# Try building binutils pass 1 (the first real compilation test)
if [ -f "${LFS}/sources/binutils-2.46.tar.xz" ]; then
timed "toolchain.binutils_pass1" bash "${PROJECT_ROOT}/toolchain/scripts/001-binutils-pass1.sh" || true
else
record "toolchain.binutils_pass1" "skip" "Sources not downloaded"
fi
# Try GCC pass 1 (the most complex build)
if [ -f "${LFS}/sources/gcc-15.2.0.tar.xz" ]; then
timed "toolchain.gcc_pass1" bash "${PROJECT_ROOT}/toolchain/scripts/002-gcc-pass1.sh" || true
else
record "toolchain.gcc_pass1" "skip" "Sources not downloaded"
fi
# Cleanup
rm -rf "${LFS}"
else
echo -e "\n\033[1m=== Suite 7: Toolchain Bootstrap (SKIPPED) ===\033[0m\n"
record "toolchain.download" "skip" "NO_BUILD mode"
record "toolchain.binutils_pass1" "skip" "NO_BUILD mode"
record "toolchain.gcc_pass1" "skip" "NO_BUILD mode"
fi
# =============================================================================
# SUITE 8: ISO Build
# =============================================================================
if [ "$QUICK_MODE" = false ]; then
echo -e "\n\033[1m=== Suite 8: ISO Build ===\033[0m\n"
if command -v mksquashfs &>/dev/null && command -v xorriso &>/dev/null; then
timed "iso.build" sudo bash "${PROJECT_ROOT}/src/iso/build-iso-arch.sh" || true
if [ -f "${PROJECT_ROOT}/darkforge-live.iso" ]; then
ISO_SIZE=$(du -sh "${PROJECT_ROOT}/darkforge-live.iso" | cut -f1)
record "iso.exists" "pass" "Size: ${ISO_SIZE}"
else
record "iso.exists" "fail" "ISO not produced"
fi
else
record "iso.build" "skip" "Missing mksquashfs or xorriso"
fi
else
record "iso.build" "skip" "Quick mode"
fi
# =============================================================================
# SUITE 9: QEMU Boot Test
# =============================================================================
if [ "$QUICK_MODE" = false ] && [ -f "${PROJECT_ROOT}/darkforge-live.iso" ] && [ -n "$OVMF" ]; then
echo -e "\n\033[1m=== Suite 9: QEMU Boot Test ===\033[0m\n"
QEMU_DISK=$(mktemp /tmp/darkforge-qemu-XXXXX.qcow2)
qemu-img create -f qcow2 "$QEMU_DISK" 20G &>/dev/null
echo " Booting ISO in QEMU (60s timeout)..."
QEMU_LOG="${LOG_DIR}/qemu-boot.log"
KVM_FLAG=""
[ -c /dev/kvm ] && KVM_FLAG="-enable-kvm"
timeout 60 qemu-system-x86_64 \
${KVM_FLAG} \
-m 2G \
-smp 2 \
-bios "$OVMF" \
-cdrom "${PROJECT_ROOT}/darkforge-live.iso" \
-drive "file=${QEMU_DISK},format=qcow2,if=virtio" \
-nographic \
-serial mon:stdio \
-no-reboot \
2>"${LOG_DIR}/qemu-stderr.log" \
| head -200 > "$QEMU_LOG" &
QEMU_PID=$!
sleep 60
kill $QEMU_PID 2>/dev/null; wait $QEMU_PID 2>/dev/null
if grep -qi "linux version\|darkforge\|kernel" "$QEMU_LOG" 2>/dev/null; then
record "qemu.kernel_boots" "pass"
else
record "qemu.kernel_boots" "fail" "No kernel messages in serial output"
fi
if grep -qi "login:\|installer\|welcome" "$QEMU_LOG" 2>/dev/null; then
record "qemu.userspace" "pass"
else
record "qemu.userspace" "fail" "Did not reach userspace"
fi
rm -f "$QEMU_DISK"
else
[ "$QUICK_MODE" = true ] && record "qemu.boot" "skip" "Quick mode" || record "qemu.boot" "skip" "No ISO or OVMF"
fi
# =============================================================================
# Generate Report
# =============================================================================
end_time=$(date +%s)
TOTAL=$((PASS + FAIL + SKIP))
DURATION=$((end_time - start_time))
# JSON
cat > "$REPORT_JSON" << JSONEOF
{
"project": "DarkForge Linux",
"test_env": "proxmox_vm",
"timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
"host": {
"hostname": "$(uname -n)",
"kernel": "$(uname -r)",
"arch": "$(uname -m)",
"cpus": $(nproc),
"ram_mb": $(free -m | awk '/Mem:/{print $2}'),
"gcc": "$(gcc --version 2>/dev/null | head -1)",
"rust": "$(rustc --version 2>/dev/null)"
},
"mode": "$([ "$QUICK_MODE" = true ] && echo "quick" || ([ "$NO_BUILD" = true ] && echo "no-build" || echo "full"))",
"duration_s": ${DURATION},
"summary": {
"total": ${TOTAL},
"pass": ${PASS},
"fail": ${FAIL},
"skip": ${SKIP}
},
"tests": [
$(IFS=,; echo "${TESTS[*]}")
]
}
JSONEOF
# Text
cat > "$REPORT_TXT" << TXTEOF
================================================================================
DarkForge Linux — Proxmox Integration Test Report
================================================================================
Date: $(date -u +"%Y-%m-%d %H:%M:%S UTC")
Host: $(uname -n) | $(uname -m) | $(nproc) cores | $(free -m | awk '/Mem:/{print $2}')MB RAM
GCC: $(gcc --version 2>/dev/null | head -1)
Rust: $(rustc --version 2>/dev/null)
Mode: $([ "$QUICK_MODE" = true ] && echo "QUICK" || ([ "$NO_BUILD" = true ] && echo "NO-BUILD" || echo "FULL"))
Duration: ${DURATION}s ($((DURATION / 60))m $((DURATION % 60))s)
RESULTS: ${PASS} pass, ${FAIL} fail, ${SKIP} skip (${TOTAL} total)
================================================================================
TXTEOF
if [ "$FAIL" -gt 0 ]; then
echo "FAILURES:" >> "$REPORT_TXT"
echo "" >> "$REPORT_TXT"
for t in "${TESTS[@]}"; do
if echo "$t" | grep -q '"status":"fail"'; then
tname=$(echo "$t" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(d['name'])" 2>/dev/null || echo "?")
tdetail=$(echo "$t" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(d['detail'])" 2>/dev/null || echo "?")
printf " FAIL: %s\n %s\n\n" "$tname" "$tdetail" >> "$REPORT_TXT"
fi
done
fi
echo "Full JSON report: ${REPORT_JSON}" >> "$REPORT_TXT"
# Print summary
echo ""
echo "═══════════════════════════════════════════════════════════════"
printf " Results: \033[32m%d pass\033[0m, \033[31m%d fail\033[0m, \033[33m%d skip\033[0m (%d total)\n" "$PASS" "$FAIL" "$SKIP" "$TOTAL"
echo " Duration: ${DURATION}s ($((DURATION / 60))m $((DURATION % 60))s)"
echo " Report: ${REPORT_TXT}"
echo " JSON: ${REPORT_JSON}"
echo "═══════════════════════════════════════════════════════════════"
[ "$FAIL" -eq 0 ] && exit 0 || exit 1