|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Runtime evidence for desktop quit releasing the backend's single-writer lock |
| 3 | +# (<T3CODE_HOME>/userdata/server-owner.sqlite) before Electron exits. |
| 4 | +# |
| 5 | +# Usage: apps/desktop/scripts/quit-lock-evidence.sh <checkout> <fixture> <run-dir> |
| 6 | +# checkout repo root with `vp run build:desktop` done (apps/server/dist + apps/desktop/dist-electron) |
| 7 | +# fixture F1 SIGTERM to the Electron main process, idle backend |
| 8 | +# F2 UI quit: window.close() over CDP (window-all-closed -> app.quit) |
| 9 | +# F3 backend SIGSTOPped (ignores SIGTERM), then SIGTERM to Electron |
| 10 | +# F4 backend SIGSTOPped, SIGTERM to Electron, SIGCONT after $F4_DELAY (3s) |
| 11 | +# F12 backend SIGKILLed before quit, then SIGTERM to Electron |
| 12 | +# run-dir fresh directory for the isolated T3CODE_HOME and XDG dirs (never ~/.t3) |
| 13 | +# |
| 14 | +# Needs headless-desktop (xvfb-run + dbus + keyring), lsof, fuser, node >= 22 (node:sqlite). |
| 15 | +# After Electron exits it records the lock holders, a BEGIN EXCLUSIVE probe, and then |
| 16 | +# relaunches on the same home to check the new backend acquires the lock. It only signals |
| 17 | +# PIDs it spawned or found under its own wrapper whose environ has this run's T3CODE_HOME. |
| 18 | +set -uo pipefail |
| 19 | + |
| 20 | +CHECKOUT=$(cd "${1:?checkout}" && pwd) |
| 21 | +FIXTURE=${2:?fixture} |
| 22 | +RUN=$(mkdir -p "${3:?run-dir}" && cd "$3" && pwd) |
| 23 | +F4_DELAY=${F4_DELAY:-3} |
| 24 | +CDP_PORT=${CDP_PORT:-9333} |
| 25 | +H=$RUN/home |
| 26 | +L=$H/userdata/server-owner.sqlite |
| 27 | +RT=$H/userdata/server-runtime.json |
| 28 | +DESKTOP=$CHECKOUT/apps/desktop |
| 29 | +ELECTRON=$(realpath "$DESKTOP/node_modules/electron/dist/electron") |
| 30 | +SEEN_PIDS=() |
| 31 | + |
| 32 | +mkdir -p "$RUN" |
| 33 | +[ -z "$(ls -A "$RUN" 2>/dev/null)" ] || { echo "run-dir must be empty: $RUN" >&2; exit 2; } |
| 34 | +mkdir -p "$H" "$RUN/xdg/config" "$RUN/xdg/data" |
| 35 | + |
| 36 | +ts() { date +%T.%N; } |
| 37 | +log() { echo "[$(ts)] $*"; } |
| 38 | +now() { date +%s.%N; } |
| 39 | +elapsed() { awk -v a="$1" -v b="$2" 'BEGIN{printf "%.3fs", b-a}'; } |
| 40 | + |
| 41 | +descendants() { |
| 42 | + ps -e -o pid=,ppid= | awk -v root="$1" ' |
| 43 | + { kids[$2]=kids[$2] " " $1 } |
| 44 | + END { q=root; while (q != "") { split(q, a, " "); q=""; for (i in a) { n=split(kids[a[i]], k, " "); for (j=1;j<=n;j++) { print k[j]; q=q " " k[j] } } } }' |
| 45 | +} |
| 46 | + |
| 47 | +home_of() { tr '\0' '\n' 2>/dev/null < "/proc/$1/environ" | sed -n 's/^T3CODE_HOME=//p'; } |
| 48 | +ours() { [ "$(home_of "$1")" = "$H" ]; } |
| 49 | + |
| 50 | +# Process table: everything under the wrapper plus every member of the backend group. |
| 51 | +ptable() { |
| 52 | + local pids |
| 53 | + pids=$( { descendants "$WRAPPER"; [ -n "${BACKEND_PGID:-}" ] && ps -e -o pid=,pgid= | awk -v g="$BACKEND_PGID" '$2==g{print $1}'; } | sort -un | paste -sd, -) |
| 54 | + [ -n "$pids" ] || { echo " (no processes)"; return; } |
| 55 | + # args reads /proc/<pid>/cmdline, which can stall behind a process in D state on a loaded box. |
| 56 | + timeout 5 ps -o pid,ppid,pgid,stat,etime,args -p "$pids" | cut -c1-200 | sed 's/^/ /' |
| 57 | +} |
| 58 | + |
| 59 | +lockstate() { |
| 60 | + log "lsof $L:"; lsof "$L" 2>/dev/null | cut -c1-160 | sed 's/^/ /' || true |
| 61 | + log "fuser:"; fuser -v "$L" 2>&1 | sed 's/^/ /' || true |
| 62 | + log "P1 probe: $(node -e 'const {DatabaseSync}=require("node:sqlite");const d=new DatabaseSync(process.argv[1]); |
| 63 | +try{d.exec("PRAGMA busy_timeout=0; BEGIN EXCLUSIVE;");d.exec("ROLLBACK");console.log("LOCK FREE")} |
| 64 | +catch(e){console.log("LOCK HELD: "+e.message)}' "$L")" |
| 65 | + log "server-runtime.json: $(cat "$RT" 2>/dev/null || echo '(absent)')" |
| 66 | +} |
| 67 | + |
| 68 | +launch() { # $1 = label |
| 69 | + local extra=() |
| 70 | + [ "$FIXTURE" = F2 ] && [ "$1" = primary ] && extra=(--remote-debugging-port="$CDP_PORT") |
| 71 | + (cd "$DESKTOP" && exec env -u VITE_DEV_SERVER_URL -u ELECTRON_RUN_AS_NODE -u T3CODE_PORT \ |
| 72 | + T3CODE_HOME="$H" XDG_CONFIG_HOME="$RUN/xdg/config" XDG_DATA_HOME="$RUN/xdg/data" \ |
| 73 | + T3CODE_DISABLE_AUTO_UPDATE=1 ELECTRON_ENABLE_LOGGING=1 \ |
| 74 | + headless-desktop "$ELECTRON" --no-sandbox --disable-gpu "${extra[@]}" dist-electron/boot.cjs \ |
| 75 | + > "$RUN/$1-stdout.log" 2>&1) & |
| 76 | + WRAPPER=$! |
| 77 | + MAIN=""; BACKEND=""; BACKEND_PGID="" |
| 78 | + log "[$1] spawned wrapper pid=$WRAPPER" |
| 79 | + local i p |
| 80 | + for i in $(seq 1 120); do |
| 81 | + for p in $(descendants "$WRAPPER"); do |
| 82 | + local args; args=$(tr '\0' ' ' 2>/dev/null < "/proc/$p/cmdline") |
| 83 | + case "$args" in |
| 84 | + "$ELECTRON "*boot.cjs*) [[ "$args" == *--type=* ]] || MAIN=$p ;; |
| 85 | + *apps/server/dist/bin.mjs*) ours "$p" && BACKEND=$p ;; |
| 86 | + esac |
| 87 | + done |
| 88 | + [ -n "$MAIN" ] && [ -n "$BACKEND" ] && lsof -t "$L" 2>/dev/null | grep -qx "$BACKEND" && break |
| 89 | + sleep 0.5 |
| 90 | + done |
| 91 | + [ -n "$BACKEND" ] && ! lsof -t "$L" 2>/dev/null | grep -qx "$BACKEND" && { log "[$1] backend $BACKEND never acquired the lock (holders: $(lsof -t "$L" 2>/dev/null | paste -sd, -))"; return 1; } |
| 92 | + if [ -z "$MAIN" ] || [ -z "$BACKEND" ]; then log "[$1] FAILED to find main=$MAIN backend=$BACKEND"; return 1; fi |
| 93 | + BACKEND_PGID=$(ps -o pgid= -p "$BACKEND" | tr -d ' ') |
| 94 | + BACKEND_PGID=$(ps -o pgid= -p "$BACKEND" | tr -d ' ') |
| 95 | + SEEN_PIDS+=("$BACKEND") |
| 96 | + log "[$1] electron main pid=$MAIN pgid=$(ps -o pgid= -p "$MAIN" | tr -d ' ') backend pid=$BACKEND pgid=$BACKEND_PGID T3CODE_HOME=$(home_of "$BACKEND")" |
| 97 | +} |
| 98 | + |
| 99 | +wait_main_exit() { # samples the table every 0.5s until the Electron main process is gone |
| 100 | + local i=0 |
| 101 | + while kill -0 "$MAIN" 2>/dev/null; do |
| 102 | + if (( i % 5 == 0 )); then log "sample (main alive):"; ptable; fi |
| 103 | + i=$((i+1)); sleep 0.1 |
| 104 | + done |
| 105 | +} |
| 106 | + |
| 107 | +relaunch_check() { |
| 108 | + local old_owner; old_owner=$(sed -n 's/.*"ownerId":"\([^"]*\)".*/\1/p' "$RT" 2>/dev/null) |
| 109 | + log "P2 relaunch on the same home (previous ownerId=${old_owner:-none})" |
| 110 | + rm -f "$RUN/relaunch.ok" |
| 111 | + if launch relaunch; then |
| 112 | + local owner; owner=$(sed -n 's/.*"ownerId":"\([^"]*\)".*/\1/p' "$RT") |
| 113 | + log "P2 relaunch: backend pid=$BACKEND holds lock: $(lsof -t "$L" 2>/dev/null | paste -sd, -) ownerId=$owner" |
| 114 | + else |
| 115 | + log "P2 relaunch: no backend acquired the lock within 60s" |
| 116 | + fi |
| 117 | + log "P2 'already running' lines in relaunch logs: $(grep -ci 'already running\|ServerAlreadyRunning' "$RUN/relaunch-stdout.log" "$H/userdata/logs/server.trace.ndjson" 2>/dev/null | paste -sd' ' -)" |
| 118 | + grep -i 'already running\|ServerAlreadyRunning' "$RUN/relaunch-stdout.log" 2>/dev/null | head -3 | cut -c1-240 | sed 's/^/ /' |
| 119 | + if [ -n "$MAIN" ]; then |
| 120 | + log "stopping relaunch: SIGTERM electron main $MAIN"; kill -TERM "$MAIN" |
| 121 | + wait_main_exit; wait "$WRAPPER" 2>/dev/null |
| 122 | + log "relaunch exited"; lockstate |
| 123 | + else |
| 124 | + kill -TERM "$WRAPPER" 2>/dev/null; wait "$WRAPPER" 2>/dev/null |
| 125 | + fi |
| 126 | +} |
| 127 | + |
| 128 | +cleanup_orphans() { |
| 129 | + local p |
| 130 | + for p in "${SEEN_PIDS[@]}"; do |
| 131 | + if kill -0 "$p" 2>/dev/null && ours "$p"; then |
| 132 | + log "CLEANUP: captured backend pid=$p still alive ($(ps -o stat=,ppid= -p "$p")); SIGCONT+SIGKILL" |
| 133 | + kill -CONT "$p"; kill -KILL "$p" |
| 134 | + fi |
| 135 | + done |
| 136 | + sleep 0.5 |
| 137 | + local leftover |
| 138 | + # Electron's setproctitle clobbers its own /proc/<pid>/environ, so also list the wrapper's tree. |
| 139 | + leftover=$( { descendants "$WRAPPER"; for p in /proc/[0-9]*; do [ "$(home_of "${p#/proc/}")" = "$H" ] && echo "${p#/proc/}"; done; } | sort -un | paste -sd, -) |
| 140 | + log "leftover processes with T3CODE_HOME=$H: ${leftover:-none}" |
| 141 | +} |
| 142 | + |
| 143 | +cdp_close() { |
| 144 | + node -e ' |
| 145 | +const port = process.argv[1]; |
| 146 | +(async () => { |
| 147 | + const targets = await (await fetch(`http://127.0.0.1:${port}/json`)).json(); |
| 148 | + const page = targets.find((t) => t.type === "page" && !t.url.startsWith("devtools")); |
| 149 | + console.log("CDP page:", page.url); |
| 150 | + const ws = new WebSocket(page.webSocketDebuggerUrl); |
| 151 | + ws.onopen = () => ws.send(JSON.stringify({ id: 1, method: "Runtime.evaluate", params: { expression: "window.close()" } })); |
| 152 | + ws.onmessage = () => { console.log("CDP window.close() sent"); process.exit(0); }; |
| 153 | + ws.onclose = () => { console.log("CDP socket closed"); process.exit(0); }; |
| 154 | + setTimeout(() => process.exit(0), 3000); |
| 155 | +})();' "$CDP_PORT" |
| 156 | +} |
| 157 | + |
| 158 | +echo "fixture=$FIXTURE checkout=$CHECKOUT head=$(git -C "$CHECKOUT" rev-parse HEAD) run=$RUN" |
| 159 | +echo "host=$(hostname) kernel=$(uname -r) node=$(node -v) electron=v$(cat "$(dirname "$ELECTRON")/version") cpus=$(nproc)" |
| 160 | +launch primary || { [ -n "$MAIN" ] && kill -TERM "$MAIN"; sleep 20; cleanup_orphans; exit 1; } |
| 161 | +[ -n "${SCREENSHOT:-}" ] && { |
| 162 | + sleep 4 |
| 163 | + # Electron's environ is clobbered, so take the display and auth file from Xvfb's argv. |
| 164 | + xvfb=$(for p in $(descendants "$WRAPPER"); do [ "$(cat /proc/$p/comm 2>/dev/null)" = Xvfb ] && tr '\0' ' ' < /proc/$p/cmdline; done) |
| 165 | + DISPLAY=$(awk '{print $2}' <<< "$xvfb") XAUTHORITY=$(sed -n 's/.*-auth \([^ ]*\).*/\1/p' <<< "$xvfb") \ |
| 166 | + import -window root "$SCREENSHOT" && log "screenshot: $SCREENSHOT" |
| 167 | +} |
| 168 | +log "running state:"; ptable; lockstate |
| 169 | + |
| 170 | +case "$FIXTURE" in |
| 171 | + F1) t0=$(now); log "QUIT: SIGTERM electron main $MAIN"; kill -TERM "$MAIN" ;; |
| 172 | + F2) t0=$(now); log "QUIT: CDP window.close()"; cdp_close 2>&1 | sed 's/^/ /' ;; |
| 173 | + F3) log "SIGSTOP backend $BACKEND"; kill -STOP "$BACKEND"; sleep 0.3 |
| 174 | + t0=$(now); log "QUIT: SIGTERM electron main $MAIN"; kill -TERM "$MAIN" ;; |
| 175 | + F4) log "SIGSTOP backend $BACKEND"; kill -STOP "$BACKEND"; sleep 0.3 |
| 176 | + t0=$(now); log "QUIT: SIGTERM electron main $MAIN"; kill -TERM "$MAIN" |
| 177 | + (sleep "$F4_DELAY"; log "SIGCONT backend $BACKEND (t0+${F4_DELAY}s)"; kill -CONT "$BACKEND") & ;; |
| 178 | + F12) log "SIGKILL backend $BACKEND before quit"; kill -KILL "$BACKEND" |
| 179 | + for _ in $(seq 1 40); do kill -0 "$BACKEND" 2>/dev/null || break; sleep 0.1; done |
| 180 | + sleep 2; log "after backend kill:"; ptable |
| 181 | + for p in $(descendants "$MAIN"); do |
| 182 | + [[ "$(tr '\0' ' ' 2>/dev/null < /proc/$p/cmdline)" == *apps/server/dist/bin.mjs* ]] && ours "$p" \ |
| 183 | + && { log "backend respawned as pid=$p"; SEEN_PIDS+=("$p"); BACKEND_PGID=$(ps -o pgid= -p "$p" | tr -d ' '); } |
| 184 | + done |
| 185 | + t0=$(now); log "QUIT: SIGTERM electron main $MAIN"; kill -TERM "$MAIN" ;; |
| 186 | + *) echo "unknown fixture $FIXTURE" >&2; exit 2 ;; |
| 187 | +esac |
| 188 | + |
| 189 | +wait_main_exit |
| 190 | +t1=$(now) |
| 191 | +log "ELECTRON MAIN EXITED after $(elapsed "$t0" "$t1")" |
| 192 | +log "table at electron exit:"; ptable |
| 193 | +lockstate |
| 194 | +wait "$WRAPPER" 2>/dev/null; wait |
| 195 | +log "wrapper exited after $(elapsed "$t0" "$(now)")" |
| 196 | +log "backend $BACKEND alive at wrapper exit: $(kill -0 "$BACKEND" 2>/dev/null && ps -o stat=,ppid= -p "$BACKEND" || echo no)" |
| 197 | +log "desktop log quit lines:" |
| 198 | +grep -iE 'before-quit|shutdown|exceeded|backend.*(exit|stop)|signal' "$RUN/primary-stdout.log" | tail -n 12 | cut -c1-200 | sed 's/^/ /' |
| 199 | +relaunch_check |
| 200 | +cleanup_orphans |
0 commit comments