|
| 1 | +#!/usr/bin/env bash |
| 2 | +# A/B: does delivering the AILANG system prompt in the PERSISTENT SYSTEM ROLE |
| 3 | +# (vs folding it into the user message, where compaction strips it on long runs) |
| 4 | +# improve convergence? This is the clean test that was never valid before — the |
| 5 | +# prior 2026-06-18 "-2/18, not the lever" A/B ran while the prompt was chars=0 |
| 6 | +# (see memory project_motoko_system_prompt_delivery / _known). |
| 7 | +# |
| 8 | +# Control = AILANG_MOTOKO_SYSTEM_ROLE=0 -> legacy fold-into-user-message |
| 9 | +# (the OLD behaviour: teaching lost to compaction on long runs) |
| 10 | +# Treatment = AILANG_MOTOKO_SYSTEM_ROLE=1 -> persistent system-role delivery |
| 11 | +# (the new default; SYSTEM_MD survives compaction) |
| 12 | +# |
| 13 | +# The end-to-end delivery guard (M-RIG-RELIABILITY) records system_md per run, |
| 14 | +# so we CONFIRM the arms actually differ: treatment must be 'set', control 'unset'. |
| 15 | +# |
| 16 | +# Usage: tools/ab_system_role.sh [docx|BENCH,LIST] [TRIALS] |
| 17 | +# docx (default) -> docx_reimplement (the frontier; ~41min/run) |
| 18 | +# |
| 19 | +# Requires the rig FREE (single motoko env-server on :8080). Run from repo root. |
| 20 | +set -euo pipefail |
| 21 | +cd "$(dirname "$0")/.." |
| 22 | + |
| 23 | +SET_ARG="${1:-docx}" |
| 24 | +TRIALS="${2:-3}" |
| 25 | +MODEL="motoko-local-qwen3-6-35b-a3b-mxfp8" |
| 26 | +LOGDIR="$HOME/dev/mk-ast/.motoko/logfile" |
| 27 | + |
| 28 | +case "$SET_ARG" in |
| 29 | + docx) BENCHES="docx_reimplement" ;; |
| 30 | + *) BENCHES="$SET_ARG" ;; |
| 31 | +esac |
| 32 | + |
| 33 | +# Coordinate with the os-rotation filler via the shared rig-lock: acquire WAIT so we block until |
| 34 | +# the current chunk releases, then HOLD it so the next filler tick defers. Any :8080 listener while |
| 35 | +# we hold the lock is an orphan — clear it. Auto-released on EXIT. |
| 36 | +# shellcheck source=/dev/null |
| 37 | +source "$(dirname "$0")/launchd/rig-lock.sh" |
| 38 | +echo "== waiting for the rig (rig-lock; any current os-rolling chunk must finish) ==" |
| 39 | +rig_lock_acquire wait |
| 40 | +# HARD SAFETY: never proceed while ANOTHER eval-suite is alive (a wedged chunk fighting over :8080 |
| 41 | +# api_errors every run at 0ms). Our own eval-suite hasn't started here, so any match is someone else's. |
| 42 | +while pgrep -f "ailang eval-suite" >/dev/null 2>&1; do |
| 43 | + echo "== another eval-suite is alive (likely a WEDGED chunk) — waiting to avoid a collision; kill it to proceed ==" |
| 44 | + sleep 60 |
| 45 | +done |
| 46 | +for _ in 1 2 3 4 5 6; do lsof -i :8080 -sTCP:LISTEN >/dev/null 2>&1 || break; sleep 10; done |
| 47 | +if lsof -i :8080 -sTCP:LISTEN >/dev/null 2>&1; then |
| 48 | + echo "== :8080 still held after 60s — clearing orphaned listener(s) (we hold the rig-lock) ==" |
| 49 | + for pid in $(lsof -ti :8080 2>/dev/null); do kill "$pid" 2>/dev/null; done |
| 50 | + sleep 5 |
| 51 | +fi |
| 52 | +if lsof -i :8080 -sTCP:LISTEN >/dev/null 2>&1; then |
| 53 | + echo "ERROR: :8080 still held after clear attempt — needs a manual kill, then re-run." >&2 |
| 54 | + exit 1 |
| 55 | +fi |
| 56 | + |
| 57 | +echo "== building current binary (delivery default-on + guard) ==" |
| 58 | +make quick-install >/dev/null |
| 59 | + |
| 60 | +OUT="eval_results/ab_sysrole_${SET_ARG}_$(date +%Y%m%d_%H%M%S)" |
| 61 | +mkdir -p "$OUT" |
| 62 | +COMMON=(--agent --models "$MODEL" --benchmarks "$BENCHES" --langs ailang --trials "$TRIALS" --microrag on) |
| 63 | + |
| 64 | +# Run one arm and record exactly which session logs it produced (single-writer under the rig-lock, |
| 65 | +# so the before/after diff is clean — no cross-arm contamination). |
| 66 | +run_arm() { |
| 67 | + local name="$1" before; shift |
| 68 | + before="$(mktemp)" |
| 69 | + ls "$LOGDIR"/session_*.jsonl 2>/dev/null | sort > "$before" || true |
| 70 | + "$@" |
| 71 | + comm -13 "$before" <(ls "$LOGDIR"/session_*.jsonl 2>/dev/null | sort) > "$OUT/$name.sessions.txt" || true |
| 72 | + rm -f "$before" |
| 73 | +} |
| 74 | + |
| 75 | +# Report the system_md delivery state across an arm's sessions — this is the |
| 76 | +# INDEPENDENT-VARIABLE CHECK: the arms are only comparable if delivery differed. |
| 77 | +delivery_state() { |
| 78 | + local list="$1" |
| 79 | + [ -s "$list" ] || { echo " (no sessions)"; return; } |
| 80 | + # shellcheck disable=SC2046 |
| 81 | + python3 - "$@" <<'PY' |
| 82 | +import json, sys |
| 83 | +from collections import Counter |
| 84 | +c = Counter() |
| 85 | +for path in [l.strip() for l in open(sys.argv[1]) if l.strip()]: |
| 86 | + state = "(no runtime_config_resolved)" |
| 87 | + try: |
| 88 | + for line in open(path): |
| 89 | + try: o = json.loads(line) |
| 90 | + except: continue |
| 91 | + if o.get("type") == "runtime_config_resolved": |
| 92 | + state = o.get("system_md", "(missing)"); break |
| 93 | + except FileNotFoundError: |
| 94 | + state = "(session file gone)" |
| 95 | + c[state] += 1 |
| 96 | +for k, v in c.most_common(): |
| 97 | + print(f" system_md={k!r}: {v} run(s)") |
| 98 | +PY |
| 99 | +} |
| 100 | + |
| 101 | +echo "== CONTROL (AILANG_MOTOKO_SYSTEM_ROLE=0 — legacy user-message fold) ==" |
| 102 | +run_arm control env AILANG_MOTOKO_SYSTEM_ROLE=0 \ |
| 103 | + ailang eval-suite "${COMMON[@]}" --output "$OUT/control" |
| 104 | + |
| 105 | +echo "== TREATMENT (AILANG_MOTOKO_SYSTEM_ROLE=1 — persistent system role) ==" |
| 106 | +run_arm treatment env AILANG_MOTOKO_SYSTEM_ROLE=1 \ |
| 107 | + ailang eval-suite "${COMMON[@]}" --output "$OUT/treatment" |
| 108 | + |
| 109 | +echo |
| 110 | +echo "================= RESULTS =================" |
| 111 | +for arm in control treatment; do |
| 112 | + echo "----- $arm: DELIVERY CHECK (independent variable) -----" |
| 113 | + delivery_state "$OUT/$arm.sessions.txt" |
| 114 | + echo "----- $arm: pass rate -----" |
| 115 | + ailang eval-summary "$OUT/$arm" 2>/dev/null || true |
| 116 | + echo "----- $arm: failure-mode histogram (compile-stuck = the dialect wall) -----" |
| 117 | + if [ -s "$OUT/$arm.sessions.txt" ]; then |
| 118 | + # shellcheck disable=SC2046 |
| 119 | + python3 tools/analyze_stuck.py $(cat "$OUT/$arm.sessions.txt") 2>/dev/null \ |
| 120 | + | grep -E "MODE:|SUMMARY|^ +[0-9]+ +(COMPILE|BEHAV|MIXED|NOT)" || true |
| 121 | + fi |
| 122 | + echo |
| 123 | +done |
| 124 | +echo "Artifacts: $OUT (per-arm session lists in *.sessions.txt)" |
| 125 | +echo "Clean iff: CONTROL system_md='unset' AND TREATMENT system_md='set'." |
| 126 | +echo "Then compare: pass-rate delta AND compile-stuck-rate delta (treatment should converge more)." |
0 commit comments