Skip to content

Commit 07ee617

Browse files
committed
fix(test): fail the index benchmark when its index is refused
bench_finish keeps the private root on a zero exit status, but the index call swallowed its failure (`|| echo '{"error":"index failed"}'`), so a refused index exited 0 and, with CBM_BENCH_KEEP_RUNTIME set, kept an empty root and wrote runtime-root.txt for an index that does not exist. benchmark-index.sh now records the CLI's exit status, keeps 00-index.json valid JSON on failure ({"error":"index failed","exit":N}), and exits 1 once every per-run file — the three timing files included — is written, when the index failed or reported no project (the search-graph twin's own test). bench_finish then cleans up as its comment always promised, and the caller's loop notices. The index call keeps its stderr in a run-private temp file, the twin's shape, and the failure prints it under `--- index stderr ---` with the first 500 bytes of the response: for a one-shot CLI, stderr is the only channel a refusal is reported on. A stale runtime-root.txt from an earlier run in the same results directory is removed at start, so a failed run leaves no handoff at all. The contract test's fixture answers `cli index_repository` with a minimal envelope when CBM_BENCH_PROBE_INDEX_OK is set, so the keep run covers the successful-index handoff and asserts project.txt; the unflagged run asserts a non-zero exit and the surfaced cause after a refused index; a new keep run with a refused index asserts a non-zero exit, the timing files, no surviving root and no handoff, a pre-seeded stale one included. Against the previous head the test fails on "benchmark-index exited 0 after a refused index". Signed-off-by: Anton Standrik <[email protected]>
1 parent 9574c31 commit 07ee617

2 files changed

Lines changed: 81 additions & 6 deletions

File tree

‎scripts/benchmark-index.sh‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ cbm_test_runtime_init
2727
# by default, or an unattended run accumulates one root per language.
2828
bench_finish() {
2929
local rc=$?
30+
[ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true
3031
if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then
3132
"$BINARY" daemon stop >/dev/null 2>&1 || true
3233
printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \
@@ -36,7 +37,13 @@ bench_finish() {
3637
fi
3738
cbm_test_runtime_cleanup "$BINARY"
3839
}
40+
BENCH_TMP=""
3941
trap bench_finish EXIT
42+
# The index call keeps its stderr, the way the search-graph twin does: for a
43+
# one-shot CLI that is the only channel a refusal is reported on, and without
44+
# it the failure below could name only its symptom.
45+
BENCH_TMP=$(mktemp -d)
46+
INDEX_ERR="$BENCH_TMP/index-stderr.log"
4047

4148
# Resolve symlinks
4249
REPO=$(cd "$REPO" && pwd -P)
@@ -54,6 +61,10 @@ bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"
5461

5562
OUT="$RESULTS_DIR/$LANG"
5663
mkdir -p "$OUT"
64+
# A handoff names this run's index or none: the evaluation loop reuses the
65+
# results directory, and a stale one would point the graph session at a root
66+
# step 8 has already removed.
67+
rm -f -- "$OUT/runtime-root.txt"
5768

5869
echo "INDEX: $LANG ($REPO)"
5970

@@ -86,7 +97,12 @@ fi
8697
# Index via CLI and capture timing
8798
START_MS=$(bench_now_ms)
8899

89-
INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
100+
# The CLI's exit status is the index's verdict. It is recorded here and acted
101+
# on below, once every per-run file is written; an empty response stays valid
102+
# JSON in 00-index.json so the failure is legible there as well.
103+
INDEX_RC=0
104+
INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>"$INDEX_ERR") ||
105+
{ INDEX_RC=$?; INDEX_JSON="{\"error\":\"index failed\",\"exit\":$INDEX_RC}"; }
90106

91107
END_MS=$(bench_now_ms)
92108
ELAPSED=$((END_MS - START_MS))
@@ -130,5 +146,20 @@ echo "$NODES" > "$OUT/nodes.txt"
130146
echo "$EDGES" > "$OUT/edges.txt"
131147
echo "$PROJECT" > "$OUT/project.txt"
132148

149+
# An index that failed, or that named no project, fails the run: after the
150+
# timing files, so the caller keeps its figures, and through the exit status,
151+
# so the caller's loop notices and bench_finish never keeps a root with no
152+
# index in it.
153+
if [ "$INDEX_RC" -ne 0 ] || [ -z "$PROJECT" ]; then
154+
echo " $LANG: index failed (cli exit $INDEX_RC, project '$PROJECT'); response in $OUT/00-index.json" >&2
155+
if [ -s "$INDEX_ERR" ]; then
156+
echo "--- index stderr ---" >&2
157+
cat "$INDEX_ERR" >&2
158+
fi
159+
echo "--- index response (first 500 bytes) ---" >&2
160+
printf '%.500s\n' "$INDEX_JSON" >&2
161+
exit 1
162+
fi
163+
133164
printf " %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \
134165
"$LANG" "$FILE_COUNT" "$LOC" "$ELAPSED" "$NODES" "$EDGES"

‎tests/test_benchmark_runtime_isolation_contract.sh‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ set -euo pipefail
99
# operator's live store, every one-shot joined the operator's account daemon,
1010
# and the timings depended on whatever that daemon was doing. Drive both with
1111
# an environment-probe fixture and require that no product process ever
12-
# receives the caller's runtime or cache, and that the index benchmark records
13-
# the setup cost it now pays explicitly.
12+
# receives the caller's runtime or cache, that the index benchmark records
13+
# the setup cost it now pays explicitly, and that a refused index fails the
14+
# run, names its cause, and leaves nothing behind.
1415

1516
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
1617
WORKDIR="$(mktemp -d)"
@@ -35,6 +36,10 @@ cat > "$ENV_PROBE" <<'EOF'
3536
printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE"
3637
if [[ "${1-} ${2-}" == "cli index_repository" ]]; then
3738
[[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG"
39+
if [[ -n "${CBM_BENCH_PROBE_INDEX_OK-}" ]]; then
40+
printf '%s\n' '{"project":"probe"}'
41+
exit 0
42+
fi
3843
echo "probe: index refused" >&2
3944
exit 1
4045
fi
@@ -78,30 +83,46 @@ assert_isolated() {
7883
[[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root"
7984
}
8085

86+
# The fixture refuses the index. That must fail the run — after the timing
87+
# files are written, so the caller keeps its figures — and name its cause: the
88+
# fixture's one stderr line has to reach the output.
8189
INDEX_LOG="$WORKDIR/index-environment.log"
90+
index_rc=0
8291
CBM_CACHE_DIR="$CALLER_CACHE" \
8392
CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
8493
CBM_BENCH_ENV_PROBE="$INDEX_LOG" \
8594
"$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \
86-
> "$WORKDIR/index.out" 2>&1 || true
95+
> "$WORKDIR/index.out" 2>&1 || index_rc=$?
96+
[[ "$index_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index"
8797
assert_isolated "benchmark-index" "$INDEX_LOG"
8898
for metric in setup-time total-time index-time; do
8999
[[ -s "$WORKDIR/results/probe/$metric.txt" ]] ||
90100
fail "benchmark-index did not record $metric.txt"
91101
done
102+
grep -q -- '--- index stderr ---' "$WORKDIR/index.out" ||
103+
fail "benchmark-index hid the index stderr behind its own message"
104+
grep -q 'probe: index refused' "$WORKDIR/index.out" ||
105+
fail "benchmark-index did not surface the cause of the index failure"
92106

93107
# The evaluation plan indexes a language and then reads that index from its own
94108
# MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a
95109
# successful run must leave its root behind and record, sourceably, the paths
96110
# that reach it — and they must be the paths the product processes actually
97-
# used. Unasked, the root is gone (asserted above).
111+
# used. Unasked, the root is gone (asserted above). The fixture answers this
112+
# index with a minimal envelope, so what is kept is an index that succeeded.
98113
KEEP_LOG="$WORKDIR/keep-environment.log"
114+
keep_rc=0
99115
CBM_CACHE_DIR="$CALLER_CACHE" \
100116
CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
101117
CBM_BENCH_ENV_PROBE="$KEEP_LOG" \
118+
CBM_BENCH_PROBE_INDEX_OK=1 \
102119
CBM_BENCH_KEEP_RUNTIME=1 \
103120
"$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \
104-
> "$WORKDIR/keep.out" 2>&1 || true
121+
> "$WORKDIR/keep.out" 2>&1 || keep_rc=$?
122+
[[ "$keep_rc" -eq 0 ]] || fail "benchmark-index failed a run whose index succeeded (exit $keep_rc)"
123+
KEPT_PROJECT=$(cat "$WORKDIR/results/keep/project.txt" 2>/dev/null || true)
124+
[[ "${KEPT_PROJECT%$'\r'}" == "probe" ]] ||
125+
fail "benchmark-index did not record the project the index reported: '${KEPT_PROJECT:-<none>}'"
105126
HANDOFF="$WORKDIR/results/keep/runtime-root.txt"
106127
[[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff"
107128
KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF")
@@ -115,6 +136,29 @@ grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" ||
115136
fail "runtime-root.txt does not name the runtime and cache the product processes used"
116137
rm -rf -- "$KEPT_ROOT"
117138

139+
# Asked to keep the runtime and refused the index, the harness has nothing worth
140+
# keeping: the run fails, no root survives, and no handoff is written — nor left
141+
# over from an earlier run, since the evaluation loop reuses the results
142+
# directory.
143+
REFUSED_LOG="$WORKDIR/keep-refused-environment.log"
144+
REFUSED_HANDOFF="$WORKDIR/results/keep-refused/runtime-root.txt"
145+
mkdir -p "${REFUSED_HANDOFF%/*}"
146+
echo 'CBM_BENCH_RUNTIME_ROOT=/stale/root/from/an/earlier/run' > "$REFUSED_HANDOFF"
147+
refused_rc=0
148+
CBM_CACHE_DIR="$CALLER_CACHE" \
149+
CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
150+
CBM_BENCH_ENV_PROBE="$REFUSED_LOG" \
151+
CBM_BENCH_KEEP_RUNTIME=1 \
152+
"$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep-refused "$REPO" "$WORKDIR/results" \
153+
> "$WORKDIR/keep-refused.out" 2>&1 || refused_rc=$?
154+
[[ "$refused_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index it was asked to keep"
155+
for metric in setup-time total-time index-time; do
156+
[[ -s "$WORKDIR/results/keep-refused/$metric.txt" ]] ||
157+
fail "benchmark-index did not record $metric.txt for a refused index"
158+
done
159+
assert_isolated "benchmark-index (keep, refused index)" "$REFUSED_LOG"
160+
[[ ! -e "$REFUSED_HANDOFF" ]] || fail "benchmark-index left a handoff for an index that failed"
161+
118162
SEARCH_LOG="$WORKDIR/search-environment.log"
119163
CBM_CACHE_DIR="$CALLER_CACHE" \
120164
CBM_RUNTIME_DIR="$CALLER_RUNTIME" \

0 commit comments

Comments
 (0)