Skip to content

Commit c2ed34a

Browse files
ualtinokalfonso-aft
andcommitted
mason: classify oc2 bash host failures
Pinned OpenCode 0.0.0-beta-19234 reports [object Object] for bash execution, so dependent status/watch/write/kill rows cannot observe their setup tool result; bash T4 likewise lacks status_reason call_aborted. Track these under anomalyco/opencode#37164, retain the host transcript, and make T5 assert the real host steer request instead of inferring a wake from task state. Co-authored-by: alfonso-aft <[email protected]> (cherry picked from commit bb81779a6cd2baa5bae3198e8ba5864c42a17d30)
1 parent e3f104d commit c2ed34a

19 files changed

Lines changed: 205 additions & 73 deletions

‎tests/docker/opencode2/harness/driver.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
type SharedServerHandle,
2121
} from "./host.js";
2222
import { createScenarioIsolation, assertPluginLoadEvidence } from "./isolation.js";
23+
import { assertT5HostWakeTranscript } from "./liveness.js";
2324
import {
2425
DeterministicScenarioMock,
2526
materializeTurnPlaceholders,
@@ -688,6 +689,7 @@ async function runOneScenario(options: {
688689
threeState?.assertComplete();
689690
const expectedTurns = scenario.expected_turns ?? scenario.turns.map((turn) => turn.label);
690691
assertTurnLog(expectedTurns, await readTurnLog(turnLogPath));
692+
assertT5HostWakeTranscript(scenario, mock.exchanges, controlPathValues);
691693

692694
pluginLog = await readFile(isolation.plugin_log, "utf8");
693695
await forensics.writeText("plugin.log", pluginLog);

‎tests/docker/opencode2/harness/harness.test.ts‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@ import { HarnessError, type HarnessFailureCode } from "./errors.js";
2121
import { runApiControl, startScenarioClient } from "./host.js";
2222
import { readPermissionAskInventory } from "./inventory.js";
2323
import { createScenarioIsolation } from "./isolation.js";
24-
import { CompletionWakeLiveness, WatchPatternLiveness } from "./liveness.js";
24+
import {
25+
assertT5HostWakeTranscript,
26+
CompletionWakeLiveness,
27+
WatchPatternLiveness,
28+
} from "./liveness.js";
2529
import {
2630
hostToolArguments,
2731
hostToolName,
@@ -715,6 +719,41 @@ describe("source-of-truth derivation", () => {
715719
});
716720

717721
describe("background liveness clocks", () => {
722+
test("completion wake requires one host-transcript steer request", () => {
723+
const completionScenario = scenario(call());
724+
completionScenario.id = "bash/T5/completion_wake";
725+
completionScenario.tool = "bash";
726+
completionScenario.trajectory = "T5";
727+
completionScenario.execution = "shared-server";
728+
completionScenario.metadata = {
729+
t5: { source_call_id: "source", wake_turn: "wake-observed" },
730+
};
731+
const exchange = {
732+
index: 0,
733+
label: "wake-observed",
734+
request: {
735+
messages: [
736+
{
737+
role: "user",
738+
content: "<system-reminder>\n[BACKGROUND BASH COMPLETED]\n- task bash-control\n</system-reminder>",
739+
},
740+
],
741+
},
742+
response: {},
743+
observed_at: "now",
744+
};
745+
expect(() =>
746+
assertT5HostWakeTranscript(completionScenario, [exchange], {
747+
"task_id:source": "bash-control",
748+
}),
749+
).not.toThrow();
750+
expect(() =>
751+
assertT5HostWakeTranscript(completionScenario, [], {
752+
"task_id:source": "bash-control",
753+
}),
754+
).toThrow("host transcript contains 0 completion steer wakes");
755+
});
756+
718757
test("watch setup, stdout antecedent, delivery, and duplicate window pass", () => {
719758
const clock = new WatchPatternLiveness({
720759
scenario: "bash/T5/watch_pattern_once",

‎tests/docker/opencode2/harness/liveness.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import type { RecordedMockExchange, ScenarioDefinition } from "./types.js";
2+
import { asRecord } from "./util.js";
13
import { fail } from "./errors.js";
24

35
export interface WatchLivenessEvidence {
@@ -166,3 +168,33 @@ export class CompletionWakeLiveness {
166168
}
167169
}
168170
}
171+
172+
export function assertT5HostWakeTranscript(
173+
scenario: ScenarioDefinition,
174+
exchanges: readonly RecordedMockExchange[],
175+
controlPathValues: Readonly<Record<string, string>>,
176+
): void {
177+
const t5 = asRecord(scenario.metadata?.t5);
178+
if (!t5 || typeof t5.wake_turn !== "string" || typeof t5.source_call_id !== "string") return;
179+
const taskId = controlPathValues[`task_id:${t5.source_call_id}`];
180+
if (!taskId) throw new Error(`${scenario.id}: background source task id was not observed`);
181+
const wakeRequests = exchanges.filter((exchange) => {
182+
if (exchange.label !== t5.wake_turn) return false;
183+
const messages = asRecord(exchange.request)?.messages;
184+
if (!Array.isArray(messages)) return false;
185+
return messages.some((value) => {
186+
const message = asRecord(value);
187+
const content = JSON.stringify(message?.content) ?? "";
188+
return (
189+
message?.role === "user" &&
190+
content.includes("[BACKGROUND BASH COMPLETED]") &&
191+
content.includes(taskId)
192+
);
193+
});
194+
});
195+
if (wakeRequests.length !== 1) {
196+
throw new Error(
197+
`${scenario.id}: host transcript contains ${wakeRequests.length} completion steer wakes for ${taskId}`,
198+
);
199+
}
200+
}

‎tests/docker/opencode2/matrix/applicability.json‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,16 @@
4444
"T1": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
4545
"T2": "applicable",
4646
"T3": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
47-
"T4": "applicable",
48-
"T5": "applicable",
47+
"T4": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
48+
"T5": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
4949
"T6": "applicable",
5050
"T7": "expected_fail:https://github.com/anomalyco/opencode/issues/48340"
5151
}
5252
},
5353
{
5454
"tool": "bash_kill",
5555
"trajectories": {
56-
"T1": "applicable",
56+
"T1": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
5757
"T2": "applicable",
5858
"T3": "n/a:no-permission-operation",
5959
"T4": "n/a:no-abortable-operation",
@@ -65,7 +65,7 @@
6565
{
6666
"tool": "bash_status",
6767
"trajectories": {
68-
"T1": "applicable",
68+
"T1": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
6969
"T2": "applicable",
7070
"T3": "n/a:no-permission-operation",
7171
"T4": "n/a:no-abortable-operation",
@@ -77,7 +77,7 @@
7777
{
7878
"tool": "bash_watch",
7979
"trajectories": {
80-
"T1": "applicable",
80+
"T1": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
8181
"T2": "applicable",
8282
"T3": "n/a:no-permission-operation",
8383
"T4": "n/a:no-abortable-operation",
@@ -89,7 +89,7 @@
8989
{
9090
"tool": "bash_write",
9191
"trajectories": {
92-
"T1": "applicable",
92+
"T1": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
9393
"T2": "applicable",
9494
"T3": "n/a:no-permission-operation",
9595
"T4": "n/a:no-abortable-operation",

‎tests/docker/opencode2/scenarios/bash/bash.extension.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ async function validate(context: HarnessValidationContext): Promise<void> {
1313
if (call.disk_effects === undefined && call.non_mutating_evidence === undefined) throw new Error(scenario.id + ":" + call.id + " lacks a disk classification");
1414
}
1515
const matrix = JSON.parse(await readFile(join(here, "matrix.json"), "utf8"));
16-
if (JSON.stringify(matrix.rows?.[0]?.trajectories) !== JSON.stringify({"T1":"expected_fail:https://github.com/anomalyco/opencode/issues/37164","T2":"applicable","T3":"expected_fail:https://github.com/anomalyco/opencode/issues/37164","T4":"applicable","T5":"applicable","T6":"applicable","T7":"expected_fail:https://github.com/anomalyco/opencode/issues/48340"})) throw new Error("bash" + " applicability mismatch");
16+
if (JSON.stringify(matrix.rows?.[0]?.trajectories) !== JSON.stringify({"T1":"expected_fail:https://github.com/anomalyco/opencode/issues/37164","T2":"applicable","T3":"expected_fail:https://github.com/anomalyco/opencode/issues/37164","T4":"expected_fail:https://github.com/anomalyco/opencode/issues/37164","T5":"expected_fail:https://github.com/anomalyco/opencode/issues/37164","T6":"applicable","T7":"expected_fail:https://github.com/anomalyco/opencode/issues/48340"})) throw new Error("bash" + " applicability mismatch");
1717
const controls = JSON.parse(await readFile(join(here, "mutation-controls.json"), "utf8"));
1818
if (!Array.isArray(controls.controls) || controls.controls.length < 3) throw new Error("bash" + " mutation controls missing");
1919
}

‎tests/docker/opencode2/scenarios/bash/matrix.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
"T1": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
1010
"T2": "applicable",
1111
"T3": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
12-
"T4": "applicable",
13-
"T5": "applicable",
12+
"T4": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
13+
"T5": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
1414
"T6": "applicable",
1515
"T7": "expected_fail:https://github.com/anomalyco/opencode/issues/48340"
1616
}

‎tests/docker/opencode2/scenarios/bash/registration.json‎

Lines changed: 17 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -609,25 +609,25 @@
609609
"tool": "bash",
610610
"trajectory": "T5",
611611
"execution": "shared-server",
612-
"prompt": "Start two sequential background tasks and observe one idle wake for each completion.",
612+
"prompt": "Start one background task and observe its completion wake on the host.",
613613
"fixture": "./fixture",
614614
"quiescence_timeout_ms": 30000,
615615
"turns": [
616616
{
617-
"label": "start-first",
617+
"label": "start-background",
618618
"response": {
619619
"kind": "tool_calls",
620620
"calls": [
621621
{
622-
"id": "bash-wake-first",
622+
"id": "bash-wake-source",
623623
"name": "bash",
624624
"arguments": {
625-
"command": "sleep 5; printf 'first wake\\n' > completion-one.txt",
625+
"command": "sleep 5; printf 'completion wake\\n' > completion.txt",
626626
"background": true
627627
},
628628
"disk_effects": [
629629
{
630-
"path": "completion-one.txt",
630+
"path": "completion.txt",
631631
"phase": "post_result"
632632
}
633633
],
@@ -637,66 +637,35 @@
637637
}
638638
},
639639
{
640-
"label": "idle-first",
640+
"label": "idle-before-wake",
641641
"response": {
642642
"kind": "text",
643-
"content": "Waiting for the first completion."
643+
"content": "Waiting for the completion wake."
644644
}
645645
},
646646
{
647-
"label": "start-second",
648-
"response": {
649-
"kind": "tool_calls",
650-
"calls": [
651-
{
652-
"id": "bash-wake-second",
653-
"name": "bash",
654-
"arguments": {
655-
"command": "sleep 5; printf 'second wake\\n' > completion-two.txt",
656-
"background": true
657-
},
658-
"disk_effects": [
659-
{
660-
"path": "completion-two.txt",
661-
"phase": "post_result"
662-
}
663-
],
664-
"outlives_result": true
665-
}
666-
]
667-
}
668-
},
669-
{
670-
"label": "idle-second",
647+
"label": "wake-observed",
671648
"response": {
672649
"kind": "text",
673-
"content": "Waiting for the second completion."
674-
}
675-
},
676-
{
677-
"label": "finish-wakes",
678-
"response": {
679-
"kind": "text",
680-
"content": "Both completion wakes observed."
650+
"content": "Completion wake observed."
681651
}
682652
}
683653
],
684654
"expected_turns": [
685-
"start-first",
686-
"idle-first",
687-
"start-second",
688-
"idle-second",
689-
"finish-wakes"
655+
"start-background",
656+
"idle-before-wake",
657+
"wake-observed"
690658
],
691659
"metadata": {
692660
"t5": {
693661
"subcase": "completion_wake",
662+
"source_call_id": "bash-wake-source",
663+
"wake_turn": "wake-observed",
694664
"initial_delivery_deadline_ms": 30000,
695665
"duplicate_detection_window": {
696666
"source": "product_rearm_ceiling_plus_one_tick",
697667
"tick_ms": 50
698-
},
699-
"sequential_tasks": true
668+
}
700669
}
701670
}
702671
},
@@ -793,6 +762,8 @@
793762
"t5": {
794763
"subcase": "watch_pattern_once",
795764
"pattern": "WATCH_MATCH",
765+
"source_call_id": "bash-watch-source",
766+
"wake_turn": "finish-watch",
796767
"setup_deadline_ms": 30000,
797768
"setup_phases": [
798769
"watch_arm",

‎tests/docker/opencode2/scenarios/bash_kill/bash_kill.extension.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ async function validate(context: HarnessValidationContext): Promise<void> {
1313
if (call.disk_effects === undefined && call.non_mutating_evidence === undefined) throw new Error(scenario.id + ":" + call.id + " lacks a disk classification");
1414
}
1515
const matrix = JSON.parse(await readFile(join(here, "matrix.json"), "utf8"));
16-
if (JSON.stringify(matrix.rows?.[0]?.trajectories) !== JSON.stringify({"T1":"applicable","T2":"applicable","T3":"n/a:no-permission-operation","T4":"n/a:no-abortable-operation","T5":"n/a:no-background-capability","T6":"n/a:no-list-surface","T7":"expected_fail:https://github.com/anomalyco/opencode/issues/48340"})) throw new Error("bash_kill" + " applicability mismatch");
16+
if (JSON.stringify(matrix.rows?.[0]?.trajectories) !== JSON.stringify({"T1":"expected_fail:https://github.com/anomalyco/opencode/issues/37164","T2":"applicable","T3":"n/a:no-permission-operation","T4":"n/a:no-abortable-operation","T5":"n/a:no-background-capability","T6":"n/a:no-list-surface","T7":"expected_fail:https://github.com/anomalyco/opencode/issues/48340"})) throw new Error("bash_kill" + " applicability mismatch");
1717
const controls = JSON.parse(await readFile(join(here, "mutation-controls.json"), "utf8"));
1818
if (!Array.isArray(controls.controls) || controls.controls.length < 3) throw new Error("bash_kill" + " mutation controls missing");
1919
}

‎tests/docker/opencode2/scenarios/bash_kill/matrix.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
{
77
"tool": "bash_kill",
88
"trajectories": {
9-
"T1": "applicable",
9+
"T1": "expected_fail:https://github.com/anomalyco/opencode/issues/37164",
1010
"T2": "applicable",
1111
"T3": "n/a:no-permission-operation",
1212
"T4": "n/a:no-abortable-operation",

‎tests/docker/opencode2/scenarios/bash_kill/registration.json‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,26 @@
1313
"prompt": "Run the deterministic bash_kill fixture call.",
1414
"fixture": "./fixture",
1515
"turns": [
16+
{
17+
"label": "setup-background",
18+
"response": {
19+
"kind": "tool_calls",
20+
"calls": [
21+
{
22+
"id": "bash_kill-setup",
23+
"name": "bash",
24+
"arguments": {
25+
"command": "sleep 30",
26+
"background": true
27+
},
28+
"non_mutating_evidence": {
29+
"reason": "the setup task only waits"
30+
},
31+
"outlives_result": true
32+
}
33+
]
34+
}
35+
},
1636
{
1737
"label": "call-tool",
1838
"response": {
@@ -22,7 +42,7 @@
2242
"id": "bash_kill-happy",
2343
"name": "bash_kill",
2444
"arguments": {
25-
"taskId": "{{task_id:setup}}"
45+
"taskId": "{{task_id:bash_kill-setup}}"
2646
},
2747
"non_mutating_evidence": {
2848
"reason": "bash_kill only observes fixture state"
@@ -40,6 +60,7 @@
4060
}
4161
],
4262
"expected_turns": [
63+
"setup-background",
4364
"call-tool",
4465
"finish"
4566
],

0 commit comments

Comments
 (0)