Skip to content

Commit 9267168

Browse files
committed
fix(evals): bound empty-prompt replay to top-level threads
1 parent 2a76656 commit 9267168

3 files changed

Lines changed: 26 additions & 24 deletions

File tree

‎evals/mcp-tool-use/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts`
2626
- `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work.
2727
- `experience`: the real mistake, in a sentence. `source`: where it happened.
2828
- `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results.
29-
- `setup.priorTurns[].toolCalls[].resultFile`: load a repository-relative file as the tool result instead of copying its contents into the case. An empty `prompt` resumes after the last prior tool result without adding a user message.
29+
- `setup.priorTurns[].toolCalls[].resultFile`: load a repository-relative file as the tool result instead of copying its contents into the case. An empty `prompt` resumes a top-level thread after its last prior tool result without adding a user message. Side-chat and subagent setups require a nonempty prompt.
3030
- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers.
3131

3232
Check that the case builds with `--dry-run --case <id>`. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass.

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 19 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -397,27 +397,25 @@ it("replays earlier capability results with the model under test as the caller",
397397
const resumed = buildMessages(makeSample({ ...testCase, prompt: "" }, base));
398398
assert.equal(resumed.at(-1)?.role, "user");
399399
assert.equal(resumed.at(-1)?.content.at(-1)?.type, "tool_result");
400-
const sideChatResume = buildMessages(
401-
makeSample(
402-
{
403-
...testCase,
404-
prompt: "",
405-
setup: {
406-
...testCase.setup,
407-
sideChat: {
408-
parentThreadId: "thread-parent",
409-
digest: ["The permissions fix is committed."],
410-
},
411-
},
412-
},
413-
base,
414-
),
415-
);
416-
assert.include(
417-
sideChatResume[0]?.content.map((block) => block.text).join("\n"),
418-
"The permissions fix is committed.",
419-
);
420-
assert.equal(sideChatResume.at(-1)?.content.at(-1)?.type, "tool_result");
400+
for (const setup of [
401+
{
402+
...testCase.setup,
403+
sideChat: { parentThreadId: "thread-parent", digest: ["The permissions fix is committed."] },
404+
},
405+
{ ...testCase.setup, role: "native-subagent" as const },
406+
{ ...testCase.setup, role: "delegated-child" as const },
407+
]) {
408+
assert.throws(
409+
() =>
410+
buildMessages(
411+
makeSample(
412+
{ ...testCase, prompt: "", setup },
413+
{ ...base, subagentContext: "Hook context" },
414+
),
415+
),
416+
/Empty-prompt replay/,
417+
);
418+
}
421419
});
422420

423421
it("gives each launched thread its own id, and reading it back shows that thread's brief", () => {

‎evals/mcp-tool-use/run.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,7 @@ export const McpToolUseCase = Schema.Struct({
277277
Schema.Struct({ parentThreadId: Schema.String, digest: Schema.Array(Schema.String) }),
278278
),
279279
}),
280-
/** Empty to resume directly after the last prior tool result. */
280+
/** Empty to resume a top-level thread without side-chat context after its last tool result. */
281281
prompt: Schema.String,
282282
/** Records for the context chips `prompt` links, projected as T3 sends them. */
283283
promptContext: Schema.optional(Schema.Struct({ records: Schema.Array(ComposerContextRecord) })),
@@ -884,6 +884,11 @@ export const buildMessages = (sample: Sample) => {
884884
};
885885
const { role, context, delegateRole, priorTurns = [] } = testCase.setup;
886886
const sideChat = testCase.setup.sideChat;
887+
if (testCase.prompt === "" && (role !== "top" || sideChat !== undefined)) {
888+
throw new Error(
889+
`${testCase.id}: Empty-prompt replay requires a top-level thread without side-chat context.`,
890+
);
891+
}
887892
const sideChatContext =
888893
sideChat === undefined
889894
? undefined
@@ -897,7 +902,6 @@ export const buildMessages = (sample: Sample) => {
897902
? "You are a subagent started by the parent agent's Agent tool. Your final message goes to the parent agent, not to the user."
898903
: undefined,
899904
context,
900-
testCase.prompt === "" ? sideChatContext : undefined,
901905
];
902906
const reminders = base.reminders.toSpliced(base.gitStatusAt, 0, reminder(gitStatus(testCase)));
903907
push("user", ...reminders, ...notes.flatMap((note) => (note ? [reminder(note)] : [])));

0 commit comments

Comments
 (0)