Skip to content

Commit c8b45c5

Browse files
committed
test(evals): keep thread link fixture status consistent
1 parent 7616014 commit c8b45c5

1 file changed

Lines changed: 1 addition & 1 deletion

File tree

‎evals/mcp-tool-use/cases/orchestration.ndjson‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
{"id": "thread-id-not-read", "area": "orchestration", "kind": "bad", "pattern": "did not read a given thread id", "experience": "The user pasted a thread id and asked why 'this t3 code thread' kept mentioning pi. The agent answered from its own context. The user: 'Use the t3 code mcp to get that thread, that's the id lol'.", "rule": "Intended: read a given thread id with t3_thread_read, with a bounded size.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Is the Muse CLI just a fork or wrapper of the Pi agent?"}, {"role": "assistant", "text": "No. The installed Muse CLI is a native Rust binary built from its own source tree. I searched the provider code for 'pi' to check, and found only T3 Code's separate Pi provider docs.", "toolCalls": [{"tool": "Bash", "input": {"command": "rg -il '\\bpi\\b' src/providers"}, "result": "src/providers/pi/README.md"}, {"tool": "Bash", "input": {"command": "file \"$(which muse)\""}, "result": "/usr/local/bin/muse: ELF 64-bit LSB pie executable, x86-64, dynamically linked, Rust toolchain 1.91, stripped"}]}], "mocks": {"t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Muse CLI integration and ACP support", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "status": "idle"}, "items": [{"position": 494, "createdBy": "assistant", "text": "Pi already replays recorded transcripts in tests, so the Muse adapter copies that pattern."}], "nextPosition": 495, "hasMore": false}}, "world": {"files": {"src/providers/pi/README.md": "# Pi provider\n\nRuns the Pi agent through its RPC mode.\n", "src/providers/muse/adapter.ts": "export const museAdapter = makeAdapter({ binary: 'muse' });\n"}}}, "prompt": "11111111-1111-4111-8111-111111111111 why does this t3 code thread keep mentioning pi?", "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}]}, "notes": "The prior turn mentions pi, which is what let the agent think 'this thread' meant itself. The same thread then read with limit 100 and got an 85k-char result (see gaps)."}
1717
{"id": "thread-mention-without-link-after-read", "area": "orchestration", "kind": "bad", "pattern": "thread mentioned without t3-thread link", "experience": "Asked to explain what went wrong in a given thread, the agent named it by id or title with no t3-thread:// link.", "rule": "To mention another thread to the user, write its `link` verbatim: <t3-thread://v2/<environmentId>/<threadId>>, with no title.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix ultrafast pricing", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "status": "idle"}, "items": [{"position": 210, "createdBy": "assistant", "toolName": "t3_thread_send", "text": "t3_thread_send threadId=44444444-4444-4444-8444-000000000001 message='Round 2. Read only, as before...'"}, {"position": 214, "createdBy": "user", "text": "Use delegate_task please, this should not be a separate thread."}], "nextPosition": 215, "hasMore": false}}}, "prompt": "11111111-1111-4111-8111-111111111111 This thread used the wrong tool. Figure out why.", "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}], "textMust": ["t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111"], "textMustNot": ["t3-thread://v1/"]}, "notes": "Real agent was Codex before the 10-21 untruncation fix, when the link rule often did not reach Codex. May pass on Claude today; keep as a regression check for the rewrite.", "maxSteps": 10}
1818
{"id": "found-thread-mentioned-without-link", "area": "orchestration", "kind": "bad", "pattern": "thread mentioned without t3-thread link", "experience": "The agent found yesterday's thread by search and told the user where they left off, but did not link the thread.", "rule": "To mention another thread to the user, write its `link` verbatim: <t3-thread://v2/<environmentId>/<threadId>>, with no title.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"t3_thread_search": {"matches": [{"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "22222222-2222-4222-8222-222222222222", "title": "Fix macOS camera permission prompt", "snippet": "...the app never asks for camera permission on mac...", "messageCreatedAt": "2026-10-19T21:04:00Z"}]}, "t3_thread_list": {"threads": [{"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix macOS camera permission prompt", "status": "idle"}]}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix macOS camera permission prompt", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "status": "idle"}, "items": [{"position": 88, "createdBy": "assistant", "text": "The fix is committed locally on branch fix/camera-entitlement in another clone. No PR is open yet."}], "nextPosition": 89, "hasMore": false}}, "shellMocks": [{"match": "gh pr list", "output": ""}]}, "prompt": "Did I ever make a PR to fix the issue where the app couldn't request camera permission on mac? I had a thread about it yesterday and forgot where we left off.", "expect": {"textMust": ["t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111"], "textMustNot": ["t3-thread://v1/"]}, "notes": "Also seen in another thread ('Found the prior audits (Jul 21, Sep 1, Oct 3 threads ...)' with no links). Original prompt also said 'get it fixed full stop'; dropped to keep the case small."}
19-
{"id": "thread-link-rewritten-with-title", "area": "orchestration", "kind": "bad", "pattern": "thread link rewritten with a title", "experience": "Thread links carried the title the agent wrote, which went stale once the user renamed the thread. The app already shows each linked thread's current title, so maintainers asked that agents write only the stable link.", "rule": "To mention another thread to the user, write its `link` verbatim, never a title of its own.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"t3_thread_list": {"threads": [{"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix flaky checkout test", "status": "running"}, {"threadId": "44444444-4444-4444-8444-000000000002", "title": "Upgrade Vite", "status": "idle"}]}}}, "prompt": "Which of my threads in this project are still running?", "expect": {"calls": [{"tool": "t3_thread_list"}], "textMust": ["\\[Fix flaky checkout test\\]\\(t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111\\)"], "textMustNot": ["t3-thread://v1/"]}, "notes": "Source: maintainer request with the v2 thread link change. The old instructions carried a titled [title](t3-thread://v1/<exact id>) template for threads without a link. The link T3 returns carries the title it knew, so a client without that thread's machine still names it; a client that knows the thread shows its current title instead."}
19+
{"id": "thread-link-rewritten-with-title", "area": "orchestration", "kind": "bad", "pattern": "thread link rewritten with a title", "experience": "Thread links carried the title the agent wrote, which went stale once the user renamed the thread. The app already shows each linked thread's current title, so maintainers asked that agents write only the stable link.", "rule": "To mention another thread to the user, write its `link` verbatim, never a title of its own.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"t3_thread_list": {"threads": [{"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix flaky checkout test", "status": "running"}]}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix flaky checkout test", "status": "running"}, "items": [], "nextPosition": 1, "hasMore": false}}}, "prompt": "Which of my threads in this project are still running?", "expect": {"calls": [{"tool": "t3_thread_list"}], "textMust": ["\\[Fix flaky checkout test\\]\\(t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111\\)"], "textMustNot": ["t3-thread://v1/"]}, "notes": "Source: maintainer request with the v2 thread link change. The old instructions carried a titled [title](t3-thread://v1/<exact id>) template for threads without a link. The link T3 returns carries the title it knew, so a client without that thread's machine still names it; a client that knows the thread shows its current title instead."}
2020
{"id": "cross-provider-delegates-with-options", "area": "orchestration", "kind": "guard", "pattern": "capabilities then cross-provider delegate_task with options", "experience": "Asked for Sol 'high' and Muse Wonder Full 'xhigh' delegates, the agent read capabilities and put effort in target.options.", "rule": "Read the selected model's descriptors from orchestrator_capabilities and put settings in target.options, not only in the prompt.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "I want this codebase reviewed by two different models. Spin up a delegate for GPT-6.1 Sol high in Codex and one for Muse Wonder Full at xhigh. Tell them both to explore the codebase and find interesting things, plus some easy wins.", "expect": {"order": ["orchestrator_capabilities", "delegate_task"], "calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "codex", "target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "high"}}, {"tool": "delegate_task", "args": {"target.providerInstanceId": "muse", "target.model": "muse-wonder-full", "target.options.reasoningEffort": "xhigh"}}], "notCalls": ["t3_thread_launch", "create_threads", "Agent"]}, "notes": "target.options may be an object or an [{id,value}] array. The runner must normalize arrays to an object before matching 'target.options.<id>'."}
2121
{"id": "same-provider-delegate-with-effort", "area": "orchestration", "kind": "guard", "pattern": "model options force delegate_task over native", "experience": "Asked for an Opus 5.5 (xhigh) delegate from a Claude parent, the agent used delegate_task with target.options effort xhigh.", "rule": "When model options are requested ... use delegate_task even for native-supported models. Native subagents cannot reliably apply T3 model options.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Spin up an Opus 5.5 delegate in Claude Code on xhigh to research why the update flow spams 'connected successfully' toasts. Have it write up the causes in priority order.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "claudeAgent", "target.model": "claude-opus-5-5", "target.options.effort": "xhigh"}}], "notCalls": ["Agent", "Workflow"]}, "notes": "A shorter text that drops the options rule could send this to the native Agent tool with model 'opus' and lose xhigh."}
2222
{"id": "async-delegate-then-end-turn", "area": "orchestration", "kind": "guard", "pattern": "async delegate then end turn", "experience": "After starting an async Sol review the agent said its completion would wake the thread and ended the turn.", "rule": "Prefer mode='async' for long work ... the task's completion still wakes this thread. (intended: end the turn; no sleep or task_status loops)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default", "task_status": "default"}, "world": {"branch": "fix/session-stop", "commits": [{"sha": "3e8d1f0", "subject": "fix(server): stop clears an orphaned task row"}], "diff": "diff --git a/src/session.ts b/src/session.ts\n--- a/src/session.ts\n+++ b/src/session.ts\n@@ -10,1 +10,1 @@\n- if (task.status !== \"running\") return;\n+ if (task.status === \"cancelled\") return emitTaskRow(task);\n", "prs": [{"number": 123, "title": "fix(server): stop clears an orphaned task row", "headRefName": "fix/session-stop"}]}}, "prompt": "Have a GPT-6.1 Sol delegate review the diff on this branch (https://github.com/acme/app/pull/123) before I merge it.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol"}}], "maxCalls": {"task_status": 0, "t3_thread_wait": 0}, "shellMustNot": ["\\bsleep\\b"]}, "notes": ""}

0 commit comments

Comments
 (0)