Skip to content

Commit 81b6c40

Browse files
feat(opencode): live tool-calling probe; skip tool-less models in auto-pick
A model can advertise tools yet emit the call as plain text (qwen2.5-coder via Ollama does; upstream anomalyco/opencode#2728, anomalyco/opencode#7030) -- OpenCode has no text fallback, so the session silently does nothing. Config can't fix it: verified against opencode 1.17.13 source that tool_call defaults to true and is metadata-only on 1.17.x. - local_llm.rs: probe_tool_call command -- one real /chat/completions request with a trivial tool; verdict = native tool_calls present vs call-written-as-text vs no call (pure parser, fixtures captured live from Ollama 0.30.10: qwen3:30b-a3b native ok, qwen2.5-coder:7b text). - Panel: "Test tool-calling" button with verdict note; dropdown marks "no tools" models; warning when the selected model reports no tools; auto-pick now excludes tools=false models (bge-m3, deepseek-r1, phi) unless nothing else is served. (cherry picked from commit c562cc8)
1 parent 9b31dc6 commit 81b6c40

5 files changed

Lines changed: 231 additions & 8 deletions

File tree

‎src-tauri/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -859,6 +859,7 @@ pub fn run() {
859859
opencode_key_set,
860860
local_llm::detect_local_providers,
861861
local_llm::list_local_models,
862+
local_llm::probe_tool_call,
862863
remove_session,
863864
suggest_session_name,
864865
git_branch,

‎src-tauri/src/local_llm.rs‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,27 @@ fn curl(url: &str, bearer: Option<&str>, max_time_s: u32) -> Option<String> {
5252
Some(String::from_utf8_lossy(&out.stdout).into_owned())
5353
}
5454

55+
/// POST a JSON body. Long timeout — a cold model may need to load into VRAM first.
56+
fn curl_post(url: &str, body: &str, bearer: Option<&str>, max_time_s: u32) -> Option<String> {
57+
let mut cmd = std::process::Command::new("curl");
58+
cmd.args([
59+
"-s",
60+
"--max-time",
61+
&max_time_s.to_string(),
62+
"-H",
63+
"Content-Type: application/json",
64+
]);
65+
if let Some(key) = bearer {
66+
cmd.args(["-H", &format!("Authorization: Bearer {key}")]);
67+
}
68+
cmd.args(["-d", body]).arg(url);
69+
let out = cmd.no_window().output().ok()?;
70+
if !out.status.success() {
71+
return None;
72+
}
73+
Some(String::from_utf8_lossy(&out.stdout).into_owned())
74+
}
75+
5576
// ---- pure parsers --------------------------------------------------------------
5677

5778
/// Ollama GET /api/version → "v0.30.10".
@@ -147,6 +168,75 @@ fn parse_openai_models(body: &str) -> Vec<LocalModel> {
147168
.collect()
148169
}
149170

171+
/// Verdict of a live tool-calling probe against a served model.
172+
#[derive(Serialize, Clone, Debug, PartialEq)]
173+
#[serde(rename_all = "camelCase")]
174+
pub struct ToolProbeResult {
175+
/// True when the model answered with a native (structured) tool call.
176+
pub native: bool,
177+
pub detail: String,
178+
}
179+
180+
/// The one-shot chat request used to probe native tool calling: a single trivial tool
181+
/// and an instruction to call it. Roomy max_tokens because thinking models (qwen3 etc.)
182+
/// burn tokens on reasoning before the call.
183+
fn tool_probe_body(model: &str) -> String {
184+
serde_json::json!({
185+
"model": model,
186+
"messages": [{ "role": "user", "content": "Call the list_files tool on path \".\" now." }],
187+
"tools": [{ "type": "function", "function": {
188+
"name": "list_files",
189+
"description": "List files in a directory",
190+
"parameters": { "type": "object",
191+
"properties": { "path": { "type": "string" } },
192+
"required": ["path"] } } }],
193+
"max_tokens": 2048
194+
})
195+
.to_string()
196+
}
197+
198+
/// Judge the probe response. A model that "supports tools" on paper can still write the
199+
/// call into content as plain text (qwen2.5-coder does) — OpenCode has no text fallback,
200+
/// so such a model silently does nothing in a session. This is the check that catches it.
201+
fn parse_tool_probe(body: &str) -> ToolProbeResult {
202+
let fail = |detail: &str| ToolProbeResult {
203+
native: false,
204+
detail: detail.to_string(),
205+
};
206+
let v: serde_json::Value = match serde_json::from_str(body) {
207+
Ok(v) => v,
208+
Err(_) => return fail("server returned a non-JSON response"),
209+
};
210+
if let Some(err) = v.get("error") {
211+
let msg = err
212+
.get("message")
213+
.and_then(|m| m.as_str())
214+
.unwrap_or("unknown server error");
215+
return fail(&format!("server error: {msg}"));
216+
}
217+
let msg = &v["choices"][0]["message"];
218+
let called = msg
219+
.get("tool_calls")
220+
.and_then(|t| t.as_array())
221+
.is_some_and(|t| !t.is_empty());
222+
if called {
223+
return ToolProbeResult {
224+
native: true,
225+
detail: "native tool-calling verified — agentic use will work".to_string(),
226+
};
227+
}
228+
let content = msg.get("content").and_then(|c| c.as_str()).unwrap_or("");
229+
if content.contains("\"name\"") || content.contains("\"arguments\"") {
230+
fail(
231+
"model wrote the tool call as plain text instead of calling it — OpenCode cannot \
232+
execute that, sessions will appear to do nothing. Pick a different model \
233+
(the qwen3 family works well).",
234+
)
235+
} else {
236+
fail("model did not call the tool — expect poor agentic behavior")
237+
}
238+
}
239+
150240
/// Derive the server origin from an OpenAI-compatible base URL by stripping the trailing
151241
/// path segment(s) we know about ("/v1", "/api"). Used to reach Ollama's native /api/*.
152242
fn origin_of(base_url: &str) -> String {
@@ -269,6 +359,33 @@ pub async fn list_local_models(
269359
.map_err(|e| format!("probe task failed: {e}"))?
270360
}
271361

362+
/// Live-test whether the served model actually does NATIVE tool calling (one chat
363+
/// request with a trivial tool). Advertised capabilities aren't enough: some models
364+
/// (e.g. qwen2.5-coder via Ollama) claim tools but emit the call as plain text, which
365+
/// OpenCode cannot execute. Slow on a cold model (load into VRAM) — up to ~2.5 min.
366+
#[tauri::command]
367+
pub async fn probe_tool_call(
368+
base_url: String,
369+
model: String,
370+
store: tauri::State<'_, std::sync::Arc<crate::store::Store>>,
371+
) -> Result<ToolProbeResult, String> {
372+
let api_key = store.opencode_key();
373+
tauri::async_runtime::spawn_blocking(move || {
374+
let base = base_url.trim().trim_end_matches('/');
375+
if !base.starts_with("http://") && !base.starts_with("https://") {
376+
return Err("base URL must start with http:// or https://".to_string());
377+
}
378+
let url = format!("{base}/chat/completions");
379+
let body = tool_probe_body(model.trim());
380+
match curl_post(&url, &body, api_key.as_deref(), 150) {
381+
Some(resp) => Ok(parse_tool_probe(&resp)),
382+
None => Err("server not reachable (or the model took too long to load)".to_string()),
383+
}
384+
})
385+
.await
386+
.map_err(|e| format!("probe task failed: {e}"))?
387+
}
388+
272389
#[cfg(test)]
273390
mod tests {
274391
use super::*;
@@ -335,6 +452,38 @@ mod tests {
335452
assert!(parse_openwebui_config(r#"{"name":"grafana"}"#).is_none());
336453
}
337454

455+
#[test]
456+
fn tool_probe_judges_native_text_and_errors() {
457+
// Shapes captured live from Ollama 0.30.10 /v1/chat/completions.
458+
let native = r#"{"choices":[{"finish_reason":"tool_calls","message":{"role":"assistant",
459+
"content":"","reasoning":"...","tool_calls":[{"id":"call_1","type":"function",
460+
"function":{"name":"list_files","arguments":"{\"path\":\".\"}"}}]}}]}"#;
461+
assert!(parse_tool_probe(native).native);
462+
463+
// qwen2.5-coder failure mode: the "call" arrives as plain text content.
464+
let text = r#"{"choices":[{"finish_reason":"stop","message":{"role":"assistant",
465+
"content":"{\n \"name\": \"list_files\",\n \"arguments\": {\"path\": \".\"}\n}"}}]}"#;
466+
let r = parse_tool_probe(text);
467+
assert!(!r.native);
468+
assert!(r.detail.contains("plain text"));
469+
470+
let chatty = r#"{"choices":[{"message":{"content":"I cannot run tools."}}]}"#;
471+
let r = parse_tool_probe(chatty);
472+
assert!(!r.native);
473+
assert!(r.detail.contains("did not call"));
474+
475+
let err = r#"{"error":{"message":"model not found"}}"#;
476+
assert!(parse_tool_probe(err).detail.contains("model not found"));
477+
assert!(!parse_tool_probe("Not Found").native);
478+
479+
// The probe body is valid JSON and carries the model id + roomy token budget.
480+
let body: serde_json::Value =
481+
serde_json::from_str(&tool_probe_body("qwen3:30b-a3b")).unwrap();
482+
assert_eq!(body["model"], "qwen3:30b-a3b");
483+
assert_eq!(body["max_tokens"], 2048);
484+
assert_eq!(body["tools"][0]["function"]["name"], "list_files");
485+
}
486+
338487
#[test]
339488
fn origin_strips_known_api_suffixes() {
340489
assert_eq!(

‎src/components/OpenCodePanel.tsx‎

Lines changed: 62 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,14 @@ const rankModels = (ms: LocalModel[]): LocalModel[] =>
3434
return (b.context ?? 0) - (a.context ?? 0);
3535
});
3636

37+
/** Auto-pick candidate list: models that declare tools=false can never drive an agent,
38+
* so they're only eligible when nothing else is served. */
39+
const pickable = (ms: LocalModel[]): LocalModel[] => {
40+
const ranked = rankModels(ms);
41+
const usable = ranked.filter((m) => m.tools !== false);
42+
return usable.length > 0 ? usable : ranked;
43+
};
44+
3745
const fmtCtx = (n?: number | null) => (n ? `${Math.round(n / 1024)}k ctx` : "");
3846

3947
export function OpenCodePanel() {
@@ -45,11 +53,14 @@ export function OpenCodePanel() {
4553
const setOpenCodeKey = useStore((s) => s.setOpenCodeKey);
4654
const detectLocalProviders = useStore((s) => s.detectLocalProviders);
4755
const listLocalModels = useStore((s) => s.listLocalModels);
56+
const probeToolCall = useStore((s) => s.probeToolCall);
4857

4958
const [statuses, setStatuses] = useState<LocalProviderStatus[] | null>(null);
5059
const [models, setModels] = useState<LocalModel[]>([]);
5160
const [busy, setBusy] = useState<string | null>(null);
5261
const [note, setNote] = useState<{ kind: "ok" | "warn"; text: string } | null>(null);
62+
const [probe, setProbe] = useState<{ kind: "ok" | "warn"; text: string } | null>(null);
63+
const [probing, setProbing] = useState(false);
5364
const [keyInput, setKeyInput] = useState("");
5465
const [showAdvanced, setShowAdvanced] = useState(false);
5566
const mounted = useRef(false);
@@ -104,7 +115,7 @@ export function OpenCodePanel() {
104115
setModels(listed);
105116
const fresh = useStore.getState().opencode;
106117
const keep = listed.find((m) => m.id === fresh.model);
107-
const best = keep ?? rankModels(listed)[0];
118+
const best = keep ?? pickable(listed)[0];
108119
const next = save({
109120
preset: target.preset,
110121
baseUrl: target.baseUrl,
@@ -171,9 +182,25 @@ export function OpenCodePanel() {
171182

172183
const pickModel = (id: string) => {
173184
const m = models.find((x) => x.id === id);
185+
setProbe(null); // verdict belongs to the previously tested model
174186
save({ model: id, ...(m?.context ? { contextLimit: m.context } : {}) });
175187
};
176188

189+
/** Live-test that the served model does NATIVE tool calls. Advertised capabilities
190+
* lie (qwen2.5-coder claims tools but prints the call as text); this catches it
191+
* before the user wonders why a session "does nothing". */
192+
const runProbe = async () => {
193+
setProbing(true);
194+
setProbe(null);
195+
const r = await probeToolCall(oc.baseUrl, oc.model);
196+
setProbe(
197+
typeof r === "string"
198+
? { kind: "warn", text: r }
199+
: { kind: r.native ? "ok" : "warn", text: `${oc.model}: ${r.detail}` },
200+
);
201+
setProbing(false);
202+
};
203+
177204
const parseLimit = (v: string): number | null => {
178205
const n = parseInt(v.replace(/[^0-9]/g, ""), 10);
179206
return Number.isFinite(n) && n > 0 ? n : null;
@@ -240,13 +267,23 @@ export function OpenCodePanel() {
240267
<div className="oc-section">
241268
<div className="section-label">
242269
Model
243-
<button
244-
className="oc-detect"
245-
onClick={() => void fetchModels()}
246-
disabled={!!busy || !oc.baseUrl.trim()}
247-
>
248-
Refresh list
249-
</button>
270+
<span className="oc-actions">
271+
<button
272+
className="oc-detect"
273+
onClick={() => void runProbe()}
274+
disabled={!!busy || probing || !oc.model.trim() || !oc.baseUrl.trim()}
275+
title="Sends one real request to check the model makes NATIVE tool calls — the thing agents live on. May take a minute if the model has to load."
276+
>
277+
{probing ? "Testing…" : "Test tool-calling"}
278+
</button>
279+
<button
280+
className="oc-detect"
281+
onClick={() => void fetchModels()}
282+
disabled={!!busy || !oc.baseUrl.trim()}
283+
>
284+
Refresh list
285+
</button>
286+
</span>
250287
</div>
251288
{models.length > 0 ? (
252289
<select
@@ -262,6 +299,7 @@ export function OpenCodePanel() {
262299
{m.id}
263300
{m.context ? ` — ${fmtCtx(m.context)}` : ""}
264301
{m.tools ? " — tools ✓" : ""}
302+
{m.tools === false ? " — no tools ✗" : ""}
265303
{m.detail ? ` — ${m.detail}` : ""}
266304
</option>
267305
))}
@@ -275,9 +313,25 @@ export function OpenCodePanel() {
275313
: "No models yet — re-scan, or type an id under Advanced."}
276314
</p>
277315
)}
316+
{probing && (
317+
<p className="trust-note">
318+
Testing {oc.model} with a real tool call — a cold model may need a minute to load…
319+
</p>
320+
)}
321+
{probe && (
322+
<p className={`trust-note${probe.kind === "warn" ? " trust-warn" : ""}`}>{probe.text}</p>
323+
)}
324+
{models.find((m) => m.id === oc.model)?.tools === false && (
325+
<p className="trust-note trust-warn">
326+
{oc.model} reports no tool-calling support — an agent can’t work with it. Pick a
327+
model marked “tools ✓”.
328+
</p>
329+
)}
278330
{lowContext && (
279331
<p className="trust-note trust-warn">
280332
{oc.contextLimit} tokens of context is tight for agentic coding — 64k+ recommended.
333+
Small models also fail at tool calling far more often; “Test tool-calling” checks
334+
before you burn a session.
281335
</p>
282336
)}
283337
</div>

‎src/store.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,12 @@ export interface LocalModel {
109109
tools?: boolean | null;
110110
}
111111

112+
/** Verdict of the live tool-calling probe (probe_tool_call). */
113+
export interface ToolProbeResult {
114+
native: boolean;
115+
detail: string;
116+
}
117+
112118
export type TabKind = "session" | "file";
113119

114120
export interface WsTab {
@@ -457,6 +463,8 @@ interface AppState {
457463
detectLocalProviders: () => Promise<LocalProviderStatus[]>;
458464
/** Models the server at baseUrl offers; a string is the error message. */
459465
listLocalModels: (baseUrl: string, preset: string) => Promise<LocalModel[] | string>;
466+
/** Live-test native tool calling on the served model; a string is the error message. */
467+
probeToolCall: (baseUrl: string, model: string) => Promise<ToolProbeResult | string>;
460468

461469
// ---- MCP server registry ----
462470
mcpServers: McpServer[];
@@ -771,6 +779,13 @@ export const useStore = create<AppState>((set, get) => {
771779
return String(e);
772780
}
773781
},
782+
probeToolCall: async (baseUrl, model) => {
783+
try {
784+
return await invoke<ToolProbeResult>("probe_tool_call", { baseUrl, model });
785+
} catch (e) {
786+
return String(e);
787+
}
788+
},
774789
completeAgentSetup: () => {
775790
localStorage.setItem(SETUP_DONE_KEY, "1");
776791
set({ agentSetupComplete: true });

‎src/theme.css‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1930,6 +1930,10 @@ button {
19301930
font-size: 11px;
19311931
padding: 2px 10px;
19321932
}
1933+
.oc-actions {
1934+
display: inline-flex;
1935+
gap: 6px;
1936+
}
19331937
.oc-presets {
19341938
display: flex;
19351939
flex-wrap: wrap;

0 commit comments

Comments
 (0)