Skip to content

Commit c31c50b

Browse files
mvanhornclaude
andcommitted
fix(model): forward Qwen timeout and only set enable_thinking when true
Two bugs made local vLLM targets score acc=0.000: the router did not forward 'timeout' to the Qwen backend (so runs used the 300s default), and qwen_backend always injected chat_template_kwargs.enable_thinking, which non-Qwen vLLM servers reject or answer with <think> output and no <answer> tag. Forward timeout and only set the field when enabled. Closes #28 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent ee9931e commit c31c50b

3 files changed

Lines changed: 231 additions & 1 deletion

File tree

‎skillopt/model/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ def chat_target(
142142
retries=retries,
143143
stage=stage,
144144
reasoning_effort=reasoning_effort,
145+
timeout=timeout,
145146
)
146147
if get_target_backend() == "minimax_chat":
147148
return _minimax.chat_target(
@@ -249,6 +250,7 @@ def chat_target_messages(
249250
tools=tools,
250251
tool_choice=tool_choice,
251252
return_message=return_message,
253+
timeout=timeout,
252254
)
253255
if get_target_backend() == "minimax_chat":
254256
return _minimax.chat_target_messages(

‎skillopt/model/qwen_backend.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,8 @@ def _chat_messages_impl(
191191
"messages": _json_safe(messages),
192192
"max_tokens": min(max_completion_tokens, config.max_tokens),
193193
}
194-
payload["chat_template_kwargs"] = {"enable_thinking": config.enable_thinking}
194+
if config.enable_thinking:
195+
payload["chat_template_kwargs"] = {"enable_thinking": True}
195196
if config.temperature is not None:
196197
payload["temperature"] = config.temperature
197198
if tools:

‎tests/test_qwen_backend.py‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
"""Tests for the OpenAI-compatible Qwen chat backend."""
2+
from __future__ import annotations
3+
4+
import importlib.util
5+
import json
6+
import os
7+
import sys
8+
import types
9+
from collections.abc import Iterator
10+
from dataclasses import fields
11+
from typing import Any
12+
13+
import pytest
14+
15+
from skillopt.envs.searchqa.evaluator import extract_answer
16+
17+
18+
_QWEN_CONFIG_ENV_KEYS = (
19+
"BASE_URL",
20+
"API_KEY",
21+
"TEMPERATURE",
22+
"TIMEOUT_SECONDS",
23+
"MAX_TOKENS",
24+
"ENABLE_THINKING",
25+
)
26+
_ENV_KEYS = ("OPTIMIZER_BACKEND", "TARGET_BACKEND") + tuple(
27+
f"{prefix}QWEN_CHAT_{key}"
28+
for prefix in ("", "OPTIMIZER_", "TARGET_")
29+
for key in _QWEN_CONFIG_ENV_KEYS
30+
)
31+
32+
33+
class _FakeResponse:
34+
def __init__(self, payload: dict[str, Any]) -> None:
35+
self._payload = payload
36+
37+
def __enter__(self) -> _FakeResponse:
38+
return self
39+
40+
def __exit__(self, exc_type: object, exc: object, traceback: object) -> None:
41+
return None
42+
43+
def read(self) -> bytes:
44+
return json.dumps(self._payload).encode("utf-8")
45+
46+
47+
class _UrlopenRecorder:
48+
def __init__(self, content: str = "<answer>yes</answer>") -> None:
49+
self.content = content
50+
self.calls: list[dict[str, Any]] = []
51+
52+
def __call__(self, request: Any, timeout: float | None = None) -> _FakeResponse:
53+
request_data = request.data.decode("utf-8")
54+
self.calls.append(
55+
{
56+
"payload": json.loads(request_data),
57+
"timeout": timeout,
58+
}
59+
)
60+
return _FakeResponse(
61+
{
62+
"choices": [
63+
{
64+
"message": {"content": self.content},
65+
"finish_reason": "stop",
66+
}
67+
],
68+
"usage": {
69+
"prompt_tokens": 2,
70+
"completion_tokens": 1,
71+
"total_tokens": 3,
72+
},
73+
}
74+
)
75+
76+
77+
class _OpenAIClientStub:
78+
def __init__(self, *args: Any, **kwargs: Any) -> None:
79+
self.args = args
80+
self.kwargs = kwargs
81+
82+
83+
def _install_openai_stub() -> None:
84+
if "openai" in sys.modules or importlib.util.find_spec("openai") is not None:
85+
return
86+
openai_stub = types.ModuleType("openai")
87+
openai_stub.AzureOpenAI = _OpenAIClientStub
88+
openai_stub.OpenAI = _OpenAIClientStub
89+
sys.modules["openai"] = openai_stub
90+
91+
92+
def _import_model_modules() -> tuple[Any, Any, Any]:
93+
_install_openai_stub()
94+
import skillopt.model as model_module
95+
from skillopt.model import backend_config, qwen_backend
96+
97+
return model_module, backend_config, qwen_backend
98+
99+
100+
def _snapshot_config(config: Any) -> dict[str, Any]:
101+
return {field.name: getattr(config, field.name) for field in fields(config)}
102+
103+
104+
def _restore_config(config: Any, snapshot: dict[str, Any]) -> None:
105+
for key, value in snapshot.items():
106+
setattr(config, key, value)
107+
108+
109+
@pytest.fixture(autouse=True)
110+
def isolate_qwen_state() -> Iterator[tuple[Any, Any]]:
111+
model_module, backend_config, qwen_backend = _import_model_modules()
112+
optimizer_config = _snapshot_config(qwen_backend.OPTIMIZER_CONFIG)
113+
target_config = _snapshot_config(qwen_backend.TARGET_CONFIG)
114+
optimizer_backend = backend_config.get_optimizer_backend()
115+
target_backend = backend_config.get_target_backend()
116+
env = {key: os.environ.get(key) for key in _ENV_KEYS}
117+
qwen_backend.reset_token_tracker()
118+
yield model_module, qwen_backend
119+
qwen_backend.reset_token_tracker()
120+
_restore_config(qwen_backend.OPTIMIZER_CONFIG, optimizer_config)
121+
_restore_config(qwen_backend.TARGET_CONFIG, target_config)
122+
backend_config.set_optimizer_backend(optimizer_backend)
123+
backend_config.set_target_backend(target_backend)
124+
for key, value in env.items():
125+
if value is None:
126+
os.environ.pop(key, None)
127+
else:
128+
os.environ[key] = value
129+
130+
131+
def _use_qwen_target(model_module: Any, qwen_backend: Any, enable_thinking: bool) -> None:
132+
model_module.set_target_backend("qwen_chat")
133+
qwen_backend.TARGET_CONFIG.base_url = "http://qwen.example/v1"
134+
qwen_backend.TARGET_CONFIG.api_key = ""
135+
qwen_backend.TARGET_CONFIG.timeout_seconds = 300.0
136+
qwen_backend.TARGET_CONFIG.max_tokens = 8000
137+
qwen_backend.TARGET_CONFIG.temperature = None
138+
qwen_backend.TARGET_CONFIG.enable_thinking = enable_thinking
139+
qwen_backend.TARGET_CONFIG.deployment = "qwen-test"
140+
141+
142+
def _record_urlopen(
143+
monkeypatch: pytest.MonkeyPatch,
144+
qwen_backend: Any,
145+
content: str = "<answer>yes</answer>",
146+
) -> _UrlopenRecorder:
147+
recorder = _UrlopenRecorder(content)
148+
monkeypatch.setattr(qwen_backend.urllib.request, "urlopen", recorder)
149+
return recorder
150+
151+
152+
def test_chat_target_omits_chat_template_kwargs_when_thinking_disabled(
153+
monkeypatch: pytest.MonkeyPatch,
154+
isolate_qwen_state: tuple[Any, Any],
155+
) -> None:
156+
model_module, qwen_backend = isolate_qwen_state
157+
_use_qwen_target(model_module, qwen_backend, enable_thinking=False)
158+
recorder = _record_urlopen(monkeypatch, qwen_backend)
159+
160+
text, usage = model_module.chat_target(
161+
"system",
162+
"user",
163+
max_completion_tokens=128,
164+
retries=1,
165+
timeout=10.0,
166+
)
167+
168+
assert text == "<answer>yes</answer>"
169+
assert usage["total_tokens"] == 3
170+
assert "chat_template_kwargs" not in recorder.calls[0]["payload"]
171+
assert recorder.calls[0]["timeout"] == 10.0
172+
173+
174+
def test_chat_target_includes_chat_template_kwargs_when_thinking_enabled(
175+
monkeypatch: pytest.MonkeyPatch,
176+
isolate_qwen_state: tuple[Any, Any],
177+
) -> None:
178+
model_module, qwen_backend = isolate_qwen_state
179+
_use_qwen_target(model_module, qwen_backend, enable_thinking=True)
180+
content = "<think>working</think>\n<answer>yes</answer>"
181+
recorder = _record_urlopen(monkeypatch, qwen_backend, content=content)
182+
183+
text, _ = model_module.chat_target(
184+
"system",
185+
"user",
186+
max_completion_tokens=128,
187+
retries=1,
188+
)
189+
190+
assert recorder.calls[0]["payload"]["chat_template_kwargs"] == {"enable_thinking": True}
191+
assert extract_answer(text) == "yes"
192+
193+
194+
def test_chat_target_messages_forwards_timeout_to_qwen_backend(
195+
monkeypatch: pytest.MonkeyPatch,
196+
isolate_qwen_state: tuple[Any, Any],
197+
) -> None:
198+
model_module, qwen_backend = isolate_qwen_state
199+
_use_qwen_target(model_module, qwen_backend, enable_thinking=False)
200+
recorder = _record_urlopen(monkeypatch, qwen_backend)
201+
202+
text, _ = model_module.chat_target_messages(
203+
[{"role": "user", "content": "question"}],
204+
max_completion_tokens=128,
205+
retries=1,
206+
timeout=10.0,
207+
)
208+
209+
assert text == "<answer>yes</answer>"
210+
assert recorder.calls[0]["timeout"] == 10.0
211+
212+
213+
def test_configure_qwen_chat_runtime_toggle_controls_payload(
214+
monkeypatch: pytest.MonkeyPatch,
215+
isolate_qwen_state: tuple[Any, Any],
216+
) -> None:
217+
model_module, qwen_backend = isolate_qwen_state
218+
_use_qwen_target(model_module, qwen_backend, enable_thinking=False)
219+
recorder = _record_urlopen(monkeypatch, qwen_backend)
220+
221+
model_module.configure_qwen_chat(enable_thinking=True)
222+
model_module.chat_target("system", "user", max_completion_tokens=128, retries=1)
223+
model_module.configure_qwen_chat(enable_thinking=False)
224+
model_module.chat_target("system", "user", max_completion_tokens=128, retries=1)
225+
226+
assert recorder.calls[0]["payload"]["chat_template_kwargs"] == {"enable_thinking": True}
227+
assert "chat_template_kwargs" not in recorder.calls[1]["payload"]

0 commit comments

Comments
 (0)