Skip to content

Commit 132ddb0

Browse files
committed
frontend: "system_reminders": "tail" moves a client's system-reminder block out of a past turn onto the last user turn
Clients inject a system-reminder block into a past turn (a mode change, an environment note) and drop it again on the next turn. The tokens a batch slot holds for that conversation's next turn are its prompt plus every token the engine generated, so they stop matching at the block, the restore fails and the whole conversation is read again: measured here on a 211990-token prompt, the shared prefix ended 155306 tokens in and 211990 were read from nothing (~88 s of prefill for a turn that changed one line). Known and unfixed upstream: anomalyco/opencode#23595 ("system-reminder keeps moving, causing unnecessary prompt processing in llama.cpp") and #21518 (queued messages serialized inconsistently), with the proposed fixes PR#27845 (anchor the reminder in place) and PR#24343 (drop stale plan reminders) both closed unmerged. Claude Code has the same effect: anthropics/claude-code#48734 (a trailing newline in a historical reminder drops cache_read to the tools block) and anthropics/claude-agent-sdk-typescript#269 (reminders stripped when the message is replayed). This is the same class of fix as pin_billing_stamp above, for the same reason: one change inside the history costs the whole conversation. The block moves to the end of the prompt, where it is the newest thing the model sees and where every turn has new tokens anyway, so the shared prefix of every later turn stays intact. Blocks already in the last user turn are left alone (no duplication); a block that is identical in a past turn and in the last turn stays where it is. Opt-in: the config's "system_reminders" is "keep" (default: render the history exactly as the client sent it) or "tail". Tests: SystemRemindersTail (moved to the last turn, flag off renders as before, no duplication).
1 parent 156b6f3 commit 132ddb0

3 files changed

Lines changed: 77 additions & 7 deletions

File tree

‎serve/frontend.py‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,41 @@ def tool_arguments(raw) -> dict:
368368
return {"arguments": json.dumps(raw, ensure_ascii=False)}
369369

370370

371-
def openai_to_messages(req: dict) -> tuple[list[dict], list[dict] | None, dict]:
371+
REMINDER = re.compile(r"<system-reminder>.*?</system-reminder>", re.S)
372+
373+
374+
def tail_system_reminders(messages: list[dict]) -> list[dict]:
375+
"""Move every system-reminder block out of the conversation's earlier turns and onto its last user turn.
376+
Clients inject such a block into a past turn (a mode change, an environment note) and drop it again on the
377+
next turn: the tokens a slot holds for the conversation's next turn then stop matching at that point, the
378+
restore fails and the whole conversation is read again (measured: 211990 tokens read from nothing, the cut
379+
155306 tokens in, at the block). opencode reports the same effect and has no fix: anomalyco/opencode#23595
380+
("system-reminder keeps moving, causing unnecessary prompt processing in llama.cpp"), #21518, and the
381+
proposed PRs #27845 and #24343, both closed unmerged; Claude Code has it too (anthropics/claude-code#48734,
382+
anthropics/claude-agent-sdk-typescript#269). This is the same class of fix as pin_billing_stamp above, for
383+
the same reason: what changes inside the history costs the whole conversation. At the end of the prompt the
384+
block is the newest thing the model sees, and every later turn keeps its shared prefix. Opt-in: the config's
385+
"system_reminders": "tail" (default "keep" renders the history exactly as the client sent it)."""
386+
last_user = max((i for i, m in enumerate(messages) if m.get("role") == "user"), default=-1)
387+
if last_user < 0 or not isinstance(messages[last_user].get("content"), str):
388+
return messages
389+
moved, tail = [], messages[last_user]["content"]
390+
for i, m in enumerate(messages):
391+
if i == last_user or not isinstance(m.get("content"), str):
392+
continue
393+
found = [b for b in REMINDER.findall(m["content"]) if b not in tail]
394+
if not found:
395+
continue
396+
moved.extend(found)
397+
m["content"] = REMINDER.sub("", m["content"]).strip()
398+
tail = "\n".join([tail] + moved)
399+
if not moved:
400+
return messages
401+
messages[last_user]["content"] = tail
402+
return messages
403+
404+
405+
def openai_to_messages(req: dict, reminders_tail: bool = False) -> tuple[list[dict], list[dict] | None, dict]:
372406
"""OpenAI Chat Completions -> (template messages, template tools, template kwargs)."""
373407
messages = []
374408
for m in _object_list(req.get("messages"), "messages"):
@@ -399,7 +433,8 @@ def openai_to_messages(req: dict) -> tuple[list[dict], list[dict] | None, dict]:
399433
kwargs = {"enable_thinking": False}
400434
elif k == "reasoning_effort" and "enable_thinking" not in kwargs:
401435
kwargs.update(effort_kwargs(v))
402-
return _late_system_to_user(messages), tools, kwargs
436+
messages = _late_system_to_user(messages)
437+
return (tail_system_reminders(messages) if reminders_tail else messages), tools, kwargs
403438

404439

405440
BILLING_HEADER = "x-anthropic-billing-header:"
@@ -431,7 +466,7 @@ def pin_billing_stamp(system: str) -> str:
431466
return "".join(s)
432467

433468

434-
def anthropic_to_messages(req: dict, think_unasked: bool = True) -> tuple[list[dict], list[dict] | None, dict]:
469+
def anthropic_to_messages(req: dict, think_unasked: bool = True, reminders_tail: bool = False) -> tuple[list[dict], list[dict] | None, dict]:
435470
"""Anthropic Messages -> (template messages, template tools, template kwargs). `think_unasked`: a request
436471
without "thinking", an effort or a budget gets the template's default (it thinks), as through 0.1.31; False
437472
renders it without thinking (#278, the config's "anthropic_thinking": "on_request")."""
@@ -493,7 +528,8 @@ def anthropic_to_messages(req: dict, think_unasked: bool = True) -> tuple[list[d
493528
# A config's reasoning_effort still applies: Service.with_shared sets output_config before this runs. A
494529
# request that gives its own reasoning_budget_tokens (#123) asks for thinking, so it thinks as before.
495530
kwargs["enable_thinking"] = False
496-
return _late_system_to_user(messages), tools, kwargs
531+
messages = _late_system_to_user(messages)
532+
return (tail_system_reminders(messages) if reminders_tail else messages), tools, kwargs
497533

498534

499535
# ------------------------------------------------------------------------------------------------ output parser

‎serve/server.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2446,6 +2446,7 @@ def __init__(self, engine: Engine, tokenizer, template: ChatTemplate, model_name
24462446
self.reasoning_budget_tokens = 0 # #123: the config's default thinking budget (0: none)
24472447
self.repeat_stop_tokens = REPEAT_STOP_TOKENS # #606: one token this many times in a row ends a reply (0: off)
24482448
self.anthropic_think_unasked = True # #278: "anthropic_thinking": "on_request" -> False
2449+
self.reminders_tail = False # opt-in: "system_reminders": "tail" (see frontend)
24492450
self.tool_call_recovery = False # opt-in: tool calls in forms next to the template's
24502451
self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +
24512452
tokenizer.encode("<|endoftext|>", parse_special=True))
@@ -4689,7 +4690,7 @@ def _captured(self, items, api):
46894690

46904691
def _openai(self, req):
46914692
req = svc.with_shared(req, "openai")
4692-
messages, tools, kw = openai_to_messages(req)
4693+
messages, tools, kw = openai_to_messages(req, svc.reminders_tail)
46934694
self._no_local_images(messages)
46944695
if tool_choice_of(req.get("tool_choice"))[0] == "none": # as the Responses route: no tools are offered
46954696
tools = None
@@ -4911,14 +4912,14 @@ def _count_tokens(self, req):
49114912
"""Anthropic's token count, which Claude Code asks for its context figures: the prompt this server would
49124913
read for the same request, rendered and tokenized - the model does not run."""
49134914
req = svc.with_shared(req, "anthropic")
4914-
messages, tools, kw = anthropic_to_messages(req, svc.anthropic_think_unasked)
4915+
messages, tools, kw = anthropic_to_messages(req, svc.anthropic_think_unasked, svc.reminders_tail)
49154916
stop_strings(req) # the same 400 as the request itself would get
49164917
self._json(200, {"input_tokens": len(svc.encode_prompt(messages, tools, kw))})
49174918

49184919
def _anthropic(self, req):
49194920
svc.load()
49204921
req = svc.with_shared(req, "anthropic")
4921-
messages, tools, kw = anthropic_to_messages(req, svc.anthropic_think_unasked)
4922+
messages, tools, kw = anthropic_to_messages(req, svc.anthropic_think_unasked, svc.reminders_tail)
49224923
self._no_local_images(messages)
49234924
if tool_choice_of(req.get("tool_choice"))[0] == "none": # Anthropic's {"type": "none"}: no tools offered
49244925
tools = None
@@ -5510,6 +5511,10 @@ def main() -> int:
55105511
if mode not in ("model", "on_request"):
55115512
raise SystemExit(f"[strata] config anthropic_thinking must be \"model\" or \"on_request\", not {mode!r}")
55125513
svc.anthropic_think_unasked = mode == "model"
5514+
reminders = str(cfg.get("system_reminders") or "keep") # "tail": move reminder blocks to the prompt's end
5515+
if reminders not in ("keep", "tail"):
5516+
raise SystemExit(f"[strata] config system_reminders must be \"keep\" or \"tail\", not {reminders!r}")
5517+
svc.reminders_tail = reminders == "tail"
55135518
recovery = cfg.get("tool_call_recovery", False) # opt-in: see OutputParser's recover
55145519
if not isinstance(recovery, bool):
55155520
raise SystemExit(f"[strata] config \"tool_call_recovery\" must be true or false, not {recovery!r}")

‎serve/test_server.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4709,6 +4709,35 @@ def test_two_turns_share_the_whole_system_prompt(self):
47094709
self.assertEqual(ids[1][:len(first) - 8], first[:len(first) - 8]) # all but the generation header
47104710

47114711

4712+
class SystemRemindersTail(unittest.TestCase):
4713+
""""system_reminders": "tail" moves a client's system-reminder block out of a past turn onto the last user
4714+
turn, so the tokens a slot holds for the conversation's next turn keep matching and the conversation is not
4715+
read again (opencode#23595: the block is dropped from the past turn on the next request, cutting the shared
4716+
prefix there - measured here at 155306 tokens in, 211990 read)."""
4717+
4718+
def user_contents(self, msgs, tail):
4719+
from serve.frontend import openai_to_messages
4720+
return [m["content"] for m in openai_to_messages({"messages": msgs}, tail)[0] if m["role"] == "user"]
4721+
4722+
def test_a_block_in_a_past_turn_moves_to_the_last_turn(self):
4723+
rem = "<system-reminder>\nYour operational mode has changed from plan to act.\n</system-reminder>"
4724+
msgs = [{"role": "user", "content": "door tiles " + rem},
4725+
{"role": "assistant", "content": "ok"},
4726+
{"role": "user", "content": "next step"}]
4727+
kept, last = self.user_contents(msgs, True)
4728+
self.assertEqual(kept, "door tiles") # the past turn is byte-stable again
4729+
self.assertTrue(last.endswith(rem)) # the block is the newest thing in the prompt
4730+
self.assertEqual(self.user_contents(msgs, False)[0], "door tiles " + rem) # off: as before
4731+
4732+
def test_a_block_already_in_the_last_turn_is_not_moved_twice(self):
4733+
rem = "<system-reminder>\nmode: act\n</system-reminder>"
4734+
msgs = [{"role": "user", "content": "a " + rem}, {"role": "assistant", "content": "b"},
4735+
{"role": "user", "content": "c " + rem}]
4736+
kept, last = self.user_contents(msgs, True)
4737+
self.assertEqual(kept, "a " + rem) # identical in a past turn: left where it is
4738+
self.assertEqual(last.count(rem), 1)
4739+
4740+
47124741
class ToolOrder(unittest.TestCase):
47134742
"""`_tool_list` returns the tools sorted by name. The template renders the tool list before the system
47144743
prompt, so a client whose tools arrive in a varying order (MCP servers registering in any sequence) moves

0 commit comments

Comments
 (0)