Skip to content

Commit 2944a88

Browse files
author
Eva
committed
fix(bench): mention the orchestrator by pubkey when posting the task
The harness posts each trial's task via `buzz messages send`, relying on `@<orchestrator-id>` name resolution. Task text is untrusted payload: when it contains @-tokens of its own — TB 2.1's large-scale-text-editing task embeds Vim macros like `:%normal! @a` — the CLI's mention resolver rejects `@a` as an unknown channel member and refuses to send, killing the trial with RuntimeLaunchError before the agent ever saw the task. Pass the orchestrator's pubkey as an explicit --mention instead. The CLI demotes unresolved @-tokens in the text to presentation-only when any explicit identity is supplied, so delivery still targets exactly the orchestrator and every @-token in the task statement becomes inert. Live repro on the relay confirmed both halves: a fenced `@a` without --mention still hard-fails (the resolver is not markdown-aware); the same content with an explicit --mention sends clean. - benchmarks/harbor-buzz-orchestra: full pytest suite, 35 passed (34 baseline + new test), ruff clean — validated against origin/main 769ac70 with this patch applied - testbed: full pytest suite, 23 passed / 1 skipped, ruff clean Originating Buzz thread: buzz://message?channel=c3252dd2-0142-4e01-88c7-a2183c3960a5&id=74a65a0990fd2197882b66b5ea2707169d4a3dbd2020d1610c45150fb99f140b Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> (cherry picked from commit e1953e2)
1 parent 1cf841a commit 2944a88

2 files changed

Lines changed: 52 additions & 6 deletions

File tree

‎benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -365,9 +365,17 @@ async def run(
365365
)
366366
# The task arrives exactly as it would in production Buzz: a
367367
# user prompt @mentioning the orchestrator. The harness never
368-
# speaks as any agent.
368+
# speaks as any agent. The orchestrator is mentioned by pubkey,
369+
# not by name resolution: task text is untrusted payload, and any
370+
# @-token inside it (e.g. Vim's `:%normal! @a`) would otherwise
371+
# fail member resolution and kill the trial before the agent
372+
# ever saw the task. An explicit --mention demotes unresolved
373+
# @-tokens in the text to presentation-only.
369374
await self._send(
370-
trial.user, trial, f"@{orchestrator.agent_id} {instruction}"
375+
trial.user,
376+
trial,
377+
f"@{orchestrator.agent_id} {instruction}",
378+
mention=orchestrator.nostr_pubkey,
371379
)
372380
final_message = await asyncio.wait_for(
373381
self._wait_for_done(
@@ -1223,13 +1231,20 @@ async def _verify_m1_output(
12231231
)
12241232

12251233
async def _send(
1226-
self, credential: AgentCredential, trial: TrialHandle, content: str
1234+
self,
1235+
credential: AgentCredential,
1236+
trial: TrialHandle,
1237+
content: str,
1238+
*,
1239+
mention: str | None = None,
12271240
) -> None:
1228-
await self._buzz_json(
1229-
credential, trial,
1241+
args = [
12301242
"messages", "send", "--channel", trial.channel_id,
12311243
"--content", content,
1232-
)
1244+
]
1245+
if mention is not None:
1246+
args += ["--mention", mention]
1247+
await self._buzz_json(credential, trial, *args)
12331248

12341249
async def _buzz_json(
12351250
self, credential: AgentCredential, trial: TrialHandle, *args: str

‎benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,37 @@ async def test_m1_output_probe_matches_grader_and_is_condition_scoped(
915915
assert bool(probed) == (condition == "M1-hello-world")
916916

917917

918+
async def test_send_mentions_by_pubkey_so_task_text_stays_inert(
919+
tmp_path, monkeypatch
920+
):
921+
"""Task text is untrusted payload: `:%normal! @a` in a task statement must
922+
not be fed to member-name resolution (it would fail and kill the trial).
923+
An explicit --mention pins delivery to the orchestrator's pubkey."""
924+
rt = runtime(tmp_path)
925+
orch = credential("orch-1", "orchestrator", "orch-model")
926+
trial = trial_handle((orch,))
927+
calls = []
928+
929+
async def buzz_json(credential, trial, *args):
930+
calls.append(args)
931+
return {}
932+
933+
monkeypatch.setattr(rt, "_buzz_json", buzz_json)
934+
935+
await rt._send(
936+
trial.user,
937+
trial,
938+
"@orch-1 run `:%normal! @a` on the file",
939+
mention=orch.nostr_pubkey,
940+
)
941+
assert calls[-1][-2:] == ("--mention", "pubkey-orch-1")
942+
943+
# Without an explicit mention the send is unchanged (name resolution).
944+
await rt._send(trial.user, trial, "plain content")
945+
assert "--mention" not in calls[-1]
946+
assert calls[-1][-2:] == ("--content", "plain content")
947+
948+
918949
async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch):
919950
rt = runtime(tmp_path, poll_seconds=0)
920951
orch = credential("orch-1", "orchestrator", "orch-model")

0 commit comments

Comments
 (0)