Skip to content

Commit 023ea2d

Browse files
committed
bench: support prompt transport ablations
1 parent bb8c50a commit 023ea2d

13 files changed

Lines changed: 352 additions & 89 deletions
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
schema_version: "1"
2+
condition: tb-harness-buzz-agent-terra-high-nobase
3+
roster:
4+
- id: solo
5+
kind: orchestrator
6+
role: solo
7+
count: 1
8+
endpoint: gpt-5.6-terra
9+
model_revision: gpt-5.6-terra
10+
harness: buzz-agent
11+
include_platform_prompt: false
12+
prompt:
13+
path: personas/bench/solo-harness-comparison.md
14+
sha256: 5a3bea67549937b6590a6d5678261cbfa08f3da67cb891dee9e81d408a93730b
15+
generation:
16+
thinking_effort: high
17+
prices:
18+
gpt-5.6-terra:
19+
input_per_million_usd: 2.0
20+
cached_input_per_million_usd: 0.2
21+
output_per_million_usd: 12.0
22+
cache_read_rate: 0.0
23+
trial_budget:
24+
timeout_seconds: 36000
25+
environment:
26+
override_cpus: 4
27+
override_memory_mb: 8192
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
schema_version: "1"
2+
condition: tb-harness-codex-terra-high-nobase
3+
roster:
4+
- id: solo
5+
kind: orchestrator
6+
role: solo
7+
count: 1
8+
endpoint: gpt-5.6-terra
9+
model_revision: gpt-5.6-terra
10+
harness: codex
11+
include_platform_prompt: false
12+
prompt:
13+
path: personas/bench/solo-harness-comparison.md
14+
sha256: 5a3bea67549937b6590a6d5678261cbfa08f3da67cb891dee9e81d408a93730b
15+
generation:
16+
thinking_effort: high
17+
prices:
18+
gpt-5.6-terra:
19+
input_per_million_usd: 2.0
20+
cached_input_per_million_usd: 0.2
21+
output_per_million_usd: 12.0
22+
cache_read_rate: 0.0
23+
trial_budget:
24+
timeout_seconds: 36000
25+
environment:
26+
override_cpus: 4
27+
override_memory_mb: 8192
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
schema_version: "1"
2+
condition: tb-harness-goose-terra-high-nobase
3+
roster:
4+
- id: solo
5+
kind: orchestrator
6+
role: solo
7+
count: 1
8+
endpoint: gpt-5.6-terra
9+
model_revision: gpt-5.6-terra
10+
harness: goose
11+
include_platform_prompt: false
12+
prompt:
13+
path: personas/bench/solo-harness-comparison.md
14+
sha256: 5a3bea67549937b6590a6d5678261cbfa08f3da67cb891dee9e81d408a93730b
15+
generation:
16+
thinking_effort: high
17+
prices:
18+
gpt-5.6-terra:
19+
input_per_million_usd: 2.0
20+
cached_input_per_million_usd: 0.2
21+
output_per_million_usd: 12.0
22+
cache_read_rate: 0.0
23+
trial_budget:
24+
timeout_seconds: 36000
25+
environment:
26+
override_cpus: 4
27+
override_memory_mb: 8192
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
schema_version: "1"
2+
condition: tb-harness-goose-terra-high-set
3+
roster:
4+
- id: solo
5+
kind: orchestrator
6+
role: solo
7+
count: 1
8+
endpoint: gpt-5.6-terra
9+
model_revision: gpt-5.6-terra
10+
harness: goose
11+
prompt:
12+
path: personas/bench/solo-harness-comparison.md
13+
sha256: 5a3bea67549937b6590a6d5678261cbfa08f3da67cb891dee9e81d408a93730b
14+
generation:
15+
thinking_effort: high
16+
extra:
17+
goose_system_prompt_mode: set
18+
prices:
19+
gpt-5.6-terra:
20+
input_per_million_usd: 2.0
21+
cached_input_per_million_usd: 0.2
22+
output_per_million_usd: 12.0
23+
cache_read_rate: 0.0
24+
trial_budget:
25+
timeout_seconds: 36000
26+
environment:
27+
override_cpus: 4
28+
override_memory_mb: 8192

‎benchmarks/harbor-buzz-orchestra/manifests/tb-solo-luna-nobase.yaml‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,16 @@
33
#
44
# Byte-identical to tb-solo-luna.yaml except for include_platform_prompt. That
55
# is the point — the pair isolates one variable, so any score, cost, or token
6-
# delta between them is attributable to the ~12KB of production-workspace
7-
# guidance buzz-acp prepends ahead of the pinned persona
6+
# delta between them is attributable to the production-workspace guidance
7+
# buzz-acp prepends ahead of the pinned persona
88
# (crates/buzz-acp/src/base_prompt.md).
99
#
10-
# Why it is worth a run: that section tells agents to sweep `buzz feed get` on
11-
# startup, look for RESEARCH/ and PLANS/ and AGENTS.md, edit inside a git
12-
# worktree, and that "publishing is optional and silence is usually correct".
13-
# In a graded container the first three waste tool rounds and can write files
14-
# the grader sees, and the fourth is how a multi-agent trial stalls. Every
15-
# persona in personas/bench/ overrides all of it in prose. This condition is
16-
# what turns that override from an assertion into a measurement.
10+
# Why it is worth a run: that section tells agents to look for workspace
11+
# guidance, edit inside a git worktree, and that "publishing is optional and
12+
# silence is usually correct". In a graded container those instructions can
13+
# waste tool rounds, write files the grader sees, or stall a multi-agent trial.
14+
# Every persona in personas/bench/ overrides all of it in prose. This condition
15+
# is what turns that override from an assertion into a measurement.
1716
#
1817
# Not a headline condition. tb-solo-luna.yaml stays the baseline, because
1918
# [Base] is what a real Buzz agent receives and the study is about Buzz. Run

‎benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py‎

Lines changed: 70 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,11 @@ def _forwarder_probe(proc: str = "/proc") -> str:
117117
return (
118118
f"for d in {proc}/[0-9]*; do "
119119
"a=$(tr '\\0' '\\n' < \"$d/cmdline\" 2>/dev/null | head -1); "
120-
f"[ \"$a\" = {shlex.quote(FORWARDER)} ] "
121-
f"&& echo \"${{d#{proc}/}}\"; done; true"
120+
f'[ "$a" = {shlex.quote(FORWARDER)} ] '
121+
f'&& echo "${{d#{proc}/}}"; done; true'
122122
)
123+
124+
123125
# Trust anchors the agent uses to reach its model provider over TLS.
124126
#
125127
# buzz-agent links reqwest's `rustls` feature, which loads roots via
@@ -358,12 +360,20 @@ async def run(
358360
if forwarder is not None:
359361
infra.append(forwarder)
360362
await self._buzz_json(
361-
trial.user, trial, "users", "set-profile", "--name",
363+
trial.user,
364+
trial,
365+
"users",
366+
"set-profile",
367+
"--name",
362368
trial.user.agent_id,
363369
)
364370
for credential in trial.credentials:
365371
await self._buzz_json(
366-
credential, trial, "users", "set-profile", "--name",
372+
credential,
373+
trial,
374+
"users",
375+
"set-profile",
376+
"--name",
367377
credential.agent_id,
368378
)
369379
agents.append(
@@ -480,9 +490,7 @@ async def run(
480490
output_tokens=trial_accounting.output_tokens,
481491
cost_usd=trial_accounting.cost_usd,
482492
metadata={
483-
"completion_message_id": (
484-
final_message["id"] if final_message else ""
485-
),
493+
"completion_message_id": (final_message["id"] if final_message else ""),
486494
"completion_message": (
487495
final_message["content"] if final_message else ""
488496
),
@@ -616,12 +624,12 @@ async def prepare_external_harnesses(
616624
"libgcc_s.so.1",
617625
"libstdc++.so.6",
618626
):
619-
uploads[f"{REMOTE_ROOT}/lib/x86_64/{name}"] = str(
620-
runtime_libs / name
621-
)
627+
uploads[f"{REMOTE_ROOT}/lib/x86_64/{name}"] = str(runtime_libs / name)
622628
if not uploads:
623629
return
624-
missing = [target for target, source in uploads.items() if not Path(source).is_file()]
630+
missing = [
631+
target for target, source in uploads.items() if not Path(source).is_file()
632+
]
625633
if missing:
626634
raise RuntimeLaunchError(
627635
"external harness binary not found for: " + ", ".join(missing)
@@ -728,11 +736,17 @@ async def _start_forwarder(
728736
if existing is not None:
729737
return _Agent(
730738
AgentCredential(
731-
agent_id="relay-forwarder", role="infra",
732-
nostr_secret_key="", nostr_pubkey="", nostr_auth_tag="",
733-
llm_endpoint="", llm_api_key="",
739+
agent_id="relay-forwarder",
740+
role="infra",
741+
nostr_secret_key="",
742+
nostr_pubkey="",
743+
nostr_auth_tag="",
744+
llm_endpoint="",
745+
llm_api_key="",
734746
),
735-
existing, log, log,
747+
existing,
748+
log,
749+
log,
736750
)
737751

738752
# Retry on EADDRINUSE anyway, as a guard for the cases adoption cannot
@@ -753,11 +767,17 @@ async def _start_forwarder(
753767
) from error
754768
forwarder = _Agent(
755769
AgentCredential(
756-
agent_id="relay-forwarder", role="infra",
757-
nostr_secret_key="", nostr_pubkey="", nostr_auth_tag="",
758-
llm_endpoint="", llm_api_key="",
770+
agent_id="relay-forwarder",
771+
role="infra",
772+
nostr_secret_key="",
773+
nostr_pubkey="",
774+
nostr_auth_tag="",
775+
llm_endpoint="",
776+
llm_api_key="",
759777
),
760-
pid, log, log,
778+
pid,
779+
log,
780+
log,
761781
)
762782
deadline = (
763783
asyncio.get_running_loop().time() + self.readiness_timeout_seconds
@@ -874,6 +894,13 @@ def _agent_env(
874894
"GOOSE_THINKING_EFFORT": effort,
875895
"OPENAI_API_KEY": credential.llm_api_key,
876896
}
897+
goose_prompt_mode = agent_class.generation.extra.get(
898+
"goose_system_prompt_mode"
899+
)
900+
if goose_prompt_mode is not None:
901+
harness_env["BUZZ_ACP_GOOSE_SYSTEM_PROMPT_MODE"] = str(
902+
goose_prompt_mode
903+
)
877904
elif agent_class.harness == "codex":
878905
harness_env = {
879906
"CODEX_PATH": f"{REMOTE_BIN}/codex",
@@ -984,7 +1011,9 @@ def _platform_prompt_env(agent_class: AgentClass) -> dict[str, str]:
9841011
"""
9851012
if agent_class.include_platform_prompt:
9861013
return {}
987-
return {"BUZZ_ACP_NO_BASE_PROMPT": "1"}
1014+
# clap's bool parser accepts explicit true/false values, not shell-style
1015+
# 1/0. This environment variable is passed as a value-bearing option.
1016+
return {"BUZZ_ACP_NO_BASE_PROMPT": "true"}
9881017

9891018
@staticmethod
9901019
def _window_env(generation: GenerationConfig) -> dict[str, str]:
@@ -1000,9 +1029,7 @@ def _window_env(generation: GenerationConfig) -> dict[str, str]:
10001029
if generation.max_output_tokens is not None:
10011030
env["BUZZ_AGENT_MAX_OUTPUT_TOKENS"] = str(generation.max_output_tokens)
10021031
if generation.context_window_tokens is not None:
1003-
env["BUZZ_AGENT_MAX_CONTEXT_TOKENS"] = str(
1004-
generation.context_window_tokens
1005-
)
1032+
env["BUZZ_AGENT_MAX_CONTEXT_TOKENS"] = str(generation.context_window_tokens)
10061033
return env
10071034

10081035
@staticmethod
@@ -1119,9 +1146,14 @@ async def _wait_for_done(
11191146
await self._raise_for_dead_agents(environment, agents)
11201147
polls += 1
11211148
messages = await self._buzz_json(
1122-
trial.user, trial,
1123-
"messages", "get", "--channel", trial.channel_id,
1124-
"--limit", "100",
1149+
trial.user,
1150+
trial,
1151+
"messages",
1152+
"get",
1153+
"--channel",
1154+
trial.channel_id,
1155+
"--limit",
1156+
"100",
11251157
)
11261158
for message in messages:
11271159
if message.get("pubkey") == orchestrator.nostr_pubkey and str(
@@ -1222,9 +1254,7 @@ async def _raise_for_dead_agents(
12221254
)
12231255

12241256
@staticmethod
1225-
async def _stop_agents(
1226-
environment: BaseEnvironment, agents: list[_Agent]
1227-
) -> None:
1257+
async def _stop_agents(environment: BaseEnvironment, agents: list[_Agent]) -> None:
12281258
"""Terminate every process of the uploaded stack (acp, agent, mcp)."""
12291259
if not agents:
12301260
return
@@ -1243,9 +1273,9 @@ async def _stop_agents(
12431273
# nothing: the container is destroyed at the end of the trial.
12441274
sweep = (
12451275
"for d in /proc/[0-9]*; do "
1246-
f"grep -aq {REMOTE_BIN} \"$d/cmdline\" 2>/dev/null "
1247-
"&& ! grep -aq relay-forwarder \"$d/cmdline\" 2>/dev/null "
1248-
"&& kill -TERM \"${d#/proc/}\" 2>/dev/null; done; true"
1276+
f'grep -aq {REMOTE_BIN} "$d/cmdline" 2>/dev/null '
1277+
'&& ! grep -aq relay-forwarder "$d/cmdline" 2>/dev/null '
1278+
'&& kill -TERM "${d#/proc/}" 2>/dev/null; done; true'
12491279
)
12501280
try:
12511281
await environment.exec(sweep)
@@ -1280,9 +1310,14 @@ async def _collect_transcript(self, trial: TrialHandle, trial_dir: Path) -> None
12801310
"""
12811311
try:
12821312
messages = await self._buzz_json(
1283-
trial.user, trial,
1284-
"messages", "get", "--channel", trial.channel_id,
1285-
"--limit", str(TRANSCRIPT_LIMIT),
1313+
trial.user,
1314+
trial,
1315+
"messages",
1316+
"get",
1317+
"--channel",
1318+
trial.channel_id,
1319+
"--limit",
1320+
str(TRANSCRIPT_LIMIT),
12861321
)
12871322
except Exception: # noqa: BLE001 — a lost transcript is not a failed trial
12881323
return

‎benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,15 @@ def validate_compaction(self) -> Self:
130130
)
131131
return self
132132

133+
@model_validator(mode="after")
134+
def validate_harness_extras(self) -> Self:
135+
goose_mode = self.extra.get("goose_system_prompt_mode")
136+
if goose_mode is not None and goose_mode not in {"append", "set"}:
137+
raise ValueError(
138+
"generation.extra.goose_system_prompt_mode must be append or set"
139+
)
140+
return self
141+
133142

134143
class AgentBudget(StrictModel):
135144
"""Optional per-agent live safety limits."""
@@ -192,8 +201,8 @@ class AgentClass(StrictModel):
192201
#
193202
# A manifest field rather than deployment config because it changes the
194203
# effective system prompt, which is what a condition *is*. The base section
195-
# is ~12KB of production-workspace guidance — startup recovery sweeps,
196-
# RESEARCH/PLANS conventions, git worktrees, "silence is usually correct" —
204+
# is production-workspace guidance — RESEARCH/PLANS conventions, git
205+
# worktrees, "silence is usually correct" —
197206
# written for a long-lived Buzz workspace rather than a graded container.
198207
# Every persona overrides it in prose, but prose is not proof, so the
199208
# ability to turn it off is what makes that override falsifiable.

0 commit comments

Comments
 (0)