Skip to content

Commit f8e1551

Browse files
atishpatelcodex
andcommitted
fix(bench): preserve task evidence across runtime merge
Co-authored-by: Codex <[email protected]> Signed-off-by: Atish Patel <[email protected]>
1 parent 09bde0b commit f8e1551

2 files changed

Lines changed: 44 additions & 23 deletions

File tree

‎benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py‎

Lines changed: 40 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import os
1616
import shlex
1717
import time
18+
import traceback
1819
from dataclasses import dataclass, field
1920
from pathlib import Path
2021
from typing import Any
@@ -24,12 +25,12 @@
2425

2526
from . import accounting, bundle
2627
from .accounting import TURN_ENDED_MARKERS, USAGE_MARKER
28+
from .evidence import build_buzz_evidence
2729
from .manifest import AgentClass, ExperimentManifest, GenerationConfig
2830
from .provisioning import AgentCredential, TrialHandle
2931
from .runtime import RuntimeResult
3032
from .task_fixtures import fixture_for
3133

32-
3334
# Tool/LLM rounds allowed per agent turn (BUZZ_AGENT_MAX_ROUNDS).
3435
#
3536
# Deliberately set high enough that no condition reaches it, because the cap is
@@ -353,6 +354,9 @@ async def run(
353354

354355
agents: list[_Agent] = []
355356
infra: list[_Agent] = []
357+
task_event_id: str | None = None
358+
final_message: dict[str, Any] | None = None
359+
evidence_exported = False
356360
trust_store = "unknown"
357361
verifier_deps = "unknown"
358362
try:
@@ -393,10 +397,16 @@ async def run(
393397
)
394398
# The task arrives exactly as it would in production Buzz: a
395399
# user prompt @mentioning the orchestrator. The harness never
396-
# speaks as any agent.
400+
# speaks as any agent. The orchestrator is mentioned by pubkey,
401+
# not by name resolution: task text is untrusted payload, and any
402+
# @-token inside it must remain part of the task rather than being
403+
# interpreted as another relay mention.
397404
active_started = time.monotonic()
398-
await self._send(
399-
trial.user, trial, f"@{orchestrator.agent_id} {instruction}"
405+
task_event = await self._send(
406+
trial.user,
407+
trial,
408+
f"@{orchestrator.agent_id} {instruction}",
409+
mention=orchestrator.nostr_pubkey,
400410
)
401411
if isinstance(task_event, dict) and isinstance(
402412
task_event.get("event_id"), str
@@ -448,12 +458,18 @@ async def run(
448458
# agent/buzz/ holding nothing but the system prompts, which reads
449459
# as "the agents never ran" rather than "teardown stalled".
450460
await self._collect_logs(environment, trial_dir)
461+
evidence_exported = await self._collect_evidence(
462+
environment=environment,
463+
trial=trial,
464+
trial_dir=trial_dir,
465+
task_event_id=task_event_id,
466+
completion_message_id=(
467+
final_message.get("id") if final_message is not None else None
468+
),
469+
)
451470
# After the agents are stopped, so the tool never reaches them, and
452471
# before Harbor's verifier phase, which is what needs it.
453472
verifier_deps = await self._preinstall_verifier_deps(environment)
454-
# Before bundle.write, so the transcript is listed in the bundle's
455-
# file index and covered by its secret scan.
456-
await self._collect_transcript(trial, trial_dir)
457473
# Accounting runs even when the trial failed or timed out: a trial
458474
# that burned tokens and then stalled still cost money, and
459475
# excluding it would bias the sweep's cost figures toward successes.
@@ -491,6 +507,15 @@ async def run(
491507
endpoints=self.endpoints,
492508
)
493509

510+
# A missing snapshot is a harness failure only for Buzz-native tasks
511+
# that grade relay state. Terminal-Bench tasks remain independently
512+
# gradable even if this diagnostic export fails.
513+
if fixture_for(trial.task_name).requires_evidence and not evidence_exported:
514+
raise RuntimeLaunchError(
515+
"failed to export buzz-evidence.json; the trial has no "
516+
"verifiable relay state and must not be scored"
517+
)
518+
494519
return RuntimeResult(
495520
input_tokens=trial_accounting.input_tokens,
496521
output_tokens=trial_accounting.output_tokens,
@@ -506,6 +531,7 @@ async def run(
506531
# often is a finding, so it is recorded per trial rather than
507532
# inferred later from an empty message id.
508533
"stopped_without_done": final_message is None,
534+
"buzz_evidence_exported": evidence_exported,
509535
"agent_runtime": "in-container",
510536
"agent_active_seconds": timing["agent_active_seconds"],
511537
"usage_settle_seconds": timing["usage_settle_seconds"],
@@ -1238,14 +1264,6 @@ async def _settle_usage(
12381264
return
12391265
await asyncio.sleep(self.poll_seconds)
12401266

1241-
@staticmethod
1242-
async def _turn_ended(environment: BaseEnvironment, agent: _Agent) -> bool:
1243-
result = await environment.exec(
1244-
f"cat {shlex.quote(agent.stdout_log)} "
1245-
f"{shlex.quote(agent.stderr_log)} 2>/dev/null"
1246-
)
1247-
return any(marker in (result.stdout or "") for marker in TURN_ENDED_MARKERS)
1248-
12491267
async def _raise_for_dead_agents(
12501268
self, environment: BaseEnvironment, agents: list[_Agent]
12511269
) -> None:
@@ -1295,15 +1313,15 @@ async def _stop_agents(environment: BaseEnvironment, agents: list[_Agent]) -> No
12951313
await environment.exec(sweep)
12961314
await asyncio.sleep(2)
12971315
await environment.exec(sweep.replace("-TERM", "-KILL"))
1298-
except Exception: # noqa: BLE001 — environment may already be gone
1316+
except Exception: # noqa: BLE001, S110 — environment may already be gone
12991317
pass
13001318

13011319
async def _collect_logs(
13021320
self, environment: BaseEnvironment, trial_dir: Path
13031321
) -> None:
13041322
try:
13051323
await environment.download_dir(REMOTE_LOGS, trial_dir)
1306-
except Exception: # noqa: BLE001 — best effort; env may be torn down
1324+
except Exception: # noqa: BLE001, S110 — best effort; env may be torn down
13071325
pass
13081326

13091327
async def _collect_transcript(self, trial: TrialHandle, trial_dir: Path) -> None:
@@ -1634,9 +1652,11 @@ def _compose_system_prompt(
16341652
"",
16351653
f"You are `{credential.agent_id}` (pubkey `{credential.nostr_pubkey}`).",
16361654
f"The team coordinates in Buzz channel `{trial.channel_id}`.",
1637-
f"Tasks come from the user `{trial.user.agent_id}` "
1638-
f"(pubkey `{trial.user.nostr_pubkey}`); address your final report "
1639-
"to them.",
1655+
(
1656+
f"Tasks come from the user `{trial.user.agent_id}` "
1657+
f"(pubkey `{trial.user.nostr_pubkey}`); address your final report "
1658+
"to them."
1659+
),
16401660
"",
16411661
]
16421662
if teammates:

‎benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
)
2525
from harbor_buzz_orchestra.manifest import ExperimentManifest
2626
from harbor_buzz_orchestra.provisioning import AgentCredential, TrialHandle
27-
from harbor_buzz_orchestra.task_fixtures import fixture_for
2827

2928

3029
def write_manifest(
@@ -554,8 +553,10 @@ async def test_launch_enables_the_usage_log_target(tmp_path):
554553
# the harness reads: tokens, and whether a turn has ended.
555554
(
556555
"buzz_acp=trace",
557-
"buzz_acp=trace,acp::usage=debug,pool::prompt=info,"
558-
"acp::stream=info,acp::tool=info",
556+
(
557+
"buzz_acp=trace,acp::usage=debug,pool::prompt=info,"
558+
"acp::stream=info,acp::tool=info"
559+
),
559560
),
560561
# A target the operator already set keeps the operator's level.
561562
(

0 commit comments

Comments
 (0)