1515import os
1616import shlex
1717import time
18+ import traceback
1819from dataclasses import dataclass , field
1920from pathlib import Path
2021from typing import Any
2425
2526from . import accounting , bundle
2627from .accounting import TURN_ENDED_MARKERS , USAGE_MARKER
28+ from .evidence import build_buzz_evidence
2729from .manifest import AgentClass , ExperimentManifest , GenerationConfig
2830from .provisioning import AgentCredential , TrialHandle
2931from .runtime import RuntimeResult
3032from .task_fixtures import fixture_for
3133
32-
3334# Tool/LLM rounds allowed per agent turn (BUZZ_AGENT_MAX_ROUNDS).
3435#
3536# Deliberately set high enough that no condition reaches it, because the cap is
@@ -353,6 +354,9 @@ async def run(
353354
354355 agents : list [_Agent ] = []
355356 infra : list [_Agent ] = []
357+ task_event_id : str | None = None
358+ final_message : dict [str , Any ] | None = None
359+ evidence_exported = False
356360 trust_store = "unknown"
357361 verifier_deps = "unknown"
358362 try :
@@ -393,10 +397,16 @@ async def run(
393397 )
394398 # The task arrives exactly as it would in production Buzz: a
395399 # user prompt @mentioning the orchestrator. The harness never
396- # speaks as any agent.
400+ # speaks as any agent. The orchestrator is mentioned by pubkey,
401+ # not by name resolution: task text is untrusted payload, and any
402+ # @-token inside it must remain part of the task rather than being
403+ # interpreted as another relay mention.
397404 active_started = time .monotonic ()
398- await self ._send (
399- trial .user , trial , f"@{ orchestrator .agent_id } { instruction } "
405+ task_event = await self ._send (
406+ trial .user ,
407+ trial ,
408+ f"@{ orchestrator .agent_id } { instruction } " ,
409+ mention = orchestrator .nostr_pubkey ,
400410 )
401411 if isinstance (task_event , dict ) and isinstance (
402412 task_event .get ("event_id" ), str
@@ -448,12 +458,18 @@ async def run(
448458 # agent/buzz/ holding nothing but the system prompts, which reads
449459 # as "the agents never ran" rather than "teardown stalled".
450460 await self ._collect_logs (environment , trial_dir )
461+ evidence_exported = await self ._collect_evidence (
462+ environment = environment ,
463+ trial = trial ,
464+ trial_dir = trial_dir ,
465+ task_event_id = task_event_id ,
466+ completion_message_id = (
467+ final_message .get ("id" ) if final_message is not None else None
468+ ),
469+ )
451470 # After the agents are stopped, so the tool never reaches them, and
452471 # before Harbor's verifier phase, which is what needs it.
453472 verifier_deps = await self ._preinstall_verifier_deps (environment )
454- # Before bundle.write, so the transcript is listed in the bundle's
455- # file index and covered by its secret scan.
456- await self ._collect_transcript (trial , trial_dir )
457473 # Accounting runs even when the trial failed or timed out: a trial
458474 # that burned tokens and then stalled still cost money, and
459475 # excluding it would bias the sweep's cost figures toward successes.
@@ -491,6 +507,15 @@ async def run(
491507 endpoints = self .endpoints ,
492508 )
493509
510+ # A missing snapshot is a harness failure only for Buzz-native tasks
511+ # that grade relay state. Terminal-Bench tasks remain independently
512+ # gradable even if this diagnostic export fails.
513+ if fixture_for (trial .task_name ).requires_evidence and not evidence_exported :
514+ raise RuntimeLaunchError (
515+ "failed to export buzz-evidence.json; the trial has no "
516+ "verifiable relay state and must not be scored"
517+ )
518+
494519 return RuntimeResult (
495520 input_tokens = trial_accounting .input_tokens ,
496521 output_tokens = trial_accounting .output_tokens ,
@@ -506,6 +531,7 @@ async def run(
506531 # often is a finding, so it is recorded per trial rather than
507532 # inferred later from an empty message id.
508533 "stopped_without_done" : final_message is None ,
534+ "buzz_evidence_exported" : evidence_exported ,
509535 "agent_runtime" : "in-container" ,
510536 "agent_active_seconds" : timing ["agent_active_seconds" ],
511537 "usage_settle_seconds" : timing ["usage_settle_seconds" ],
@@ -1238,14 +1264,6 @@ async def _settle_usage(
12381264 return
12391265 await asyncio .sleep (self .poll_seconds )
12401266
1241- @staticmethod
1242- async def _turn_ended (environment : BaseEnvironment , agent : _Agent ) -> bool :
1243- result = await environment .exec (
1244- f"cat { shlex .quote (agent .stdout_log )} "
1245- f"{ shlex .quote (agent .stderr_log )} 2>/dev/null"
1246- )
1247- return any (marker in (result .stdout or "" ) for marker in TURN_ENDED_MARKERS )
1248-
12491267 async def _raise_for_dead_agents (
12501268 self , environment : BaseEnvironment , agents : list [_Agent ]
12511269 ) -> None :
@@ -1295,15 +1313,15 @@ async def _stop_agents(environment: BaseEnvironment, agents: list[_Agent]) -> No
12951313 await environment .exec (sweep )
12961314 await asyncio .sleep (2 )
12971315 await environment .exec (sweep .replace ("-TERM" , "-KILL" ))
1298- except Exception : # noqa: BLE001 — environment may already be gone
1316+ except Exception : # noqa: BLE001, S110 — environment may already be gone
12991317 pass
13001318
13011319 async def _collect_logs (
13021320 self , environment : BaseEnvironment , trial_dir : Path
13031321 ) -> None :
13041322 try :
13051323 await environment .download_dir (REMOTE_LOGS , trial_dir )
1306- except Exception : # noqa: BLE001 — best effort; env may be torn down
1324+ except Exception : # noqa: BLE001, S110 — best effort; env may be torn down
13071325 pass
13081326
13091327 async def _collect_transcript (self , trial : TrialHandle , trial_dir : Path ) -> None :
@@ -1634,9 +1652,11 @@ def _compose_system_prompt(
16341652 "" ,
16351653 f"You are `{ credential .agent_id } ` (pubkey `{ credential .nostr_pubkey } `)." ,
16361654 f"The team coordinates in Buzz channel `{ trial .channel_id } `." ,
1637- f"Tasks come from the user `{ trial .user .agent_id } ` "
1638- f"(pubkey `{ trial .user .nostr_pubkey } `); address your final report "
1639- "to them." ,
1655+ (
1656+ f"Tasks come from the user `{ trial .user .agent_id } ` "
1657+ f"(pubkey `{ trial .user .nostr_pubkey } `); address your final report "
1658+ "to them."
1659+ ),
16401660 "" ,
16411661 ]
16421662 if teammates :
0 commit comments