Skip to content

Commit 1aca01e

Browse files
Add buffered Responses for native agents
1 parent 77ad165 commit 1aca01e

19 files changed

Lines changed: 2364 additions & 29 deletions

‎benchmarks/native-sdk/README.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,3 +117,27 @@ SDK_BENCH_SERVER_PYTHON="$PEER_PY" "$NATIVE_PY" "$BENCH/worker.py" \
117117
A `--profile` row is explicitly diagnostic. The comparison rejects diagnostic/profiled/instrumented rows rather than mixing them into latency/CPU summaries. Profiling retains the existing exercise window, which includes setup/warmup/close; instrumented counts or times are not uninstrumented latency shares.
118118

119119
Generated metadata, errors and logs can contain local absolute paths. Review and separately sanitize a public export, preserving original hashes and recording the export's own digest. This directory contains no historical raw results, profiles, installed environments or credentials. Record the source-manifest hash with results; a manifest identifies source bytes, not a performance qualification.
120+
121+
## Optional native buffered Responses mode
122+
123+
The default command keeps the native provider's streaming transport and omits the new provider keyword, so it can still benchmark older native SDK wheels. A native wheel that supports `NativeProvider(response_mode="buffered")` can opt into final-result HTTP JSON delivery with `compare.py --v1-response-mode buffered`. Unsupported wheels fail explicitly; the benchmark never retries in another mode. The stable v0 adapter uses SSE for live workloads and JSON for final-result workloads. Its behavior does not change when selecting the native mode.
124+
125+
Select only final-result workloads for a buffered comparison, for example:
126+
127+
```sh
128+
"$PEER_PY" "$BENCH/compare.py" \
129+
--v0-python "$STABLE_PY" --v1-python "$NATIVE_PY" \
130+
--v0-version 0.24.0 --v1-version 0.24.0-alpha.6 \
131+
--server-python "$PEER_PY" --output-dir "$RESULTS/buffered" \
132+
--v1-response-mode buffered --samples 5 --iterations 20 \
133+
--cases oneshot_text oneshot_tool oneshot_output_64k_1_plain \
134+
oneshot_output_64k_256_plain oneshot_output_64k_4096_plain
135+
```
136+
137+
Omit `--v1-response-mode buffered` and use a fresh output directory for the default SSE control. Keep the same source, wheel, dependencies, interpreter, CA and host conditions. The three 64 KiB cases are final-only `Agent.run` calls in independent fresh conversations, with one prefill and 20 measured calls by default. Their chunk setting controls the SSE reference workload; a JSON response has zero SSE chunks. This removes peer framing work too, so wall-time differences do not isolate client processing. Client process CPU and peer CPU remain separate.
138+
139+
Rows record the native response mode and expected wire transport, checked against all actual request receipts, per-response chunk counts and total emitted chunks. Both modes retain the same final output, input/history and tool-result checks. Buffered comparisons reject live-observation workloads. They report no first-delta metric or token-live claim; buffered progress arrives after the complete body. Existing live streaming cursor, gap and exact chunk checks remain unchanged.
140+
141+
`elapsed_ms`, `cpu_ms` and latency p50/p95 still cover measured operations after prefill. For oneshot cases, these are warmed fresh-conversation measurements. New `full_owner_wall_ms` and `full_owner_cpu_ms` span Adapter construction through successful joined provider/telemetry exit, including prefill and ordinary client scheduling, prompt work and in-scope peer-stat requests. They exclude SDK import, peer startup and final correctness validation. These spans overlap the existing metrics and must not be added to them. Failed runs cannot qualify.
142+
143+
`charged_operations = concurrency * (prefill + iterations)` counts turns, including prefill. It counts fresh conversations only for oneshot cases; conversation cases reuse Sessions. Summaries apply the existing paired estimator to `full_owner_wall_ms_per_operation` and `full_owner_cpu_ms_per_operation`, retaining raw totals and matched vectors. Import-only rows have no owner/operation/transport metrics. RSS stays unqualified and does not enter comparisons. The existing confidence-interval limitations and engineering parity rule still apply.
Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"compare.py": "684a4dba0201b4d6fa23f5ff6decf64334af0596a532daeaa1891624c01ca007",
2+
"compare.py": "69faa65cd95e352d00858eb74de31b182b018fb05e6eed09452eb43d07eabf0d",
33
"owned_process.py": "9be3bd2a11154dec666a943172fd7645e040ef7279f0f53633d5c3f22d168e4e",
44
"protocol_peer.py": "912165f4e3d2abc8d45a0690e08e7b564fcc0562d1ea1a7725de92ba0f841bee",
55
"v0_adapter.py": "36d8e51ce6743c82bc73385c79241a2dddcd2b259ef7f3207f53d0721fccf6d2",
6-
"v1_adapter.py": "5649d0688172337c760af79db245258990be81b0ac6af19b31f74b480f11a350",
6+
"v1_adapter.py": "63e6a993a82c83ec6bfe3f76cb1fab601e25dd92f73e68c5f77423fe274f784b",
77
"validate_peer.py": "5d0829b7cc68310cbfc861bff8927b2e51051757a7d74e0a1b8faa48ab87b527",
8-
"worker.py": "266ee4533be914889fcd00e6fc748d1e8fb078fb54b1012223e99bd6f703d7df",
9-
"workloads.py": "ba5539876ecc2b0ef7d3c0911b89006b207c3fa92c79505a976107736bb82062"
8+
"worker.py": "7ff1b76eb65ff46767318b5b4316503b2297026b2a60f588c3f02f83acda2dcb",
9+
"workloads.py": "a86d44a82b16c6fcb4097ce8a4487093ddc676029836020eebff46c6cf013546"
1010
}

‎benchmarks/native-sdk/compare.py‎

Lines changed: 79 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
sys.path.insert(0, str(Path(__file__).resolve().parent))
2020
from owned_process import _note, defer_cleanup_signals, terminate_group
21-
from worker import reject_inherited_overrides
21+
from worker import reject_inherited_overrides, validate_transport
2222
from workloads import WORKLOADS, get_workload
2323

2424

@@ -78,7 +78,17 @@ def comparison(stable, native):
7878
}
7979

8080

81-
def validate_sample(row, *, side, case, sample, expected_versions, identities, iterations=None):
81+
def validate_sample(
82+
row,
83+
*,
84+
side,
85+
case,
86+
sample,
87+
expected_versions,
88+
identities,
89+
iterations=None,
90+
v1_response_mode="streaming",
91+
):
8292
"""Check scheduling, declared identities and existing worker count receipts."""
8393
if row.get("errors") != 0:
8494
raise ValueError("Worker validation failed")
@@ -124,12 +134,57 @@ def validate_sample(row, *, side, case, sample, expected_versions, identities, i
124134
)
125135
if row.get("workload") != workload.dictionary():
126136
raise ValueError("Worker workload differs from the scheduled work")
137+
if v1_response_mode not in ("streaming", "buffered"):
138+
raise ValueError("Unknown scheduled native response mode")
139+
expected_mode = v1_response_mode if side == "v1" and case != "startup_import" else None
140+
if "response_mode" not in row or row["response_mode"] != expected_mode:
141+
raise ValueError("Native response mode differs from the scheduled mode")
142+
expected_transport = (
143+
None
144+
if case == "startup_import"
145+
else (
146+
"sse"
147+
if (workload.stream if side == "v0" else v1_response_mode == "streaming")
148+
else "json"
149+
)
150+
)
151+
if "wire_transport" not in row or row["wire_transport"] != expected_transport:
152+
raise ValueError("Wire transport metadata differs from the scheduled mode")
127153
if case == "startup_import":
128154
if any(
129-
name in row for name in ("operations", "server", "completed_operations", "tool_calls")
155+
name in row
156+
for name in (
157+
"operations",
158+
"server",
159+
"completed_operations",
160+
"tool_calls",
161+
"charged_operations",
162+
"full_owner_wall_ms",
163+
"full_owner_cpu_ms",
164+
)
130165
):
131166
raise ValueError("Import-only row contains operation/server work")
132167
else:
168+
if v1_response_mode == "buffered" and workload.stream:
169+
raise ValueError("Buffered benchmarks cannot measure token-live workloads")
170+
validate_transport(
171+
workload, row.get("server", {}), engine=side, response_mode=v1_response_mode
172+
)
173+
charged = workload.concurrency * (workload.prefill + workload.iterations)
174+
if type(row.get("charged_operations")) is not int or row["charged_operations"] != charged:
175+
raise ValueError("Charged operation count differs from the scheduled work")
176+
for clock in ("wall", "cpu"):
177+
total = row.get(f"full_owner_{clock}_ms")
178+
per_operation = row.get(f"full_owner_{clock}_ms_per_operation")
179+
if (
180+
type(total) not in (int, float)
181+
or not math.isfinite(total)
182+
or total <= 0
183+
or type(per_operation) not in (int, float)
184+
or not math.isfinite(per_operation)
185+
or not math.isclose(per_operation, total / charged, rel_tol=1e-12)
186+
):
187+
raise ValueError("Invalid full-owner metric or operation denominator")
133188
count = workload.concurrency * workload.iterations
134189
measured_tools = count * workload.tools_per_round * workload.tool_rounds
135190
all_tools = (
@@ -168,7 +223,7 @@ def validate_sample(row, *, side, case, sample, expected_versions, identities, i
168223
identities[side] = identity
169224

170225

171-
def summarize(records, *, expected_versions, iterations=None):
226+
def summarize(records, *, expected_versions, iterations=None, v1_response_mode="streaming"):
172227
summary, identities = {}, {}
173228
for case, sides in records.items():
174229
if set(sides) != {"v0", "v1"}:
@@ -195,6 +250,7 @@ def summarize(records, *, expected_versions, iterations=None):
195250
expected_versions=expected_versions,
196251
identities=identities,
197252
iterations=iterations,
253+
v1_response_mode=v1_response_mode,
198254
)
199255
fields = ["import_ms"]
200256
if case != "startup_import":
@@ -207,6 +263,8 @@ def summarize(records, *, expected_versions, iterations=None):
207263
"elapsed_ms",
208264
"cpu_ms",
209265
"final_close_ms",
266+
"full_owner_wall_ms_per_operation",
267+
"full_owner_cpu_ms_per_operation",
210268
]
211269
if case != "startup_import" and WORKLOADS[case].stream:
212270
fields += ["first_delta_p50_ms", "first_delta_p95_ms"]
@@ -241,6 +299,9 @@ def main():
241299
parser.add_argument("--v1-python", type=Path, required=True)
242300
parser.add_argument("--v0-version", required=True)
243301
parser.add_argument("--v1-version", required=True)
302+
parser.add_argument(
303+
"--v1-response-mode", choices=["streaming", "buffered"], default="streaming"
304+
)
244305
parser.add_argument("--server-python", type=Path, required=True)
245306
parser.add_argument("--output-dir", type=Path, required=True)
246307
parser.add_argument("--samples", type=int, default=5)
@@ -265,6 +326,12 @@ def main():
265326
or not args.v1_version.strip()
266327
):
267328
parser.error("Use positive counts/timeout, distinct cases and explicit expected versions")
329+
if args.v1_response_mode == "buffered" and any(
330+
case != "startup_import" and WORKLOADS[case].stream for case in args.cases
331+
):
332+
parser.error(
333+
"Buffered comparisons require final-result workloads; select --cases explicitly"
334+
)
268335
expected_versions = {"v0": args.v0_version, "v1": args.v1_version}
269336
identities = {}
270337

@@ -295,6 +362,7 @@ def stop_requested(signum, _frame):
295362
},
296363
"sample_order": [],
297364
"expected_sdk_versions": expected_versions,
365+
"v1_response_mode": args.v1_response_mode,
298366
"completed": False,
299367
"rss_scope": "Unqualified raw process ru_maxrss only; no RSS comparison or verdict",
300368
"statistics": (
@@ -341,6 +409,8 @@ def stop_requested(signum, _frame):
341409
"--sample-index",
342410
str(sample),
343411
]
412+
if side == "v1" and args.v1_response_mode == "buffered":
413+
command += ["--response-mode", "buffered"]
344414
if case == "startup_import":
345415
command.append("--import-only")
346416
else:
@@ -421,6 +491,7 @@ def stop_requested(signum, _frame):
421491
expected_versions=expected_versions,
422492
identities=identities,
423493
iterations=args.iterations,
494+
v1_response_mode=args.v1_response_mode,
424495
)
425496
records[case][side].append(data)
426497
value = data.get("latency_p50_ms", data["import_ms"])
@@ -430,7 +501,10 @@ def stop_requested(signum, _frame):
430501
flush=True,
431502
)
432503
summary = summarize(
433-
records, expected_versions=expected_versions, iterations=args.iterations
504+
records,
505+
expected_versions=expected_versions,
506+
iterations=args.iterations,
507+
v1_response_mode=args.v1_response_mode,
434508
)
435509
(args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")
436510
primary = summary[case]["primary"]

‎benchmarks/native-sdk/tests/test_comparison.py‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ def row(side, sample, case="oneshot_text"):
2323
workload = get_workload("oneshot_text" if case == "startup_import" else case)
2424
value = sample + 10.0
2525
result = {
26+
"response_mode": "streaming" if side == "v1" and case != "startup_import" else None,
27+
"wire_transport": None
28+
if case == "startup_import"
29+
else ("sse" if side == "v1" or workload.stream else "json"),
2630
"engine": side,
2731
"case": case,
2832
"sample_index": sample,
@@ -57,6 +61,30 @@ def row(side, sample, case="oneshot_text"):
5761
* workload.tools_per_round
5862
* workload.tool_rounds,
5963
)
64+
charged = workload.concurrency * (workload.prefill + workload.iterations)
65+
result["charged_operations"] = charged
66+
for clock in ("wall", "cpu"):
67+
result[f"full_owner_{clock}_ms"] = value * charged
68+
result[f"full_owner_{clock}_ms_per_operation"] = value
69+
streaming = side == "v1" or workload.stream
70+
receipts = [
71+
{
72+
"stream": streaming,
73+
"round": round_index,
74+
"text_chunks": workload.chunks
75+
if streaming and round_index == workload.tool_rounds
76+
else 0,
77+
}
78+
for _ in range(charged)
79+
for round_index in range(workload.tool_rounds + 1)
80+
]
81+
result["server"] = {
82+
"requests": len(receipts),
83+
"receipts": receipts,
84+
"errors": 0,
85+
"receipts_dropped": 0,
86+
"emitted_text_chunks": charged * workload.chunks if streaming else 0,
87+
}
6088
for field in (
6189
"cold_first_result_ms",
6290
"construction_ms",
@@ -289,6 +317,110 @@ def test_scheduled_sample_is_checked_before_append(self):
289317
self.assertEqual(identities, {})
290318

291319

320+
class BufferedQualificationTests(unittest.TestCase):
321+
def test_buffered_final_result_keeps_json_receipts_and_no_live_metrics(self):
322+
value = records("oneshot_output_64k_256_plain")
323+
for item in value["oneshot_output_64k_256_plain"]["v1"]:
324+
item.update(response_mode="buffered", wire_transport="json")
325+
item["server"]["emitted_text_chunks"] = 0
326+
for receipt in item["server"]["receipts"]:
327+
receipt.update(stream=False, text_chunks=0)
328+
summary = compare.summarize(value, expected_versions=VERSIONS, v1_response_mode="buffered")
329+
metrics = summary["oneshot_output_64k_256_plain"]["metrics"]
330+
self.assertNotIn("first_delta_p50_ms", metrics)
331+
self.assertEqual(
332+
metrics["full_owner_cpu_ms_per_operation"]["stable_values"], [10, 11, 12, 13, 14]
333+
)
334+
335+
def test_mode_transport_and_denominator_drift_fail_closed(self):
336+
for change in (
337+
lambda item: item.update(response_mode="buffered"),
338+
lambda item: item.update(wire_transport="json"),
339+
lambda item: item["server"]["receipts"][0].update(stream=False),
340+
lambda item: item["server"].update(emitted_text_chunks=0),
341+
lambda item: item.update(charged_operations=20),
342+
lambda item: item.update(full_owner_cpu_ms_per_operation=1),
343+
lambda item: item.update(full_owner_cpu_ms=float("nan")),
344+
):
345+
value = records()
346+
change(value["oneshot_text"]["v1"][0])
347+
with self.subTest(change=change), self.assertRaises(ValueError):
348+
compare.summarize(value, expected_versions=VERSIONS)
349+
350+
def test_stable_live_stream_is_sse_and_buffered_live_is_rejected(self):
351+
value = records("warm_stream")
352+
compare.summarize(value, expected_versions=VERSIONS)
353+
value["warm_stream"]["v0"][0]["server"]["receipts"][0]["stream"] = False
354+
with self.assertRaises(ValueError):
355+
compare.summarize(value, expected_versions=VERSIONS)
356+
with self.assertRaises(ValueError):
357+
compare.summarize(
358+
records("warm_stream"), expected_versions=VERSIONS, v1_response_mode="buffered"
359+
)
360+
361+
def test_large_cases_remain_independent_final_result_operations(self):
362+
for chunks in (1, 256, 4096):
363+
workload = get_workload(f"oneshot_output_64k_{chunks}_plain")
364+
self.assertEqual(
365+
(
366+
workload.mode,
367+
workload.stream,
368+
workload.output_chars,
369+
workload.chunks,
370+
workload.prefill,
371+
workload.iterations,
372+
),
373+
("oneshot", False, 65536, chunks, 1, 20),
374+
)
375+
376+
377+
class FullOwnerBoundaryTests(unittest.IsolatedAsyncioTestCase):
378+
async def test_owner_clocks_include_construction_prefill_and_joined_exit(self):
379+
import worker
380+
381+
events = []
382+
counter = 0
383+
384+
def cpu():
385+
nonlocal counter
386+
counter += 1
387+
events.append("cpu")
388+
return counter * 1_000_000
389+
390+
class Adapter:
391+
def __init__(self, *args, **kwargs):
392+
self.tool_calls = 0
393+
events.append("construct")
394+
395+
async def __aenter__(self):
396+
events.append("enter")
397+
return self
398+
399+
async def __aexit__(self, *args):
400+
events.append("joined_exit")
401+
402+
async def run(self, prompt):
403+
events.append("run")
404+
return {"text": "answer", "deltas": [], "first_delta_ns": None}
405+
406+
workload = get_workload("oneshot_text", iterations=1)
407+
with (
408+
mock.patch.object(worker.time, "process_time_ns", side_effect=cpu),
409+
mock.patch.object(
410+
worker, "stats_at", return_value={"process_cpu_ns": 0, "monotonic_ns": 1}
411+
),
412+
mock.patch.object(worker, "expected_text", return_value="answer"),
413+
):
414+
result = await worker.exercise(Adapter, workload, "unused", 0, None)
415+
self.assertLess(events.index("cpu"), events.index("construct"))
416+
self.assertLess(events.index("joined_exit"), len(events) - 1)
417+
self.assertEqual(events[-1], "cpu")
418+
self.assertEqual(events.count("run"), 2)
419+
self.assertEqual(result["charged_operations"], 2)
420+
self.assertEqual(result["full_owner_cpu_ms"], 3)
421+
self.assertEqual(result["full_owner_cpu_ms_per_operation"], 1.5)
422+
423+
292424
class InstrumentationTests(unittest.TestCase):
293425
def test_known_inherited_overrides_are_rejected_even_when_empty(self):
294426
for name in (

0 commit comments

Comments
 (0)