Skip to content

Commit 5e22427

Browse files
authored
benchmarking: add a turnover mode to agentsession (agent-substrate#3421)
### Summary Adds `--agentsession-awake` (paired with `--target-rps`) to `agentsession` so benchmarks can maintain a large pool of registered sessions while keeping a fixed number resident and swapping idle awake sessions with parked ones at a controlled rate. ### How It Works - **Registration (`createSession`)**: When `--agentsession-awake > 0` or `--lifecycle-mode none`, each session runs step 1 as soon as its actor serves so the initial actor/snapshot holds real memory and disk state. In `--lifecycle-mode none` at `--agentsession-think-scale 1`, `personal-assistant-light` waits 55m before step 1, so a 20m resident density run would otherwise measure empty sandboxes. - **Fill**: Once session registration has been idle for 1 minute (`registrationQuiet`), the conductor wakes `--agentsession-awake` sessions (`fillConcurrency = 8` at a time) and logs `agentsession: turnover fill done`. - **Turnover (`--target-rps`)**: Each second, the conductor swaps `--target-rps` idle awake sessions (`cmdPark`) with parked sessions (`cmdWake`). Each woken session immediately runs its next scripted step while marked `busy` and returns to `idle` when the step finishes. - **Measurement isolation**: - Registration parks record under `SuspendActor_Register` / `SuspendActorAsync_Register` so `SuspendActor` and `SuspendActorAsync` hold only turnover parks. - `runner.py` parses `agentsession: turnover fill done` (`locust/turnover_window.py`), starts `server_telemetry`'s steady-state window there, and appends a `turnover_window` row to `stats.jsonl` with per-request counts, failures, and rates from `stats_history.csv` over the turnover phase. - **Observability**: Exports `agentsession_turnover_sessions{state="awake|parked|waking|parking"}`. ### Testing ```bash go test -race -count=3 -shuffle=on ./internal/benchmarking/boomer/agentsession/... python3 -P -m unittest discover -s benchmarking/locust/unit_tests -p "test_*.py" ``` --------- Signed-off-by: Chenyi Wang <[email protected]>
1 parent c53f4b2 commit 5e22427

16 files changed

Lines changed: 1502 additions & 78 deletions

‎benchmarking/locust/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ COPY benchmarking/locust/cluster_facts.py /app/cluster_facts.py
6161
COPY benchmarking/locust/server_telemetry.py /app/server_telemetry.py
6262
COPY benchmarking/locust/phase_breakdown.py /app/phase_breakdown.py
6363
COPY benchmarking/locust/ladder_steps.py /app/ladder_steps.py
64+
COPY benchmarking/locust/turnover_window.py /app/turnover_window.py
6465
COPY benchmarking/analysis/phase_report.py /app/phase_report.py
6566

6667
ENV PYTHONPATH=/app:/app/deps

‎benchmarking/locust/common/agentsession_config.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,3 +96,14 @@ def add_agentsession_arguments(parser: LocustArgumentParser) -> None:
9696
),
9797
include_in_web_ui=True,
9898
)
99+
parser.add_argument(
100+
"--agentsession-awake",
101+
type=int,
102+
default=0,
103+
env_var="LOCUST_AGENTSESSION_AWAKE",
104+
help=(
105+
"Turnover mode when > 0: keep this many sessions awake and swap "
106+
"--target-rps sessions/s with parked ones (default: 0, off)"
107+
),
108+
include_in_web_ui=True,
109+
)

‎benchmarking/locust/common/boomer_config.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ class that consumes it.
8181
"--agentsession-snapshot-steps": str,
8282
"--agentsession-phase-in": float,
8383
"--agentsession-resume-retry-budget": float,
84+
"--agentsession-awake": int,
8485
"--total-actors": int,
8586
"--spawn-concurrency": int,
8687
"--actor-deadline": float,

‎benchmarking/locust/common/rate_config.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# limitations under the License.
1414

1515
"""Request-rate runtime flag, for boomer user classes that pace
16-
(resumepaused)."""
16+
(resumepaused, and agentsession in turnover mode)."""
1717

1818
from locust import events
1919
from locust.argument_parser import LocustArgumentParser
@@ -27,8 +27,8 @@ def add_rate_arguments(parser: LocustArgumentParser) -> None:
2727
default=0.0,
2828
env_var="LOCUST_TARGET_RPS",
2929
help=(
30-
"Requests per second the whole boomer worker paces to, for user "
31-
"classes that pace (resumepaused). 0 removes the cap. "
30+
"Requests/s cap for resumepaused, or swaps/s for agentsession "
31+
"turnover mode (--agentsession-awake). 0 disables both. "
3232
"A --ladder step with @target_rps changes it while the run "
3333
"continues"
3434
),

‎benchmarking/locust/runner.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
from server_telemetry import extract_and_record_server_telemetry
5656
from phase_breakdown import append_phase_breakdown
5757
from ladder_steps import append_ladder_steps
58+
from turnover_window import append_turnover_window, fill_done_ts
5859

5960
# Path inside the locust image to the boomer-worker binary baked in by
6061
# benchmarking/locust/Dockerfile.
@@ -638,6 +639,18 @@ def main() -> None:
638639
except Exception as e:
639640
tee(logs, f"Warning: Failed to record cluster facts: {e}")
640641

642+
# A turnover run is measured from the end of its fill, past the
643+
# registration of its fleet: the server telemetry's steady window and
644+
# the turnover_window row start there. Other runs have no such line.
645+
turnover_start = None
646+
try:
647+
with open(logs_path, errors="replace") as f:
648+
turnover_start = fill_done_ts(f)
649+
if turnover_start is not None:
650+
tee(logs, f"Turnover fill done at {turnover_start}; measuring the turnover from there")
651+
except Exception as e:
652+
tee(logs, f"Warning: Failed to read the turnover start: {e}")
653+
641654
# Server telemetry is a Prometheus time-window query, so it runs
642655
# either way. A run too loaded to write a CSV is the one its
643656
# bin-packing, PSI and snapshot numbers matter most for.
@@ -655,6 +668,7 @@ def main() -> None:
655668
logs=logs,
656669
lag_s=args.atelet_lag_s,
657670
worker_pods=diff_worker_pods(pods_before, pods_after, evicted_uids),
671+
steady_not_before=turnover_start,
658672
)
659673
except Exception as e:
660674
tee(logs, f"Warning: Failed to harvest server telemetry: {e}")
@@ -672,6 +686,16 @@ def main() -> None:
672686
except Exception as e:
673687
tee(logs, f"Warning: Failed to summarize ladder steps: {e}")
674688

689+
if turnover_start is not None:
690+
try:
691+
if append_turnover_window(
692+
jsonl_path, turnover_start, stats_history_csv, data_ts,
693+
args.tag, args.name, logs,
694+
):
695+
tee(logs, f"Appended the turnover_window row to {jsonl_path}")
696+
except Exception as e:
697+
tee(logs, f"Warning: Failed to summarize the turnover window: {e}")
698+
675699
# The phase breakdown is additive too. It reads the node logs now,
676700
# while the pods still exist: the orchestrator deletes them as soon
677701
# as this process exits. The window reaches back to this run's start

‎benchmarking/locust/server_telemetry.py‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,27 @@ def get_steady_state_window(
163163
return start_ts, end_ts
164164

165165

166+
def start_steady_window_at(
167+
window: tuple[int, int],
168+
not_before: int | None,
169+
log: Callable[[str], None] | None = None,
170+
) -> tuple[int, int]:
171+
"""The steady-state window, started at not_before when that falls inside
172+
[start, end). A turnover run is measured from the end of its fill
173+
(turnover_window.py), which comes after all its users have started."""
174+
start, end = window
175+
if not_before is None:
176+
return start, end
177+
if start <= not_before < end:
178+
return not_before, end
179+
if log is not None:
180+
log(
181+
f"Turnover start {not_before} lies outside detected steady-state "
182+
f"window [{start}, {end}]; keeping [{start}, {end}]"
183+
)
184+
return start, end
185+
186+
166187
def _parse_instant_float(res: list[dict[str, Any]]) -> float | None:
167188
if res and "value" in res[0]:
168189
try:
@@ -802,12 +823,14 @@ def extract_and_record_server_telemetry(
802823
logs: TextIO | None = None,
803824
lag_s: int = 70,
804825
worker_pods: dict[str, int | None] | None = None,
826+
steady_not_before: int | None = None,
805827
) -> None:
806828
"""Entry point called by runner.py to query Prometheus and persist artifacts.
807829
808830
Waits until `end_ts + lag_s` first, so the atelet's last export of the run
809831
has been scraped; lag_s must cover its export interval plus one scrape.
810832
`worker_pods` (WORKER_POD_FIELDS) joins `oom_events`; null when not given.
833+
`steady_not_before`, when inside the steady-state window, starts it.
811834
"""
812835
def log(msg: str) -> None:
813836
if logs:
@@ -820,8 +843,10 @@ def log(msg: str) -> None:
820843
time.sleep(wait_s)
821844

822845
log(f"Harvesting Prometheus metrics from {prom_url} over [{start_ts}, {end_ts}]...")
823-
steady_start, steady_end = get_steady_state_window(
824-
stats_history_csv, start_ts, end_ts
846+
steady_start, steady_end = start_steady_window_at(
847+
get_steady_state_window(stats_history_csv, start_ts, end_ts),
848+
steady_not_before,
849+
log=log,
825850
)
826851
log(
827852
f"Detected steady-state window: [{steady_start}, {steady_end}] "
Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""The turnover window of an agentsession run in turnover mode.
16+
17+
In turnover mode (--agentsession-awake) a run first registers its whole
18+
fleet: each session creates its actor, runs step 1 and parks. The conductor
19+
then wakes the awake sessions, and only after that fill does it swap at
20+
--target-rps. stats.jsonl's other rows cover the whole run, so their rates
21+
divide by the time spent registering. This finds the end of the fill in the
22+
line boomer logs then, and writes one `turnover_window` row with each
23+
request's count, failures and rate from there to the end of the run, read
24+
from stats_history.csv. runner.py also starts the server telemetry's
25+
steady-state window there.
26+
27+
Like the server telemetry, this is additive: runner.py logs a failure here
28+
and keeps the locust measurements.
29+
"""
30+
31+
import csv
32+
import json
33+
import re
34+
from datetime import datetime, timezone
35+
from pathlib import Path
36+
from typing import Any, Iterable, TextIO
37+
38+
# What agentsession logs once its conductor has woken the awake sessions,
39+
# right before it starts to swap.
40+
FILL_DONE_MSG = "agentsession: turnover fill done"
41+
42+
# The prefix runner.py puts on each line boomer writes to the run's log.
43+
BOOMER_PREFIX = "[boomer] "
44+
45+
# An RFC 3339 time as Go writes it. Its fraction can hold nanoseconds, which
46+
# datetime does not parse, and only the seconds matter here.
47+
_TIME_RE = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)(?:\.\d+)?(Z|[+-]\d\d:\d\d)$")
48+
49+
50+
def _unix(ts: str) -> int:
51+
m = _TIME_RE.match(ts)
52+
if not m:
53+
raise ValueError(f"not an RFC 3339 time: {ts!r}")
54+
t = datetime.strptime(m[1], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc)
55+
offset = 0
56+
if m[2] != "Z":
57+
sign = 1 if m[2][0] == "+" else -1
58+
offset = sign * (int(m[2][1:3]) * 3600 + int(m[2][4:6]) * 60)
59+
return int(t.timestamp()) - offset
60+
61+
62+
def fill_done_ts(lines: Iterable[str]) -> int | None:
63+
"""Unix seconds of the first fill-done line in the run's log, or None
64+
when there is none: the run was not in turnover mode, or its fill never
65+
finished."""
66+
for line in lines:
67+
if not line.startswith(BOOMER_PREFIX) or FILL_DONE_MSG not in line:
68+
continue
69+
try:
70+
entry = json.loads(line[len(BOOMER_PREFIX):])
71+
if entry.get("msg") == FILL_DONE_MSG:
72+
return _unix(entry["time"])
73+
except (AttributeError, KeyError, TypeError, ValueError):
74+
continue
75+
return None
76+
77+
78+
def window_counts(
79+
stats_history_csv: Path, start: int
80+
) -> tuple[int, dict[str, tuple[int, int]]] | None:
81+
"""The time of the last sample in stats_history.csv, and each request's
82+
requests and failures from start to it, or None when no sample comes
83+
after start. The counts come from the cumulative columns of the
84+
per-request rows that --csv-full-history writes: the last sample's minus
85+
the last one at or before start."""
86+
before: dict[str, tuple[int, int]] = {}
87+
after: dict[str, tuple[int, int]] = {}
88+
end = None
89+
with open(stats_history_csv, encoding="utf-8") as f:
90+
for row in csv.DictReader(f):
91+
name = row.get("Name") or ""
92+
if name in ("", "Aggregated", "Total"):
93+
continue
94+
try:
95+
ts = int(row["Timestamp"])
96+
counts = (int(row["Total Request Count"]), int(row["Total Failure Count"]))
97+
except (KeyError, TypeError, ValueError):
98+
continue
99+
if ts <= start:
100+
before[name] = counts
101+
else:
102+
after[name] = counts
103+
end = ts if end is None else max(end, ts)
104+
if end is None:
105+
return None
106+
return end, {
107+
name: (requests - before.get(name, (0, 0))[0], failures - before.get(name, (0, 0))[1])
108+
for name, (requests, failures) in after.items()
109+
}
110+
111+
112+
def window_record(
113+
start: int,
114+
end: int,
115+
counts: dict[str, tuple[int, int]],
116+
timestamp: str,
117+
tag: str,
118+
test_name: str,
119+
) -> dict[str, Any]:
120+
"""The window as a stats.jsonl row: each request's count, failures and
121+
rate. The rate counts every request, failed or not, like locust's
122+
Requests/s."""
123+
secs = end - start
124+
m: dict[str, Any] = {"window_start": start, "window_end": end, "window_s": secs}
125+
for name, (requests, failures) in sorted(counts.items()):
126+
m[f"requests_{name}"] = requests
127+
m[f"failures_{name}"] = failures
128+
m[f"rps_{name}"] = round(requests / secs, 4) if secs > 0 else None
129+
return {
130+
"timestamp": timestamp,
131+
"tag": tag,
132+
"test_name": test_name,
133+
"metric": "turnover_window",
134+
"measurements": m,
135+
}
136+
137+
138+
def append_turnover_window(
139+
jsonl_path: Path,
140+
start: int,
141+
stats_history_csv: Path,
142+
timestamp: str,
143+
tag: str,
144+
test_name: str,
145+
logs: TextIO,
146+
) -> bool:
147+
"""Append the turnover_window row of a run whose fill ended at start to
148+
jsonl_path, and return whether there was one to append."""
149+
if not stats_history_csv.exists():
150+
_log(logs, f"No {stats_history_csv}; no turnover_window row")
151+
return False
152+
counts = window_counts(stats_history_csv, start)
153+
if counts is None:
154+
_log(logs, f"No stats_history sample after the turnover start {start}; no turnover_window row")
155+
return False
156+
end, per_request = counts
157+
row = window_record(start, end, per_request, timestamp, tag, test_name)
158+
with open(jsonl_path, "a", encoding="utf-8") as f:
159+
f.write(json.dumps(row) + "\n")
160+
return True
161+
162+
163+
def _log(logs: TextIO, msg: str) -> None:
164+
print(msg, flush=True)
165+
logs.write(msg + "\n")
166+
logs.flush()

‎benchmarking/locust/unit_tests/test_boomer_config.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,17 @@ def test_build_config_json_phase_in_and_retry_budget(self):
6060
self.assertEqual(parsed["agentsession_phase_in"], 300.0)
6161
self.assertEqual(parsed["agentsession_resume_retry_budget"], 0.0)
6262

63+
def test_build_config_json_turnover(self):
64+
raw = boomer_config.build_config_json([
65+
"--agentsession-awake",
66+
"650",
67+
"--target-rps",
68+
"2.5",
69+
])
70+
parsed = json.loads(raw)
71+
self.assertEqual(parsed["agentsession_awake"], 650)
72+
self.assertEqual(parsed["target_rps"], 2.5)
73+
6374

6475
if __name__ == "__main__":
6576
unittest.main()

‎benchmarking/locust/unit_tests/test_server_telemetry.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -808,6 +808,35 @@ def ladder(jsonl_path, log_path, prom_url, *_):
808808
self.assertIn("Failed to summarize ladder steps: prometheus down", out.getvalue())
809809
self.assertTrue(upload.called)
810810

811+
# The fill-done line of a turnover run starts the telemetry's steady
812+
# window and the turnover_window row; a run without one gets neither.
813+
@mock.patch.object(runner, "upload")
814+
def test_turnover_start_reaches_telemetry_and_window(self, _up):
815+
for line, start in (
816+
('[boomer] {"time":"2026-10-30T00:04:10Z","msg":"agentsession: turnover fill done"}\n',
817+
1793318650),
818+
('[boomer] {"time":"2026-10-30T00:04:10Z","msg":"dynconfig applied"}\n', None),
819+
):
820+
with self.subTest(start=start):
821+
def run_test(_args, _csv_prefix, logs, _traces, line=line):
822+
logs.write(line)
823+
logs.flush()
824+
return 1
825+
826+
telemetry, window = mock.Mock(), mock.Mock(return_value=True)
827+
with mock.patch.object(runner, "run_test", run_test), \
828+
mock.patch.object(runner, "extract_and_record_server_telemetry", telemetry), \
829+
mock.patch.object(runner, "append_turnover_window", window), \
830+
mock.patch.object(sys, "argv", ARGV + ["--no-cluster-facts",
831+
"--allow-empty-stats"]), \
832+
contextlib.redirect_stdout(io.StringIO()):
833+
runner.main()
834+
self.assertEqual(telemetry.call_args.kwargs["steady_not_before"], start)
835+
if start is None:
836+
self.assertFalse(window.called)
837+
else:
838+
self.assertEqual(window.call_args.args[1], start)
839+
811840

812841
if __name__ == "__main__":
813842
unittest.main()

0 commit comments

Comments
 (0)