Skip to content

Commit 0ddbf78

Browse files
authored
benchmarking: add --test-name to run selected orchestrator tests (agent-substrate#3464)
The orchestrator ran every entry in the `--tests` file, so running one test meant copying it into a new file first. `--test-name NAME` runs only the named entries and can be repeated. The orchestrator now selects and validates the entries before it waits for Docker and clones the repo, so a mistyped name fails right away. A relative `--tests` path now resolves against the working directory instead of the clone. <!-- e.g. "AI-assisted, I reviewed and tested it." No AI co-author trailers. --> AI use: AI-assisted, I reviewed and tested it <!-- Breaking change? Describe what users must do on upgrade and ask for the `breaking-change` label. --> - [x] Tests pass - [x] Docs updated Signed-off-by: Haiyan Meng <[email protected]>
1 parent a1ee475 commit 0ddbf78

4 files changed

Lines changed: 52 additions & 25 deletions

File tree

‎benchmarking/automation/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ To run the cold resume tests from your workstation instead, see
4040
requests=limits and Envoy's thread count to its own `envoyCpu`, then
4141
waits for the rollout before deploying workloads. The teardown after
4242
each test redeploys substrate, so the pin never outlives its run.
43-
6. For each test in `tests.yaml`:
43+
6. For each test in `tests.yaml`, or only those named by `--test-name`:
4444
- Submits a Job using the just-built image for the test's type
4545
(`runner-job.yaml.tmpl` for locust,
4646
`nighthawk-ingress-runner-job.yaml.tmpl` for nighthawk-ingress).

‎benchmarking/automation/ateapi-cold-resume.md‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -66,17 +66,8 @@ write access to the results bucket. See
6666

6767
## 4. Pick the tests
6868

69-
The orchestrator runs every test in the file you pass to `--tests`, so copy
70-
the entries you want into a new file:
71-
72-
```bash
73-
python3 - <<'PY' > /tmp/cold-tests.yaml
74-
import yaml
75-
want = {"ateapi_cold_resume_pause_fake_smoke"}
76-
t = yaml.safe_load(open("benchmarking/automation/tests.yaml"))["tests"]
77-
print(yaml.safe_dump({"tests": [e for e in t if e["name"] in want]}, sort_keys=False))
78-
PY
79-
```
69+
The orchestrator runs every test in the file you pass to `--tests`. To run
70+
only some of them, pass `--test-name NAME` once per entry, as in step 6.
8071

8172
| Test | Load | Duration |
8273
|---|---|---|
@@ -127,7 +118,8 @@ PY
127118
$S/benchmarking/automation/venv/bin/python -u $D/run.py \
128119
--repo file://$S --branch $(git branch --show-current) \
129120
--dest gs://<results-bucket>/<prefix> \
130-
--tests /tmp/cold-tests.yaml \
121+
--tests $S/benchmarking/automation/tests.yaml \
122+
--test-name ateapi_cold_resume_pause_fake_smoke \
131123
--target-cluster-dir $D/target-clusters \
132124
--manifests-dir $S/benchmarking/automation/manifests \
133125
--junit-output $D/junit.xml 2>&1 | tee $D/orchestrator.log

‎benchmarking/automation/orchestrator.py‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818
lives in a testtypes/ module (see that package's docstring for the hook
1919
contract). The flow, with the type hooks marked:
2020
21-
1. Clone the substrate branch once; wait for the DIND sidecar.
22-
2. Validate every tests.yaml entry before any cluster work
23-
[hook: validate].
21+
1. Pick the tests.yaml entries named by --test-name (default: all) and
22+
validate them before any other work [hook: validate].
23+
2. Clone the substrate branch once; wait for the DIND sidecar.
2424
3. Per test entry:
2525
a. Source the target cluster's env; gcloud/docker/kubectl setup and
2626
the runner image build [hook: build_image] are cached per
@@ -91,6 +91,13 @@ def parse_args() -> argparse.Namespace:
9191
default="/etc/orchestrator/tests.yaml",
9292
help="Path to the tests YAML file (mounted from a ConfigMap)",
9393
)
94+
p.add_argument(
95+
"--test-name",
96+
action="append",
97+
default=[],
98+
metavar="NAME",
99+
help="Run only the named test entry; repeatable. Default: all entries",
100+
)
94101
p.add_argument(
95102
"--target-cluster-dir",
96103
default=TARGET_CLUSTER_DIR,
@@ -108,6 +115,19 @@ def parse_args() -> argparse.Namespace:
108115
return p.parse_args()
109116

110117

118+
def select_tests(tests: list[dict], names: list[str]) -> list[dict]:
119+
"""Return the entries of tests named in names, in file order, or all of
120+
them when names is empty. Raise ValueError on a name not in tests."""
121+
if not names:
122+
return tests
123+
unknown = sorted(set(names) - {t.get("name") for t in tests})
124+
if unknown:
125+
raise ValueError(
126+
f"--test-name values not in the tests file: {', '.join(unknown)}"
127+
)
128+
return [t for t in tests if t.get("name") in names]
129+
130+
111131
def source_env(path: str) -> None:
112132
result = subprocess.run(
113133
["bash", "-c", f'set -a; source "{path}"; env'],
@@ -555,6 +575,15 @@ def write_junit_xml(
555575

556576
def main() -> None:
557577
args = parse_args()
578+
tests = yaml.safe_load(Path(args.tests).read_text())["tests"]
579+
try:
580+
tests = select_tests(tests, args.test_name)
581+
print(f"Running {len(tests)} test(s)", flush=True)
582+
validate_and_normalize_tests(tests)
583+
# Checks BENCHMARK_METADATA only; cluster vars may not be set yet.
584+
run_metadata()
585+
except ValueError as e:
586+
sys.exit(str(e))
558587

559588
# Config-independent setup: DIND + clone the substrate branch once.
560589
wait_for_docker()
@@ -574,15 +603,6 @@ def main() -> None:
574603
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
575604
print(f"Building commit {commit}", flush=True)
576605

577-
tests = yaml.safe_load(Path(args.tests).read_text())["tests"]
578-
print(f"Running {len(tests)} test(s)", flush=True)
579-
try:
580-
validate_and_normalize_tests(tests)
581-
# Checks BENCHMARK_METADATA only; cluster vars may not be set yet.
582-
run_metadata()
583-
except ValueError as e:
584-
sys.exit(str(e))
585-
586606
# Per-target-cluster caches: re-running setup for the same target
587607
# cluster is wasted work, so we track what was last set up and only
588608
# redo it when the target cluster name changes (substrate images get

‎benchmarking/automation/test_orchestrator.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,21 @@
2525
from testtypes import locust
2626

2727

28+
class SelectTestsTest(unittest.TestCase):
29+
TESTS = [{"name": "a"}, {"name": "b"}, {"name": "c"}]
30+
31+
def test_no_names_runs_all(self):
32+
self.assertEqual(orchestrator.select_tests(self.TESTS, []), self.TESTS)
33+
34+
def test_keeps_file_order(self):
35+
got = orchestrator.select_tests(self.TESTS, ["c", "a"])
36+
self.assertEqual([t["name"] for t in got], ["a", "c"])
37+
38+
def test_unknown_name(self):
39+
with self.assertRaisesRegex(ValueError, "x, y"):
40+
orchestrator.select_tests(self.TESTS, ["a", "y", "x"])
41+
42+
2843
class DeployWorkloadsFakeAteletTest(unittest.TestCase):
2944
@mock.patch("orchestrator.run")
3045
def test_fake_atelet_tuning_goes_to_deploy_sh(self, run):

0 commit comments

Comments
 (0)