Skip to content

Commit 678ec3a

Browse files
Control bytecode cache state in native benchmarks
1 parent 7d8aacc commit 678ec3a

5 files changed

Lines changed: 38 additions & 11 deletions

File tree

‎benchmarks/native-sdk/README.md‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,17 +29,19 @@ uv venv --python "$BASE_PY" "$LAB/peer"
2929
STABLE_PY="$LAB/stable/bin/python"
3030
NATIVE_PY="$LAB/native/bin/python"
3131
PEER_PY="$LAB/peer/bin/python"
32-
uv pip install --python "$STABLE_PY" 'openai-agents==0.24.0' \
32+
uv pip install --compile-bytecode --python "$STABLE_PY" 'openai-agents==0.24.0' \
3333
'openai==3.29.0' 'pydantic==2.12.3' 'typing-extensions==4.16.0'
34-
uv pip install --python "$NATIVE_PY" "$WHEEL" \
34+
uv pip install --compile-bytecode --python "$NATIVE_PY" "$WHEEL" \
3535
'pydantic==2.12.3' 'typing-extensions==4.16.0' \
3636
'jsonschema==4.26.0' 'referencing==0.37.0'
37-
uv pip install --python "$PEER_PY" \
37+
uv pip install --compile-bytecode --python "$PEER_PY" \
3838
'aiohttp==3.14.3' 'openai==3.41.0' 'pydantic==2.12.3'
3939
```
4040

4141
The wheel hook builds the native extension and matching execution helpers. Install that exact wheel into the fresh environment; do not substitute an editable checkout or an unrelated native binary. The dependency versions above come from existing compatible environment records. They are not complete dependency locks. Keep the wheel hash, source commit, Python build and complete installed package inventories with results. Use `uv pip freeze --python <interpreter>` to record each environment. If registry access is unavailable, supply the exact stable release wheel locally.
4242

43+
Compile bytecode for every environment before measurement, as shown above. Workers and peers run with `-B` to prevent cache writes; they still read existing caches. Unequal cache coverage can change first-call CPU even when package versions match. Keep before/after inventories of installed Python sources and valid, non-optimized bytecode caches, including their interpreter cache tag and source validation. Worker rows record the effective cache settings, but those settings alone do not prove complete or equal cache coverage.
44+
4345
## Verify and run a small correctness check
4446

4547
Before execution, verify the eight source hashes and inventory against `benchmark-source-manifest.json`. Keep exactly eight `.py` files beside `compare.py`; it records every sibling Python file. Run ordinary Python without optimization or instrumentation. The controller and worker reject known inherited Python path/home, allocator, preload, profiling and scheduler overrides, including empty environment values. They also reject active Python trace/profile hooks and profiling `-X` options. Clear the listed controls rather than relying on a shell that happens to omit them. `PYTHONSAFEPATH` and the controller's fixed `PYTHONHASHSEED=0` remain supported. These checks cover known controls, not every possible external profiler or modified interpreter.
@@ -102,7 +104,7 @@ conversation beyond the observation buffer's retention and rejection of a genuin
102104
SDK gap. It reports correctness only:
103105

104106
```sh
105-
uv pip install --python "$NATIVE_PY" 'aiohttp==3.14.3'
107+
uv pip install --compile-bytecode --python "$NATIVE_PY" 'aiohttp==3.14.3'
106108
"$NATIVE_PY" "$BENCH/tests/native_stream_cursor.py" -v
107109
```
108110

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
2-
"compare.py": "69faa65cd95e352d00858eb74de31b182b018fb05e6eed09452eb43d07eabf0d",
2+
"compare.py": "dd853b968679dd5b5e1d833516abf52ea9e139169dfbd74b375a59ef033b470f",
33
"owned_process.py": "9be3bd2a11154dec666a943172fd7645e040ef7279f0f53633d5c3f22d168e4e",
44
"protocol_peer.py": "912165f4e3d2abc8d45a0690e08e7b564fcc0562d1ea1a7725de92ba0f841bee",
55
"v0_adapter.py": "36d8e51ce6743c82bc73385c79241a2dddcd2b259ef7f3207f53d0721fccf6d2",
66
"v1_adapter.py": "63e6a993a82c83ec6bfe3f76cb1fab601e25dd92f73e68c5f77423fe274f784b",
77
"validate_peer.py": "5d0829b7cc68310cbfc861bff8927b2e51051757a7d74e0a1b8faa48ab87b527",
8-
"worker.py": "7ff1b76eb65ff46767318b5b4316503b2297026b2a60f588c3f02f83acda2dcb",
8+
"worker.py": "7fcc4b120ccbb5f3e1c8da49b792a7e323b634454618f4abdc6d8c4b5e132167",
99
"workloads.py": "a86d44a82b16c6fcb4097ce8a4487093ddc676029836020eebff46c6cf013546"
1010
}

‎benchmarks/native-sdk/compare.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,17 +115,22 @@ def validate_sample(
115115
"cache_tag": str,
116116
"pointer_bits": int,
117117
"abiflags": str,
118+
"dont_write_bytecode": bool,
119+
"pycache_prefix": type(None),
118120
}
119121
if (
120122
not isinstance(build, dict)
121-
or any(type(build.get(name)) is not kind for name, kind in build_types.items())
123+
or any(
124+
name not in build or type(build[name]) is not kind for name, kind in build_types.items()
125+
)
122126
or build["implementation"] != "cpython"
123127
or not build["cache_tag"]
128+
or not build["dont_write_bytecode"]
124129
or type(row.get("python")) is not str
125130
or not row["python"]
126131
or build["version"] != row["python"]
127132
):
128-
raise ValueError("Missing or inconsistent CPython build/ABI identity")
133+
raise ValueError("Missing or inconsistent CPython runtime identity")
129134
module = row.get("module_file")
130135
if not isinstance(module, str) or not Path(module).is_absolute():
131136
raise ValueError("Worker module path is missing or not absolute")
@@ -401,6 +406,7 @@ def stop_requested(signum, _frame):
401406
executable = args.v0_python if side == "v0" else args.v1_python
402407
command = [
403408
str(executable.absolute()),
409+
"-B",
404410
str(directory / "worker.py"),
405411
"--engine",
406412
side,

‎benchmarks/native-sdk/tests/test_comparison.py‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ def row(side, sample, case="oneshot_text"):
4646
"cache_tag": "cpython-test",
4747
"pointer_bits": 64,
4848
"abiflags": "",
49+
"dont_write_bytecode": True,
50+
"pycache_prefix": None,
4951
},
5052
"module_file": f"/opt/{side}/site-packages/agents/__init__.py",
5153
"workload": workload.dictionary(),
@@ -207,19 +209,33 @@ def test_per_arm_and_shared_identity_drift_are_rejected(self):
207209
with self.subTest(mutate=mutate, index=index), self.assertRaises(ValueError):
208210
self.summarize(value)
209211

210-
def test_python_build_abi_fields_cannot_be_omitted_or_mistyped(self):
211-
for field in ("cache_tag", "pointer_bits", "abiflags"):
212+
def test_python_runtime_fields_cannot_be_omitted_or_mistyped(self):
213+
for field in (
214+
"cache_tag",
215+
"pointer_bits",
216+
"abiflags",
217+
"dont_write_bytecode",
218+
"pycache_prefix",
219+
):
212220
for malformed in (False, True):
213221
value = records()
214222
for side in ("v0", "v1"):
215223
build = value["oneshot_text"][side][0]["python_build"]
216224
if malformed:
217-
build[field] = None
225+
build[field] = "/unexpected-cache" if field == "pycache_prefix" else None
218226
else:
219227
del build[field]
220228
with self.subTest(field=field, malformed=malformed), self.assertRaises(ValueError):
221229
self.summarize(value)
222230

231+
def test_bytecode_writes_are_rejected_even_when_both_arms_agree(self):
232+
value = records()
233+
for side in ("v0", "v1"):
234+
for item in value["oneshot_text"][side]:
235+
item["python_build"]["dont_write_bytecode"] = False
236+
with self.assertRaises(ValueError):
237+
self.summarize(value)
238+
223239
def test_parity_failure_keeps_successful_collection_complete(self):
224240
previous = signal.getsignal(signal.SIGTERM)
225241
self.addCleanup(signal.signal, signal.SIGTERM, previous)

‎benchmarks/native-sdk/worker.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,7 @@ def stop_requested(signum, _frame):
397397
child = subprocess.Popen(
398398
[
399399
server_python,
400+
"-B",
400401
str(Path(__file__).with_name("protocol_peer.py")),
401402
"--tools-per-round",
402403
str(workload.tools_per_round),
@@ -456,6 +457,8 @@ def stop_requested(signum, _frame):
456457
"cache_tag": sys.implementation.cache_tag,
457458
"pointer_bits": sys.maxsize.bit_length() + 1,
458459
"abiflags": getattr(sys, "abiflags", ""),
460+
"dont_write_bytecode": sys.dont_write_bytecode,
461+
"pycache_prefix": sys.pycache_prefix,
459462
}
460463
result["module_file"] = str(Path(importlib.import_module("agents").__file__).resolve())
461464
except BaseException as error:

0 commit comments

Comments
 (0)