Skip to content

Commit d24a823

Browse files
authored
ci: mask the engine version in the replay-off golden pins (#634)
Fixes the red `test_replay_off_golden` on main after the v1.8.0 release commit. The replay-off golden pins six byte digests of sink and report dumps. Those dumps carry the engine version (the `engine` provenance `major`/`minor`/`patch` and the sink header's `"engine"` string), so the release bump changed the digests without changing any trade or report data: the byte counts stayed identical. - `scripts/check_replay_off_golden.py`: masks ONLY those version fields, alongside the build-identity fields it already masked. Each field must occur exactly once; no byte that carries trade or report data is masked. - `scripts/test_check_replay_off_golden.py` (new): a 1.7.0-shaped and a 1.8.0-shaped input mask to equal bytes, and a changed report byte still fails the pin. - `docker/replay_off_golden.json`: the six digests re-recorded from a measured run with the new mask, so the next version bump keeps them valid. - `scripts/test_run_unittest_shards.py`: registers the new test. Verified: `test_replay_off_golden` and `test_replay_rq_fixture` pass on this head in a cloud Release build. 🤖 Generated with [Claude Code](https://claude.com/claude-code)
1 parent 9836309 commit d24a823

4 files changed

Lines changed: 236 additions & 21 deletions

File tree

‎docker/replay_off_golden.json‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
{
2-
"source": "pineforge-engine main 95f4fd1f408bf0ded941ecc4144b278a026a677f, default (no replay switch), unjailed, wall clock fixed at 1.0, run id replay-off-golden; the same digests were measured on main 165449786bbf68e8b474f17abe924c938d351cd6; default_json digests are taken after the build identity (the two engine strings inside the fingerprint token and plain provenance, and the token and digest values derived from them) is masked as scripts/check_replay_off_golden.py describes",
2+
"source": "pineforge-engine main 95f4fd1f408bf0ded941ecc4144b278a026a677f, default (no replay switch), unjailed, wall clock fixed at 1.0, run id replay-off-golden, first recorded when VERSION was 1.7.0; re-recorded on tag v1.8.0 (9836309e3) with the build-identity mask extended to the engine major, minor and patch numbers and the sink header engine version, so the digests no longer depend on VERSION: the six byte counts grew by exactly the bytes the extended mask adds (102 in each default JSON, 11 in each sink stream). default_json digests are taken after the build identity (the engine version string, commit, major, minor and patch inside the fingerprint token and plain provenance, the sink header engine version, and the token and digest values derived from them) is masked as scripts/check_replay_off_golden.py describes",
33
"scenarios": {
44
"fifo": {
5-
"default_json": {"sha256": "bce6dd88125b289d6c5a6210336ff7a82b1b56b731eb8a67d3f12d18940219e9", "bytes": 9186},
6-
"primary_sink": {"sha256": "dfa2679110fbbf9f24f43c902e2e3a2fc33a1d804a202b8554632206d790686e", "bytes": 1554}
5+
"default_json": {"sha256": "d8c77d7c35e7e962301d3da4f26418263305356efe179bd77ed1ce92704087a5", "bytes": 9288},
6+
"primary_sink": {"sha256": "0d9248209e98566e11cdfa2a01d8546f6683ca0814aa0d10a417b256f46fd6f3", "bytes": 1565}
77
},
88
"any": {
9-
"default_json": {"sha256": "dfb18cecb8c2d6d4d4fce14796e4ed822f99533b8c218a9b222010dcba6dc423", "bytes": 8837},
10-
"primary_sink": {"sha256": "64852f4bb3b132efdd085a9a9c092de7b89224d7bf69bbefe4b779a851e81b14", "bytes": 1440}
9+
"default_json": {"sha256": "583147793c4ce4df4d3ba908f0651612145d5dbad0d9152020ca2d31f03ffcc0", "bytes": 8939},
10+
"primary_sink": {"sha256": "0037d808ad1b93612089a4be892b71f2fd636569803c33c06e033b34ca125c9a", "bytes": 1451}
1111
},
1212
"ordinary": {
13-
"default_json": {"sha256": "8d86996e004714f1ae9cc079be28dc9a2f2657c80b8b47e7c08d9a51b5ed7b35", "bytes": 8848},
14-
"primary_sink": {"sha256": "7c7f414f108b930471026d66174c219304454ffb19232f5d77ea962a7c1455ce", "bytes": 1453}
13+
"default_json": {"sha256": "42c766c21c33f6a0bf3427a368fc55bfe213bd7a96fbe47f60fcf439010ef50a", "bytes": 8950},
14+
"primary_sink": {"sha256": "848828b8b336b006a5b7f0e6064bea7421e525a2fb795f52114e3894a3969696", "bytes": 1464}
1515
}
1616
}
1717
}

‎scripts/check_replay_off_golden.py‎

Lines changed: 57 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -60,15 +60,38 @@ def worker(runner_path, library, shape, primary):
6060
return runner.main(argv)
6161

6262

63+
def _mask_engine_version_numbers(data, engine, location):
64+
# Match the scalar engine object, allowing braces inside JSON strings.
65+
# Same-named members in trades or runtime are not build identity.
66+
blocks = list(re.finditer(rb'"engine":\{(?:[^{}"]|"(?:[^"\\]|\\.)*")*\}', data))
67+
if len(blocks) != 1:
68+
raise RuntimeError(f"engine provenance not found exactly once in the {location}")
69+
block = blocks[0].group()
70+
for key in ("major", "minor", "patch"):
71+
prefix = f'"{key}":'.encode("ascii")
72+
if len(re.findall(rb'"%s"\s*:' % key.encode("ascii"), block)) != 1:
73+
raise RuntimeError(f"engine {key} not found exactly once in the {location}")
74+
needle = prefix + json.dumps(engine[key]).encode("ascii")
75+
# A delimiter keeps major:1 from matching the prefix of major:10.
76+
block, count = re.subn(re.escape(needle) + rb"(?=[,}])",
77+
lambda _: prefix + b'"<build-identity>"', block)
78+
if count != 1:
79+
raise RuntimeError(f"engine {key} not found exactly once in the {location}")
80+
match = blocks[0]
81+
return data[:match.start()] + block + data[match.end():]
82+
83+
6384
def mask_build_identity(text, document):
64-
"""The report names the build that produced it only inside fingerprint.token
65-
(base64 of the canonical provenance JSON, whose engine block holds
66-
version_string, a git describe, and commit_sha, the short commit) and in
67-
fingerprint.digest (sha256 of those bytes). Two builds of different commits
68-
can never share those two values. They are replaced by a fixed token in the
69-
raw bytes, and the decoded provenance with only the two engine strings
70-
blanked is appended in canonical form, so everything else the fingerprint
71-
carries is still compared. Every other byte is compared as produced."""
85+
"""Mask only engine build identity and its derived fingerprint values.
86+
87+
The engine's version_string, commit_sha and major/minor/patch numbers
88+
appear in plain fingerprint.provenance and in the base64 token. The token
89+
and its sha256 digest are replaced in the raw report. Decoded provenance
90+
with those five members masked, and a hash of the token's own bytes with
91+
the same masking, are appended so every other value and byte spelling
92+
remains compared. The sink header's version is handled separately by
93+
mask_sink_build_identity. Every other byte is compared as produced.
94+
"""
7295
fingerprint = document.get("fingerprint")
7396
if not isinstance(fingerprint, dict):
7497
raise RuntimeError("default report has no fingerprint")
@@ -81,37 +104,57 @@ def mask_build_identity(text, document):
81104
provenance = json.loads(token_bytes)
82105
engine = provenance.get("engine")
83106
if (not isinstance(engine, dict) or "version_string" not in engine
84-
or "commit_sha" not in engine):
107+
or "commit_sha" not in engine
108+
or any(type(engine.get(key)) is not int for key in ("major", "minor", "patch"))):
85109
raise RuntimeError("fingerprint provenance has no engine identity")
86110
masked = text
87-
# The report also carries the same provenance as a plain object next to the
88-
# token: its engine block names the build twice more, as plain members.
111+
# The report also carries plain provenance next to the token. Replace
112+
# its engine identity without reserializing any surrounding byte.
89113
for key in ("version_string", "commit_sha"):
90114
needle = f'"{key}":{json.dumps(engine[key])}'.encode("utf-8")
91115
if masked.count(needle) != 1:
92116
raise RuntimeError(f"engine {key} not found exactly once in the raw report")
93117
masked = masked.replace(needle, f'"{key}":"<build-identity>"'.encode("utf-8"))
118+
masked = _mask_engine_version_numbers(masked, engine, "raw report")
94119
engine["version_string"] = engine["commit_sha"] = "<build-identity>"
95120
for value in (token, digest):
96121
needle = json.dumps(value).encode("utf-8")
97122
if masked.count(needle) != 1:
98123
raise RuntimeError("fingerprint value not found exactly once in the raw report")
99124
masked = masked.replace(needle, b'"<build-identity>"')
100-
# The token's own bytes (spelling, key order, separators) with only the two
101-
# engine strings replaced, and the digest scheme, so neither is hidden.
125+
# The token's own bytes (spelling, key order, separators) with only engine
126+
# build identity replaced, and the digest scheme, so neither is hidden.
102127
spelled = token_bytes
103128
for key in ("version_string", "commit_sha"):
104129
spelled, count = re.subn(rb'("%s":)"[^"]*"' % key.encode("ascii"),
105130
rb'\1"<build-identity>"', spelled)
106131
if count != 1:
107132
raise RuntimeError(f"engine {key} not found exactly once in the token bytes")
133+
spelled = _mask_engine_version_numbers(spelled, engine, "token bytes")
134+
for key in ("major", "minor", "patch"):
135+
engine[key] = "<build-identity>"
108136
return (masked + b"\nprovenance-without-build-identity "
109137
+ json.dumps(provenance, sort_keys=True, separators=(",", ":")).encode("utf-8")
110138
+ b"\ntoken-bytes-without-build-identity-sha256 "
111139
+ hashlib.sha256(spelled).hexdigest().encode("ascii")
112140
+ b"\ndigest-scheme sha256: length " + str(len(digest)).encode("ascii") + b"\n")
113141

114142

143+
def mask_sink_build_identity(data):
144+
"""Replace exactly one engine version in the header, retaining every body byte."""
145+
header, newline, body = data.partition(b"\n")
146+
document = json.loads(header)
147+
if (not newline or not isinstance(document, dict)
148+
or document.get("schema") != "pf-report-sink"
149+
or not isinstance(document.get("engine"), str)
150+
or len(re.findall(rb'"engine"\s*:', header)) != 1):
151+
raise RuntimeError("sink header engine not found exactly once")
152+
needle = b'"engine":' + json.dumps(document["engine"], ensure_ascii=False).encode("utf-8")
153+
if header.count(needle) != 1:
154+
raise RuntimeError("sink header engine not found exactly once")
155+
return header.replace(needle, b'"engine":"<build-identity>"') + newline + body
156+
157+
115158
def measure(runner, library, shape, directory):
116159
directory.mkdir(parents=True, exist_ok=True)
117160
(directory / "chart.csv").write_text(chart_text(shape), encoding="utf-8")
@@ -144,7 +187,7 @@ def measure(runner, library, shape, directory):
144187
data = primary.read_bytes()
145188
assert data.endswith(b"\n") and b'"pf-report-sink"' in data
146189
assert b'"pf-report-replay"' not in data
147-
results["primary_sink"] = data
190+
results["primary_sink"] = mask_sink_build_identity(data)
148191
return results
149192

150193

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
#!/usr/bin/env python3
2+
"""Real-byte regression tests for the default-off build identity mask."""
3+
from __future__ import annotations
4+
5+
import base64
6+
import hashlib
7+
import json
8+
from pathlib import Path
9+
import subprocess
10+
import tempfile
11+
import unittest
12+
from unittest import mock
13+
14+
from check_replay_off_golden import mask_build_identity, measure
15+
16+
17+
def compact(value, *, canonical=False):
18+
# These fixtures use ASCII keys and integers, for which this is the
19+
# runner's canonical provenance encoding.
20+
return json.dumps(value, sort_keys=canonical, separators=(",", ":")).encode("utf-8")
21+
22+
23+
def report(version=(1, 7, 0), *, price=128, applied="fifo", provenance_input=None,
24+
missing=None, raw_edit=None, token_edit=None):
25+
engine = {
26+
"version_string": ".".join(map(str, version)),
27+
"major": version[0], "minor": version[1], "patch": version[2],
28+
"commit_sha": "0123456",
29+
}
30+
if missing is not None:
31+
del engine[missing]
32+
provenance = {
33+
"engine": engine,
34+
"applied": {"inputs": {"shape": applied if provenance_input is None else provenance_input},
35+
"overrides": {}},
36+
"runtime": {"major": 1, "minor": 7, "patch": 0},
37+
}
38+
token_bytes = compact(provenance, canonical=True)
39+
if token_edit is not None:
40+
token_bytes = token_edit(token_bytes)
41+
document = {
42+
"trades": [{"entry_price": price, "major": 1, "minor": 7, "patch": 0}],
43+
"applied_inputs": {"shape": applied},
44+
"fingerprint": {
45+
"token": base64.b64encode(token_bytes).decode("ascii"),
46+
"digest": "sha256:" + hashlib.sha256(token_bytes).hexdigest(),
47+
"provenance": provenance,
48+
},
49+
}
50+
text = compact(document) + b"\n"
51+
if raw_edit is not None:
52+
text = raw_edit(text)
53+
return text, json.loads(text)
54+
55+
56+
def primary(version="1.7.0", *, price=128, run_id="replay-off-golden", edit=None):
57+
header = compact({
58+
"schema": "pf-report-sink", "version": 1, "run_id": run_id,
59+
"engine": version, "records": [],
60+
})
61+
if edit is not None:
62+
header = edit(header)
63+
# The identical version-looking body string must remain compared.
64+
return (header + b"\n" + compact(["T", price, "1.7.0"]) + b"\n"
65+
+ compact({"trailer": "pf-report-sink", "version": 1, "counts": {"T": 1}})
66+
+ b"\n")
67+
68+
69+
class ReplayOffGoldenTests(unittest.TestCase):
70+
def masked_primary(self, data):
71+
# Only the external producer is substituted. measure(), the default
72+
# masker, and the sink masker all execute on the real fixture bytes.
73+
default_text, _ = report()
74+
with tempfile.TemporaryDirectory(prefix="replay-off-mask-test-") as scratch:
75+
directory = Path(scratch)
76+
77+
def produce(command, **kwargs):
78+
if "--primary" in command:
79+
(directory / "primary.ndjson").write_bytes(data)
80+
return subprocess.CompletedProcess(command, 0, default_text, b"")
81+
82+
with mock.patch("check_replay_off_golden.subprocess.run", side_effect=produce):
83+
return measure(Path("runner.py"), Path("strategy.so"), "fifo",
84+
directory)["primary_sink"]
85+
86+
def test_default_versions_are_equal_and_non_identity_bytes_stay_visible(self):
87+
baseline = mask_build_identity(*report())
88+
# Mutation control: reverting numeric masking leaves minor:7 versus
89+
# minor:8 in the raw report, decoded provenance and token spelling.
90+
for version in ((1, 8, 0), (2, 7, 0), (1, 7, 1), (12, 34, 56)):
91+
with self.subTest(version=version):
92+
self.assertEqual(baseline, mask_build_identity(*report(version)))
93+
for changes in ({"price": 129}, {"applied": "lifo"}, {"provenance_input": "lifo"}):
94+
with self.subTest(non_identity=changes):
95+
self.assertNotEqual(baseline, mask_build_identity(*report(**changes)))
96+
for before, after in (
97+
(b'"entry_price":128,"major":1', b'"entry_price":128,"major":2'),
98+
(b'"runtime":{"major":1', b'"runtime":{"major":2')):
99+
with self.subTest(non_engine_numbers=before):
100+
edit = lambda data: data.replace(before, after)
101+
self.assertNotEqual(baseline, mask_build_identity(*report(raw_edit=edit)))
102+
input_edit = lambda data: data.replace(b'"shape":"fifo"', b'"shape":"lifo"')
103+
self.assertNotEqual(baseline, mask_build_identity(*report(token_edit=input_edit)))
104+
runtime_edit = lambda data: data.replace(b'"runtime":{"major":1',
105+
b'"runtime":{"major":2')
106+
self.assertNotEqual(baseline, mask_build_identity(*report(token_edit=runtime_edit)))
107+
# Token bytes are pinned even when their decoded value is unchanged.
108+
spaced = lambda data: data.replace(b'"applied":{', b'"applied": {')
109+
self.assertNotEqual(baseline, mask_build_identity(*report(token_edit=spaced)))
110+
111+
def test_default_missing_or_duplicate_engine_numbers_are_refused(self):
112+
for key, value in (("major", 1), ("minor", 7), ("patch", 0)):
113+
member = f'"{key}":{value}'.encode("ascii")
114+
with self.subTest(key=key, location="decoded provenance", defect="missing"):
115+
with self.assertRaisesRegex(RuntimeError, "engine.*identity"):
116+
mask_build_identity(*report(missing=key))
117+
for location in ("raw_edit", "token_edit"):
118+
for defect in ("missing", "duplicate same", "duplicate different", "duplicate spaced"):
119+
# Restrict the mutation to the engine object; the fixture
120+
# deliberately has identically named non-engine numbers.
121+
def edit(data, *, member=member, key=key, defect=defect):
122+
start = data.index(b'"engine":{')
123+
end = data.index(b"}", start)
124+
block = data[start:end]
125+
if defect == "missing":
126+
replacement = b""
127+
elif defect == "duplicate same":
128+
replacement = member + b"," + member + b","
129+
elif defect == "duplicate spaced":
130+
replacement = member + b"," + member.replace(b":", b" :") + b","
131+
else:
132+
replacement = member + b',"' + key.encode("ascii") + b'":99,'
133+
block = block.replace(member + b",", replacement)
134+
return data[:start] + block + data[end:]
135+
136+
with self.subTest(key=key, location=location, defect=defect):
137+
error = ("engine.*identity" if location == "token_edit" and defect == "missing"
138+
else f"engine {key}.*exactly once")
139+
with self.assertRaisesRegex(RuntimeError, error):
140+
mask_build_identity(*report(**{location: edit}))
141+
142+
def test_sink_versions_are_equal_and_non_identity_bytes_stay_visible(self):
143+
original = primary()
144+
baseline = self.masked_primary(original)
145+
for version in ("1.8.0", "2.7.0", "1.7.1", "12.34.56"):
146+
with self.subTest(version=version):
147+
self.assertEqual(baseline, self.masked_primary(primary(version)))
148+
header, newline, body = original.partition(b"\n")
149+
self.assertEqual(baseline, header.replace(b'"engine":"1.7.0"',
150+
b'"engine":"<build-identity>"') + newline + body)
151+
for changes in ({"price": 129}, {"run_id": "different-run"}):
152+
with self.subTest(non_identity=changes):
153+
self.assertNotEqual(baseline, self.masked_primary(primary(**changes)))
154+
changed_body = original.replace(b'["T",128,"1.7.0"]', b'["T",128,"1.8.0"]')
155+
self.assertNotEqual(baseline, self.masked_primary(changed_body))
156+
changed_schema = lambda data: data.replace(b'"version":1', b'"version":2')
157+
self.assertNotEqual(baseline, self.masked_primary(primary(edit=changed_schema)))
158+
159+
def test_sink_missing_or_duplicate_header_engine_is_refused(self):
160+
for replacement in (b"", b'"engine":"1.7.0","engine":"1.8.0",',
161+
b'"engine":"1.7.0","engine" :"1.7.0",'):
162+
with self.subTest(replacement=replacement):
163+
edit = lambda data: data.replace(b'"engine":"1.7.0",', replacement)
164+
with self.assertRaisesRegex(RuntimeError, "sink header engine.*exactly once"):
165+
self.masked_primary(primary(edit=edit))
166+
167+
168+
if __name__ == "__main__":
169+
unittest.main()

‎scripts/test_run_unittest_shards.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222
import ci_preflight # noqa: E402
2323
import ci_verifier_classes as verifier_classes # noqa: E402
2424
import run_unittest_shards as shards_runner # noqa: E402
25+
# This module is an explicit preflight entry point. Import the regression
26+
# TestCase so unittest also runs it here, without a second CI registration.
27+
from test_check_replay_off_golden import ReplayOffGoldenTests # noqa: E402,F401
2528

2629
SCRIPT = ROOT / 'scripts' / 'run_unittest_shards.py'
2730

0 commit comments

Comments
 (0)