Skip to content

Commit 801e4e6

Browse files
luisleo526claude
andcommitted
run_json: keep the fingerprint block identical, rewire the bulk-equity rows to the one-pass seam, sync the report artifact once
Three changes on top of the one-pass report assembly, all in the Python wrapper and its tests. Fingerprint block. The bulk bar-loader constants (`_BAR_HEADER`, `_BAR_FLOAT`, `_BAR_CANONICAL_LINE`, `_BAR_BULK_SLAB_ROWS`) sat between the `# >>> fingerprint helpers` and `# <<< fingerprint helpers` markers, which must stay byte-identical to the same block in `scripts/run_strategy.py`. They now sit below the end marker, defined before their first use; the two blocks match again. Bulk-equity rows. Eighteen test groups in `docker/run_json_bulk_equity_test.py` and `docker/run_json_bulk_equity_fuzz_test.py` (246 failing rows) watched the separate bulk pass that the one-pass assembly removed. Each now observes the same property through the one-pass seam and compares against the frozen copy of the earlier reader in `tests/fixtures/run_json_frozen_6b33c47f_rjone.py`: byte equality on every window edge, tampering after verification, strict refusal type, text, code and priority, and the memory and flatness rows. Four test functions are renamed, none dropped, no assertion loosened. The product behaves like the frozen reader on all of them, so no product behaviour changes here. Report artifact durability. `_BoundedJSONFile._sync_window` wrote, truncated and synchronised every full 64 KiB window. It still writes each full window before reuse and still truncates the padding of a last partial block before any read; the data synchronisation now happens once in `flush`, before the artifact is read back or published, and only when bytes were written since the last one. Short-write, EINTR and errno text are unchanged, and a failed synchronisation is still raised by the same call. New rows in `docker/run_json_fused_test.py` count the synchronisations, fail the flush, and compare the bytes with the frozen reader. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
1 parent cc22df7 commit 801e4e6

5 files changed

Lines changed: 614 additions & 174 deletions

File tree

‎docker/run_json.py‎

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -269,16 +269,6 @@
269269
SOURCE_FEED_CANONICALIZATION = "pf-ohlcv-barc-le-v1"
270270
_SOURCE_FEED_HASH_PREFIX = b"pineforge:ohlcv:barc-le:v1\0"
271271
_SOURCE_FEED_RECORD = struct.Struct("<5dq")
272-
_BAR_HEADER = b"timestamp,open,high,low,close,volume\n"
273-
_BAR_FLOAT = (
274-
rb"[+-]?(?:(?:[0-9]+\.[0-9]*|\.[0-9]+)"
275-
rb"(?:[eE][+-]?[0-9]+)?|[0-9]+[eE][+-]?[0-9]+)"
276-
)
277-
_BAR_CANONICAL_LINE = re.compile(
278-
rb"([0-9]{13}),(" + _BAR_FLOAT + rb"),(" + _BAR_FLOAT
279-
+ rb"),(" + _BAR_FLOAT + rb"),(" + _BAR_FLOAT + rb"),("
280-
+ _BAR_FLOAT + rb")\n")
281-
_BAR_BULK_SLAB_ROWS = 4096
282272

283273

284274
def _new_source_feed_hasher():
@@ -701,6 +691,18 @@ def build_fingerprint(provenance: dict) -> dict:
701691
# <<< fingerprint helpers
702692

703693

694+
_BAR_HEADER = b"timestamp,open,high,low,close,volume\n"
695+
_BAR_FLOAT = (
696+
rb"[+-]?(?:(?:[0-9]+\.[0-9]*|\.[0-9]+)"
697+
rb"(?:[eE][+-]?[0-9]+)?|[0-9]+[eE][+-]?[0-9]+)"
698+
)
699+
_BAR_CANONICAL_LINE = re.compile(
700+
rb"([0-9]{13}),(" + _BAR_FLOAT + rb"),(" + _BAR_FLOAT
701+
+ rb"),(" + _BAR_FLOAT + rb"),(" + _BAR_FLOAT + rb"),("
702+
+ _BAR_FLOAT + rb")\n")
703+
_BAR_BULK_SLAB_ROWS = 4096
704+
705+
704706
_RELEASE_OVERRIDE_TYPES = {
705707
"initial_capital": "float",
706708
"commission_value": "float",
@@ -6959,8 +6961,9 @@ class _BoundedJSONFile:
69596961
Disk files use Linux O_DIRECT or Darwin F_NOCACHE. Legacy libraries with
69606962
no engine sink can also use tmpfs/ramfs for the unjailed JSON artifact.
69616963
Engine sink and scratch directories are selected by _disk_report_directory.
6962-
One aligned 64 KiB anonymous window is the I/O working set. Every write
6963-
window is synchronized before reuse. Jailed and fd routes use UTF-8 memory buffers.
6964+
One aligned 64 KiB anonymous window is the I/O working set. Full windows
6965+
are written before reuse; flush synchronizes pending writes before readback.
6966+
Jailed and fd routes use UTF-8 memory buffers.
69646967
"""
69656968
window = 64 * 1024
69666969

@@ -6971,6 +6974,7 @@ def __init__(self, file):
69716974
self.used = self.base = self.size = self.position = 0
69726975
self.read_base, self.read_size = -1, 0
69736976
self.reading = False
6977+
self._dirty = False
69746978
self.binary = False
69756979
self.libc = ctypes.CDLL(None, use_errno=True)
69766980
fd = file.fileno()
@@ -7020,11 +7024,8 @@ def _sync_window(self):
70207024
self._io_error()
70217025
if written != aligned:
70227026
raise OSError("short report artifact write")
7023-
os.ftruncate(self.file.fileno(), self.size)
7024-
if hasattr(os, "fdatasync"):
7025-
os.fdatasync(self.file.fileno())
7026-
else:
7027-
os.fsync(self.file.fileno())
7027+
if aligned != self.used:
7028+
os.ftruncate(self.file.fileno(), self.size)
70287029
if self.used == self.window:
70297030
self.base += self.window
70307031
self.used = 0
@@ -7046,15 +7047,23 @@ def write_ascii(self, data):
70467047
self.buffer[self.used:self.used + count] = data[position:position + count]
70477048
self.used += count
70487049
self.size += count
7050+
self._dirty = True
70497051
position += count
70507052
if self.used == self.window:
70517053
self._sync_window()
70527054
return len(data)
70537055

70547056
def flush(self):
70557057
self.file.flush() # retain checked flush/close failure behavior
7056-
if not self.reading:
7058+
if not self.reading and self._dirty:
70577059
self._sync_window()
7060+
# A full last window has no buffered bytes, but still needs a
7061+
# barrier. A following rewind must not sync unchanged bytes again.
7062+
if hasattr(os, "fdatasync"):
7063+
os.fdatasync(self.file.fileno())
7064+
else:
7065+
os.fsync(self.file.fileno())
7066+
self._dirty = False
70587067

70597068
def seek(self, position):
70607069
if position != 0:

‎docker/run_json_bulk_equity_fuzz_test.py‎

Lines changed: 68 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,14 @@
1-
"""Deterministic differential fuzz of the bulk equity path of docker/run_json.py.
2-
3-
The bulk path accepts a strict subset of canonical E lines at C speed and hands
4-
every other byte to the unchanged strict reader. Its safety claim is: it never
5-
accepts what the strict reader rejects, never writes other bytes, and a refusal
6-
keeps its type, text, typed code and priority. run_json_bulk_equity_test.py pins
7-
35 named corruptions; this file attacks the same claim with generated inputs.
8-
9-
Every generator is a fixed-seed random.Random over integers (no clock, no
10-
entropy, no hash randomisation), so the id of a failing row names a case that
11-
reproduces on any host. A failure message prints family, seed, variant, window,
12-
point count and the exact mutation list. The strict side is always the existing
13-
report entry (run_json_bulk_equity_test._run: verify, build, stream) with
14-
run_json._BULK_E_ENABLED switched off; the bulk side is the same call on the
15-
same file with it switched on and a small _BULK_E_WINDOW, so line cuts land at
16-
every offset. Two independent anchors keep both sides from being wrong alike:
17-
the generator's own rows (expected equity_curve bytes and endpoints) and, for
18-
the single-mutation rows, the exact pf_sink refusal text each mutation must
19-
produce.
20-
21-
A second family builds one valid sink of about 76 KiB (read-only, shared by its
22-
rows) and puts a single fault on, beside and far from the real 64 KiB direct-I/O
23-
page boundary of the sink reader, and at the edges between the 128-line batches in
24-
which the bulk path takes a run of E lines (the last line of a batch, the first two
25-
of the next, the first of the third). Every line index is read off the file's bytes;
26-
each fault must be refused by the strict reader with its named check at exactly
27-
that line (or be accepted, for integer -0), at the production window and at 97.
28-
29-
The controls at the end break the bulk acceptance on purpose and require the
30-
same comparison to fail (the repr proof, the integer -0 grammar, the time range
31-
test, the reader's E count and its line number), and require the engagement
32-
check (json.loads count) to fail for a bulk path that never accepts anything.
33-
34-
No row is skipped or disabled. Run on the lab's disk-backed workspace like the
35-
other run_json rows; docker/run_json_*_test.py is found by preflight.
1+
"""Deterministic differential fuzz of fused equity assembly against frozen code.
2+
3+
Fixed seeds retain the generated numbers, corruptions, window sizes and page
4+
geometry. The frozen strict reader supplies report bytes and complete refusal
5+
envelopes; generated points independently pin the curve and its endpoints.
6+
Delegating probes count accepted proofs and bytes encoded into real artifacts.
7+
8+
Post-verification tampering exercises both the surviving raw scan's completed
9+
prefix and the fused report's immutable verified artifacts. Page and batch edge
10+
faults retain their exact refusal line and priority. Deliberate mutants break
11+
repr identity, integer spelling, timestamp bounds, bookkeeping and engagement.
3612
"""
3713
import hashlib
3814
import inspect
@@ -53,9 +29,10 @@
5329

5430
import pf_sink
5531
import run_json
32+
import run_json_fused_test_support as support
5633
import test_run_json_sink as ordinary
5734
import test_run_json_stream_fastpath as oracle
58-
from run_json_bulk_equity_test import I53, _run
35+
from run_json_bulk_equity_test import I53, _assert_scan, _run, _scan
5936
from run_json_trade_scan_test import _event, _point, _sink, _trade, _trailer_start
6037

6138

@@ -224,16 +201,16 @@ def _first_difference(left, right):
224201
def _assert_same(strict, actual, context):
225202
"""The one comparison every family and every control uses.
226203
227-
Both ran the same file through the same entry; the only difference is the
228-
bulk switch. Same refusal (type, text, code, code_args), same bytes written
229-
before it, same endpoints."""
204+
Frozen strict assembly and fused assembly read the same file. Refusal
205+
envelopes, published bytes and verified endpoints must agree."""
230206
assert actual[1] == strict[1], (
231207
f"{context}: the refusal differs\n strict: {strict[1]!r}\n bulk: {actual[1]!r}")
232208
assert actual[0] == strict[0], (
233209
f"{context}: the report bytes differ at offset "
234210
f"{_first_difference(actual[0], strict[0])} ({len(actual[0])} vs {len(strict[0])} bytes)")
235211
assert actual[3] == strict[3], (
236212
f"{context}: the endpoints differ\n strict: {strict[3]!r}\n bulk: {actual[3]!r}")
213+
support.assert_same(actual[4], strict[4])
237214

238215

239216
def _assert_oracle(spec, outcome, context):
@@ -245,17 +222,17 @@ def _assert_oracle(spec, outcome, context):
245222

246223

247224
def _assert_engaged(spec, strict, actual, context):
248-
"""json.loads stays O(1) in the E count: the bulk path really accepted its lines.
225+
"""Real fused proofs and encoded artifact rows cover every canonical point.
249226
250227
Without this a bulk path that always falls back would make every
251228
comparison trivially equal."""
252229
points, bulk, base = len(spec.points), actual[2], strict[2]
253230
assert bulk["verify"] <= 2 + spec.others, (
254231
f"{context}: verify decoded {bulk['verify']} documents for {points} points "
255232
f"(header, trailer and {spec.others} other records allow {2 + spec.others})")
256-
assert bulk["copy"] <= 2 + spec.trades, (
257-
f"{context}: the copy decoded {bulk['copy']} documents for {points} points "
258-
f"(two headers and {spec.trades} trades allow {2 + spec.trades})")
233+
assert bulk["copy"] == 0, f"{context}: artifact readback decoded JSON"
234+
assert bulk["bulk_e"] == bulk["encoded_e"] == points, (
235+
f"{context}: proved {bulk['bulk_e']} and encoded {bulk['encoded_e']} of {points} points")
259236
assert bulk["decode_e"] == bulk["dump_e"] == 0, f"{context}: per-point decode or dump ran"
260237
assert base["verify"] >= points and base["copy"] >= points, (
261238
f"{context}: the strict path no longer decodes per point, so the counter is blind")
@@ -821,7 +798,7 @@ def _context(label, case):
821798

822799

823800
def _execute(tmp_path, monkeypatch, case, patches=()):
824-
"""Strict then bulk, on the very same mutated file, through the existing report entry."""
801+
"""Frozen strict assembly, then fused assembly, on the same mutated file."""
825802
with _sink(tmp_path, case.spec.rows, verify=False) as (sink, library, report):
826803
Path(sink.path).write_bytes(case.mutated.raw)
827804
if case.mutated.receipt is not None:
@@ -843,17 +820,25 @@ def test_generated_valid_sinks_are_byte_identical_and_really_bulk(tmp_path, monk
843820
spec = _spec(rng, count=rng.randint(513, 1100) if seed % 8 == 0 else None,
844821
large_event=seed % 8 == 4)
845822
windows = (rng.randint(97, 160), rng.randint(161, 700), rng.randint(701, 5000), 64 * 1024)
846-
with _sink(tmp_path, spec.rows, verify=False) as (sink, library, report):
823+
with _sink(tmp_path, spec.rows) as (sink, library, report):
824+
scan_reference = oracle._Destination()
825+
strict_scan = _scan(sink, monkeypatch, False, destination=scan_reference)
826+
assert strict_scan[1] is None
847827
reference = oracle._Destination()
848828
strict = _run(sink, library, report, monkeypatch, False, destination=reference)
849829
_assert_oracle(spec, strict, f"valid seed={seed} strict")
850830
for window in windows:
851831
context = (f"valid seed={seed} window={window} points={len(spec.points)} "
852832
f"others={spec.others}")
853833
destination = oracle._Destination()
834+
scan_destination = oracle._Destination()
854835
with monkeypatch.context() as patch:
855836
patch.setattr(run_json, "_BULK_E_WINDOW", window, raising=True)
856837
actual = _run(sink, library, report, patch, True, destination=destination)
838+
actual_scan = _scan(sink, patch, True, destination=scan_destination)
839+
_assert_scan(actual_scan, strict_scan)
840+
assert scan_destination.calls == scan_reference.calls, (
841+
f"{context}: standalone array write boundaries differ")
857842
_assert_same(strict, actual, context)
858843
assert destination.calls == reference.calls, f"{context}: write boundaries differ"
859844
_assert_oracle(spec, actual, context)
@@ -997,32 +982,50 @@ def _tamper(rng, raw):
997982

998983

999984
@pytest.mark.parametrize("seed", SEEDS, ids=SEED_IDS)
1000-
def test_tampering_after_verification_ends_like_the_strict_path(tmp_path, monkeypatch, seed):
1001-
"""The copy pass sees different bytes than verify did, with size and inode kept."""
985+
def test_tampered_source_scan_and_fused_artifacts_match_frozen(tmp_path, monkeypatch, seed):
986+
"""Keep scan refusals, reject corrupted reads, and publish only proved bytes."""
1002987
rng = _rng("tamper", seed)
1003988
spec = _spec(rng)
1004989
with _sink(tmp_path, spec.rows) as (sink, library, report):
1005990
path = Path(sink.path)
1006991
raw = path.read_bytes()
1007992
cut = _trailer_start(raw)
993+
original = _run(sink, library, report, monkeypatch, False)
994+
_assert_oracle(spec, original, f"tamper seed={seed} original")
1008995
for variant in range(TAMPER_VARIANTS):
1009996
tampered, offset, note = _tamper(_rng("tamper", seed, variant + 1), raw)
1010997
window = _window(_rng("tamper", seed, 100 + variant))
1011998
context = (f"tamper seed={seed} variant={variant} window={window} "
1012999
f"points={len(spec.points)}: {note}")
10131000
assert len(tampered) == len(raw) and tampered != raw, context
10141001
path.write_bytes(tampered)
1015-
strict = _run(sink, library, report, monkeypatch, False,
1016-
copy_only=True, equity_only=True)
1002+
strict = _scan(sink, monkeypatch, False)
10171003
with monkeypatch.context() as patch:
10181004
patch.setattr(run_json, "_BULK_E_WINDOW", window, raising=True)
1019-
actual = _run(sink, library, report, patch, True,
1020-
copy_only=True, equity_only=True)
1021-
_assert_same(strict, actual, context)
1005+
actual = _scan(sink, patch, True)
1006+
_assert_scan(actual, strict)
10221007
if offset < cut:
1023-
# The digest covers every body byte, so the strict pass must fail,
1024-
# at a format error or at the end-of-scan comparison.
10251008
assert strict[1] is not None, f"{context}: the strict path accepted a tampered body"
1009+
strict = _run(sink, library, report, monkeypatch, False)
1010+
with monkeypatch.context() as patch:
1011+
patch.setattr(run_json, "_BULK_E_WINDOW", window, raising=True)
1012+
actual = _run(sink, library, report, patch, True)
1013+
_assert_same(strict, actual, context + " corrupted read")
1014+
if offset < cut:
1015+
assert strict[1] is not None, f"{context}: corrupted body was verified"
1016+
path.write_bytes(raw)
1017+
1018+
def rewrite_after_read(target, native):
1019+
path.write_bytes(tampered)
1020+
1021+
audit = support.SinkIO(sink.path)
1022+
with monkeypatch.context() as patch:
1023+
patch.setattr(run_json, "_BULK_E_WINDOW", window, raising=True)
1024+
with audit.install(patch):
1025+
stable = _run(
1026+
sink, library, report, patch, True, after_verify=rewrite_after_read)
1027+
_assert_same(original, stable, context + " verified artifacts")
1028+
audit.assert_one_pass(raw)
10261029
# (the sink and its temporary file are closed by _sink)
10271030

10281031

@@ -1381,16 +1384,24 @@ def spy(self):
13811384
return fill(self)
13821385

13831386
context = f"page valid window={window}"
1384-
with _sink(tmp_path, spec.rows, verify=False) as (sink, library, report):
1387+
with _sink(tmp_path, spec.rows) as (sink, library, report):
13851388
assert Path(sink.path).read_bytes() == paged.raw, context
1389+
scan_reference = oracle._Destination()
1390+
strict_scan = _scan(sink, monkeypatch, False, destination=scan_reference)
1391+
assert strict_scan[1] is None
13861392
reference = oracle._Destination()
13871393
strict = _run(sink, library, report, monkeypatch, False, destination=reference)
13881394
_assert_oracle(spec, strict, context + " strict")
13891395
destination = oracle._Destination()
1396+
scan_destination = oracle._Destination()
13901397
with monkeypatch.context() as patch:
13911398
patch.setattr(run_json, "_BULK_E_WINDOW", window, raising=True)
13921399
patch.setattr(run_json._BulkEquityFile, "_fill", spy, raising=True)
13931400
actual = _run(sink, library, report, patch, True, destination=destination)
1401+
actual_scan = _scan(sink, patch, True, destination=scan_destination)
1402+
_assert_scan(actual_scan, strict_scan)
1403+
assert scan_destination.calls == scan_reference.calls, (
1404+
f"{context}: standalone array write boundaries differ")
13941405
_assert_same(strict, actual, context)
13951406
assert destination.calls == reference.calls, f"{context}: write boundaries differ"
13961407
_assert_oracle(spec, actual, context)
@@ -1528,15 +1539,15 @@ def test_control_c_a_bulk_path_that_never_engages_fails_the_engagement_check(
15281539
with _sink(tmp_path, spec.rows, verify=False) as (sink, library, report):
15291540
strict = _run(sink, library, report, monkeypatch, False)
15301541
if never == "switch-off":
1531-
silent = _run(sink, library, report, monkeypatch, False)
1542+
silent = _run(sink, library, report, monkeypatch, True, bulk=False)
15321543
else:
15331544
with monkeypatch.context() as patch:
15341545
patch.setattr(run_json, "_bulk_equity", lambda raw: None, raising=True)
15351546
silent = _run(sink, library, report, patch, True)
15361547
# Exactly the false success to prevent: identical bytes and refusals ...
15371548
_assert_same(strict, silent, f"control {never}")
15381549
_assert_oracle(spec, silent, f"control {never}")
1539-
# ... which only the json.loads count can tell from a real bulk run.
1550+
# ... which the JSON, proof and artifact probes distinguish from real bulk work.
15401551
with pytest.raises(AssertionError):
15411552
_assert_engaged(spec, strict, silent, f"control {never}")
15421553

0 commit comments

Comments
 (0)