Skip to content

Commit c32c732

Browse files
luisleo526claude
andauthored
test(live): match delivery receipts by request sequence (CR-223) (#478)
* test(live): match delivery receipts by request sequence * docs: record CR-223 receipt assertion handoff * chore: drop lane hand-off files from the product tree Remove DONE and REPORT.md, which 5d8b9fb added at the repository root. They are lane hand-off artifacts, not product files; lab verify record's no-behaviour diff rule refuses them. Their text is kept in the lane folder outside the repository. No source, test or build file changes. Co-Authored-By: Claude Code <[email protected]> --------- Co-authored-by: Claude Code <[email protected]>
1 parent 20267dd commit c32c732

9 files changed

Lines changed: 210 additions & 107 deletions

‎tests/native_live_e2e.py‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
import base64
1818
import struct
1919

20+
from native_live_receipts import receipts_by_sequence
21+
2022
runner, library = sys.argv[1:]
2123
received = []
2224
responses = []
@@ -26,7 +28,8 @@
2628

2729

2830
def ordered_receipts():
29-
return sorted(received, key=lambda event: event['sequence'])
31+
indexed = receipts_by_sequence(received)
32+
return [indexed[sequence] for sequence in sorted(indexed)]
3033

3134

3235
class Receiver(BaseHTTPRequestHandler):
@@ -157,16 +160,21 @@ def invoke(extra, success=True, stdin=None):
157160
options = ['--mode', 'bars', '--feed', str(feed), '--ledger', str(ledger)]
158161
invoke(options+['--max-attempts', '1'])
159162
assert len(received) == 4
160-
first = received[0]
163+
original = receipts_by_sequence(received)
161164
invoke(options)
162165
assert len(received) == 4
163166
with sqlite3.connect(ledger) as db:
164167
deployment = db.execute('SELECT identity FROM metadata WHERE singleton=1').fetchone()[0]
168+
failed = receipts_by_sequence(json.loads(row[0]) for row in
169+
db.execute('SELECT payload FROM events WHERE acknowledged=0'))
170+
assert len(failed) == 1
171+
assert failed == {sequence: original[sequence] for sequence in failed}
165172
retry = subprocess.run([runner, 'redeliver', '--ledger', str(ledger), '--deployment', deployment,
166173
'--target', 'default', '--failed-only'],
167174
capture_output=True, text=True, env=dict(os.environ, PINEFORGE_TEST_HMAC=secret))
168175
assert retry.returncode == 0, retry.stderr
169-
assert received[4] == first
176+
assert len(received) == 5
177+
assert receipts_by_sequence(received[4:]) == failed
170178
assert len({e['event_id'] for e in received}) == 4
171179

172180
# Redirect refusal is permanent and does not stop later actions.
@@ -189,16 +197,22 @@ def invoke(extra, success=True, stdin=None):
189197
with sqlite3.connect(ledger) as db:
190198
assert db.execute('SELECT COUNT(*) FROM inputs').fetchone()[0] == 1
191199
assert db.execute('SELECT COUNT(*) FROM events').fetchone()[0] == 4
192-
first = received[0]
200+
assert len(received) == 4
201+
original = receipts_by_sequence(received)
193202
invoke(options)
194203
assert len(received) == 4 and len({e['event_id'] for e in received}) == 4
195204
with sqlite3.connect(ledger) as db:
196205
deployment = db.execute('SELECT identity FROM metadata WHERE singleton=1').fetchone()[0]
206+
failed = receipts_by_sequence(json.loads(row[0]) for row in
207+
db.execute('SELECT payload FROM events WHERE acknowledged=0'))
208+
assert len(failed) == 1
209+
assert failed == {sequence: original[sequence] for sequence in failed}
197210
retry = subprocess.run([runner, 'redeliver', '--ledger', str(ledger), '--deployment', deployment,
198211
'--target', 'default', '--failed-only'],
199212
capture_output=True, text=True, env=dict(os.environ, PINEFORGE_TEST_HMAC=secret))
200213
assert retry.returncode == 0, retry.stderr
201-
assert received[4] == first
214+
assert len(received) == 5
215+
assert receipts_by_sequence(received[4:]) == failed
202216

203217
# A later invalid event in a batch must not commit any of the message.
204218
received.clear()

‎tests/native_live_equivalence_e2e.py‎

Lines changed: 43 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
88
The observer is compiled once into each corpus strategy library. Qualification
99
compares reconstructed actions to the native accessor from the same stream run,
10-
before comparing any runner output. All comparisons preserve physical order and
11-
binary64 values. State-selected splits and completion filtering are restricted
10+
before comparing any runner output. Engine comparisons preserve physical order
11+
and binary64 values; HTTP receipts are matched by submission sequence.
12+
State-selected splits and completion filtering are restricted
1213
to this contiguous 24x7 ETH tape, not shortened/session-based buckets.
1314
Optional tick checks use explicitly synthetic O-H-L-C prints derived from the
1415
shipped 1m corpus, not historical observed trades. They prove same-print replay
@@ -39,6 +40,8 @@
3940
import threading
4041
import traceback
4142

43+
from native_live_receipts import receipts_by_sequence
44+
4245

4346
PROBES = {
4447
"ta-rsi14-cross-50-01": {"market"},
@@ -411,8 +414,9 @@ def push(current, packet):
411414

412415

413416
def ordered_delivery_effects(attempts, effects, redelivery=None):
414-
sequences = [effect["sequence"] for effect in effects]
415-
if sorted(sequences) != list(range(1, len(effects) + 1)):
417+
indexed_effects = receipts_by_sequence(effects)
418+
sequences = range(1, len(effects) + 1)
419+
if set(indexed_effects) != set(sequences):
416420
raise RuntimeError("effect sequence gap or duplication")
417421
if len({effect["event_id"] for effect in effects}) != len(effects):
418422
raise RuntimeError("duplicate logical effect")
@@ -425,25 +429,22 @@ def ordered_delivery_effects(attempts, effects, redelivery=None):
425429
or attempt["idempotency_header"] != payload.get("delivery_id", event_id)):
426430
raise RuntimeError("idempotency header/payload mismatch")
427431
if event_id not in bodies:
428-
first_attempts.append(payload["sequence"])
432+
first_attempts.append(payload)
429433
bodies[event_id] = attempt["body"]
430434
elif bodies[event_id] != attempt["body"]:
431435
raise RuntimeError("retry body changed")
432-
if first_attempts != list(range(1, len(effects) + 1)):
433-
raise RuntimeError("first-attempt deliveries are reordered or incomplete")
434-
for effect in effects:
435-
if effect != json.loads(bodies[effect["event_id"]]):
436-
raise RuntimeError("effect body differs from its delivery")
436+
indexed_attempts = receipts_by_sequence(first_attempts)
437+
if set(indexed_attempts) != set(indexed_effects):
438+
raise RuntimeError("first-attempt deliveries are incomplete or unexpected")
439+
if indexed_effects != indexed_attempts:
440+
raise RuntimeError("effect body differs from its delivery")
441+
ordered_effects = [indexed_effects[sequence] for sequence in sequences]
437442
if redelivery is None:
438-
if sequences != sorted(sequences):
439-
raise RuntimeError("first-pass effects are reordered")
440-
return effects
443+
return ordered_effects
441444
boundary = redelivery["effects_before"]
442445
if not 0 <= boundary <= len(effects):
443446
raise RuntimeError("invalid explicit-redelivery boundary")
444-
initial = sequences[:boundary]
445-
if initial != sorted(initial):
446-
raise RuntimeError("first-pass effects are reordered")
447+
seen_effects = receipts_by_sequence(effects[:boundary])
447448
selected = set(redelivery["event_ids"])
448449
if not selected <= set(bodies):
449450
raise RuntimeError("explicit redelivery selected an unknown event")
@@ -458,24 +459,27 @@ def ordered_delivery_effects(attempts, effects, redelivery=None):
458459
raise RuntimeError("invalid explicit-redelivery invocation boundary")
459460
invocation_attempts = attempts[invocation["attempts_before"]:following["attempts_before"]]
460461
invocation_effects = effects[invocation["effects_before"]:following["effects_before"]]
461-
redelivery_sequences = [json.loads(attempt["body"])["sequence"]
462-
for attempt in invocation_attempts if attempt["event_id"] in selected]
463-
if redelivery_sequences != sorted(redelivery_sequences):
464-
raise RuntimeError("explicit redelivery attempts are reordered")
465-
redelivery_effects = [effect["sequence"] for effect in invocation_effects
466-
if effect["event_id"] in selected]
467-
if redelivery_effects != sorted(redelivery_effects):
468-
raise RuntimeError("explicit redelivery effects are reordered")
469-
for effect in invocation_effects:
470-
if effect["event_id"] in selected and not any(
471-
attempt["event_id"] == effect["event_id"] and attempt["status"] == 200
472-
for attempt in invocation_attempts):
473-
raise RuntimeError("explicit redelivery effect has no successful attempt")
474-
first_pass = [effect["sequence"] for index, effect in enumerate(effects)
475-
if index < boundary or effect["event_id"] not in selected]
476-
if first_pass != sorted(first_pass):
477-
raise RuntimeError("non-redelivered effects are reordered")
478-
return sorted(effects, key=lambda effect: effect["sequence"])
462+
successful = {}
463+
for attempt in invocation_attempts:
464+
if attempt["status"] == 200:
465+
payload = json.loads(attempt["body"])
466+
successful[payload["sequence"]] = payload
467+
expected_effects = {sequence: payload for sequence, payload in successful.items()
468+
if sequence not in seen_effects}
469+
if receipts_by_sequence(invocation_effects) != expected_effects:
470+
raise RuntimeError("explicit redelivery effects differ from new successful attempts")
471+
seen_effects.update(expected_effects)
472+
if seen_effects != indexed_effects:
473+
raise RuntimeError("explicit redelivery effects are incomplete")
474+
# The final invocation must replay the whole selection, including effects
475+
# already accepted before an acknowledgement was lost and the process restarted.
476+
replayed = {json.loads(attempt["body"])["sequence"] for attempt in invocation_attempts
477+
if attempt["event_id"] in selected}
478+
expected_replay = {sequence for sequence, payload in indexed_attempts.items()
479+
if payload["event_id"] in selected}
480+
if replayed != expected_replay:
481+
raise RuntimeError("explicit redelivery selection is incomplete")
482+
return ordered_effects
479483

480484

481485
def require_replay_after_restart(attempts, event_id, invocation):
@@ -525,8 +529,7 @@ def do_POST(self):
525529
if not rejected:
526530
owner.database.execute("INSERT OR IGNORE INTO effects VALUES(NULL,?,?)", (event_id, body))
527531
owner.database.commit()
528-
count = owner.database.execute("SELECT COUNT(*) FROM effects").fetchone()[0]
529-
wait_ack = not rejected and is_new and count == owner.restart_action
532+
wait_ack = not rejected and is_new and payload["sequence"] == owner.restart_action
530533
if wait_ack:
531534
owner.accepted.set()
532535
owner.release_ack.wait(120)
@@ -550,7 +553,7 @@ def log_message(self, *unused):
550553
write_json(self.routes, {"schema_version": 1, "default_target": "default",
551554
"targets": {"default": {"url": f"http://127.0.0.1:{self.server.server_address[1]}/actions",
552555
"secret_env": "PINEFORGE_EQUIVALENCE_HMAC"}}, "rules": [],
553-
"delivery": {"max_in_flight": 1, "transport_retries": 2,
556+
"delivery": {"max_in_flight": 8, "transport_retries": 2,
554557
"retry_backoff_ms": [10, 20]}})
555558

556559
def finish(self, output):
@@ -664,7 +667,7 @@ def run_case(strategy, batch, bars, rows, timeframe, split, output, runner,
664667
if not expected:
665668
raise RuntimeError("NOT EXERCISED: no live actions")
666669
receiver = MockReceiver(output, fail_first, crash_action)
667-
result["delivery_policy"] = "single-flight, two explicit transport retries"
670+
result["delivery_policy"] = "max_in_flight=8, two explicit transport retries"
668671
command = [str(runner), "run", "--strategy", str(strategy.path), "--warmup", str(output / "warmup.csv"),
669672
"--input-tf", str(timeframe), "--script-tf", str(timeframe), "--session", "24x7", "--timezone", "UTC",
670673
"--chart-timezone", "UTC", "--mode", "ticks" if packets is not None else "bars",
@@ -720,7 +723,8 @@ def run_case(strategy, batch, bars, rows, timeframe, split, output, runner,
720723
if set(rejected) != {row["event_id"] for row in effects} or any(count != 1 for count in rejected.values()):
721724
raise RuntimeError("fail-first coverage missing")
722725
if restart:
723-
require_replay_after_restart(attempts, effects[crash_action - 1]["event_id"], restart_invocation)
726+
crashed_effect = receipts_by_sequence(effects)[crash_action]
727+
require_replay_after_restart(attempts, crashed_effect["event_id"], restart_invocation)
724728
with sqlite3.connect(output / "orders.sqlite3") as ledger:
725729
ledger_hashes = [row[0] for row in ledger.execute("SELECT state_hash FROM inputs ORDER BY input_index")]
726730
pending = ledger.execute("SELECT COUNT(*) FROM events WHERE acknowledged=0").fetchone()[0]

‎tests/native_live_genuine_retry_e2e.py‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
)
1919
from native_live_test_process import isolated_output
2020
from native_live_sanitizer import finalize_sanitizer, prepare_sanitizer, run_with_finalizer
21+
from native_live_receipts import receipts_by_sequence
2122

2223

2324
def synthetic_fixture():
@@ -39,29 +40,34 @@ def synthetic_fixture():
3940

4041

4142
def check_invocation_order(attempts, effects, invocations):
43+
try:
44+
ordered_delivery_effects(attempts, effects)
45+
except RuntimeError as error:
46+
raise AssertionError(f"delivery invariants failed: {error}") from error
4247
limits = {"attempts_before": len(attempts), "effects_before": len(effects)}
4348
assert invocations and all(invocations[0][key] == 0 for key in limits), (
4449
"initial invocation must precede every attempt/effect", invocations)
45-
first_pass = []
46-
for index, (invocation, following) in enumerate(zip(invocations, invocations[1:] + [limits])):
50+
phases = list(zip(invocations, invocations[1:] + [limits]))
51+
for index, (invocation, following) in enumerate(phases):
4752
assert all(0 <= invocation[key] <= following[key] <= limit
4853
for key, limit in limits.items()), (
4954
"invalid invocation boundary", index, invocation, following, limits)
55+
seen_effects = {}
56+
for index, (invocation, following) in enumerate(phases):
5057
phase_attempts = attempts[invocation["attempts_before"]:following["attempts_before"]]
5158
phase_effects = effects[invocation["effects_before"]:following["effects_before"]]
52-
sequences = [effect["sequence"] for effect in phase_effects]
53-
assert sequences == sorted(sequences), (
54-
"invocation effects are reordered", index, invocation["command"], sequences)
55-
successful = {attempt["event_id"] for attempt in phase_attempts if attempt["status"] == 200}
56-
assert all(effect["event_id"] in successful for effect in phase_effects), (
57-
"invocation effect has no successful attempt in its phase", index, sequences, successful)
58-
if invocation["command"][1] == "redeliver":
59-
attempt_sequences = [json.loads(attempt["body"])["sequence"] for attempt in phase_attempts]
60-
assert attempt_sequences == sorted(attempt_sequences), (
61-
"redelivery invocation attempts are reordered", index, attempt_sequences)
62-
else:
63-
first_pass.extend(sequences)
64-
assert first_pass == sorted(first_pass), ("first-pass effects are reordered", first_pass)
59+
successful = {}
60+
for attempt in phase_attempts:
61+
if attempt["status"] == 200:
62+
payload = json.loads(attempt["body"])
63+
successful[payload["sequence"]] = payload
64+
expected_effects = {sequence: payload for sequence, payload in successful.items()
65+
if sequence not in seen_effects}
66+
indexed_effects = receipts_by_sequence(phase_effects)
67+
assert indexed_effects == expected_effects, (
68+
"invocation effects differ from new successful attempts in their phase",
69+
index, invocation["command"], indexed_effects, expected_effects)
70+
seen_effects.update(indexed_effects)
6571

6672

6773
def check_case(result, directory, reference, total_inputs):

‎tests/native_live_receipts.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
"""Sequence indexes for complete live-delivery receipt phases."""
2+
3+
4+
def receipts_by_sequence(receipts, sequence_of=lambda receipt: receipt["sequence"]):
5+
"""Keep each complete receipt and reject duplicates instead of overwriting them."""
6+
indexed = {}
7+
for receipt in receipts:
8+
sequence = sequence_of(receipt)
9+
if sequence in indexed:
10+
raise RuntimeError(f"duplicate receipt sequence: {sequence}")
11+
indexed[sequence] = receipt
12+
return indexed

‎tests/native_live_routing_e2e.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import threading
1717
import time
1818

19+
from native_live_receipts import receipts_by_sequence
20+
1921

2022
runner, library = sys.argv[1:3]
2123
secrets = {'MAIN_HMAC': 'main-loopback-secret', 'ENTRY_HMAC': 'entry-loopback-secret',
@@ -554,7 +556,12 @@ def committed():
554556
process.kill(); process.wait()
555557
main_receiver.clear(); main_receiver.mode = 'ok'
556558
run(crash_ledger, crash_routes)
557-
assert [row['action']['sequence'] for row in main_receiver.rows] == [1, 2, 3, 4]
559+
assert len(main_receiver.rows) == 4
560+
receipt_sequence = lambda body: json.loads(body)['sequence']
561+
recovered = receipts_by_sequence((row['body'] for row in main_receiver.rows), receipt_sequence)
562+
expected = receipts_by_sequence((row[2].encode() for row in before_crash[1]), receipt_sequence)
563+
assert set(recovered) == set(range(1, 5))
564+
assert recovered == expected
558565
assert stored(crash_ledger) == before_crash
559566
run(crash_ledger, crash_routes)
560567
assert len(main_receiver.rows) == 4

0 commit comments

Comments
 (0)