Skip to content

Commit 878af53

Browse files
luisleo526claude
andcommitted
Merge main into the memory-cache branch
Brings in the jailed temporary-file witness test and the strategy runner changes that landed on main. The merge applied without conflicts and changes no file of the cache work. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
2 parents 55cc25a + fc62dd9 commit 878af53

9 files changed

Lines changed: 831 additions & 7 deletions

‎docker/pineforge_sink_finalize.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@
3535
PATH is a dedicated output pathname owned exclusively by this invocation.
3636
File mode runs as a uid permitted to write the output and spool directories,
3737
without RLIMIT_FSIZE=0.
38+
PINEFORGE_RUN_JSON_PROFILE accepts unjailed (default) or jailed. File mode is excluded under jailed before touching
39+
PATH or creating a spool: exit 3, with a text diagnostic naming finalizer_file_output_no_jailed_route on stderr only when --receipt-fd is not 2.
40+
With --receipt-fd 2, the caller sees exit 3 and the ordinary failure receipt only, indistinguishable from a malformed stream.
41+
Stdout mode with a positive --max-bytes remains accepted.
3842
An old entry at PATH is unlinked before receiving input, so a failed run
3943
cannot leave an old success behind. A random mode-0600 spool is created
4044
in DIR (default: PATH's directory), filled, fsynced, closed, and verified
@@ -100,7 +104,7 @@ class engine_fault: the run is refunded and the caller may retry with a larger s
100104
Exit codes:
101105
0 complete and verified; sink delivered/published and receipt written
102106
2 usage error
103-
3 malformed stream: JSON/schema/line rule, run-id mismatch, or E/T/O/K order
107+
3 malformed stream (JSON/schema/line rule, run-id mismatch, or E/T/O/K order) or a host-profile refusal (text diagnostic naming the reason on stderr only when --receipt-fd is not 2; with --receipt-fd 2, the ordinary failure receipt only, indistinguishable from a malformed stream)
104108
4 verification failure: trailer grammar, counts, SHA-256, or extra bytes
105109
5 input/output I/O failure (including receipt/rename)
106110
6 EOF without a trailer, including empty input/final fragments
@@ -409,6 +413,7 @@ def _diagnostic(message: str, receipt_fd: int, *, document: dict | None = None)
409413

410414

411415
def main(argv: list[str] | None = None) -> int:
416+
profile = os.environ.get("PINEFORGE_RUN_JSON_PROFILE", "unjailed")
412417
parser = _parser()
413418
args = parser.parse_args(argv)
414419
if args.in_fd == args.receipt_fd:
@@ -427,11 +432,17 @@ def main(argv: list[str] | None = None) -> int:
427432
status = 0
428433
failure_diagnostic = {}
429434
try:
435+
if profile not in ("unjailed", "jailed"):
436+
raise _Failure(3, "PINEFORGE_RUN_JSON_PROFILE must be unjailed or jailed")
430437
if stdout_mode:
431438
with io.BufferedReader(_StreamInput(args.in_fd, receipt, args.max_bytes),
432439
buffer_size=CHUNK_BYTES) as stream:
433440
_verify(None, args.run_id, receipt, stream=stream, out_fd=1)
434441
else:
442+
if profile == "jailed":
443+
raise _Failure(
444+
3, "jailed profile excludes pineforge-sink-finalize file-output mode: "
445+
"finalizer_file_output_no_jailed_route")
435446
directory = (os.path.abspath(args.spool_dir) if args.spool_dir is not None
436447
else os.path.dirname(out))
437448
_unlink(out)

‎docker/pineforge_sink_finalize_test.py‎

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,58 @@ def test_pipe_round_trip_from_path_writer(tmp_path, stdout_mode):
397397
assert_success(result, receipt, out, spool, payload)
398398

399399

400+
@pytest.mark.parametrize("profile", [None, "unjailed"], ids=["default", "explicit"])
401+
def test_file_mode_runs_normally_outside_jailed_profile(tmp_path, monkeypatch, profile):
402+
# Making the file-mode refusal unconditional must turn this round trip red.
403+
if profile is None:
404+
monkeypatch.delenv("PINEFORGE_RUN_JSON_PROFILE", raising=False)
405+
else:
406+
monkeypatch.setenv("PINEFORGE_RUN_JSON_PROFILE", profile)
407+
payload = fixture_stream(tmp_path)
408+
result, receipt, out, spool = invoke(
409+
tmp_path, payload, extra=("--max-bytes", str(len(payload))))
410+
assert_success(result, receipt, out, spool, payload)
411+
assert tuple(receipt) == RECEIPT_KEYS
412+
413+
414+
@pytest.mark.parametrize("profile,message", [
415+
("jailed", "jailed profile excludes pineforge-sink-finalize file-output mode: "
416+
"finalizer_file_output_no_jailed_route"),
417+
("invalid", "PINEFORGE_RUN_JSON_PROFILE must be unjailed or jailed"),
418+
])
419+
@pytest.mark.parametrize("receipt_channel", ["stdout", "pipe", "stderr"])
420+
@pytest.mark.parametrize("existing_output", [False, True])
421+
def test_host_profile_refusals_keep_text_diagnostics_and_ordinary_receipts(
422+
tmp_path, monkeypatch, profile, message, receipt_channel, existing_output):
423+
monkeypatch.setenv("PINEFORGE_RUN_JSON_PROFILE", profile)
424+
payload = fixture_stream(tmp_path)
425+
output = tmp_path / "out.sink"
426+
previous = b"previous output\n"
427+
if existing_output:
428+
output.write_bytes(previous)
429+
result, receipt, out, spool = invoke(
430+
tmp_path, payload, out=output, receipt_pipe=receipt_channel != "stdout",
431+
extra=("--receipt-fd", "2") if receipt_channel == "stderr" else ())
432+
assert result.returncode == 3, result.stderr
433+
if receipt_channel == "stderr": # fd 2 is silent by design: ordinary receipt only.
434+
assert receipt is None and result.stdout == b""
435+
receipt = json.loads(result.stderr)
436+
assert result.stderr == encoded(receipt)
437+
else:
438+
assert result.stdout == (b"" if receipt_channel == "pipe" else result.receipt_bytes)
439+
assert result.stderr == f"pineforge-sink-finalize: {message}\n".encode()
440+
assert tuple(receipt) == RECEIPT_KEYS
441+
assert "code" not in receipt and "args" not in receipt
442+
assert receipt["complete"] is False and receipt["sha256"] == ""
443+
assert receipt["sink_bytes"] == 0
444+
assert all(receipt["count_" + tag.lower()] == 0 for tag in TAGS)
445+
if existing_output:
446+
assert out.read_bytes() == previous
447+
else:
448+
assert not os.path.lexists(out)
449+
assert list(spool.iterdir()) == []
450+
451+
400452
@pytest.mark.parametrize("tiny,unix_socket,receipt_pipe,launcher", [
401453
(True, False, False, False), (True, True, True, False),
402454
(False, False, True, True),
@@ -779,11 +831,11 @@ def test_receipt_contract_text_lists_every_receipt_shape():
779831
for label, keys in re.findall(r"Keys \(([^)]+)\):\s*([a-z0-9_,\s]+)\.", section)
780832
}
781833
assert described == {
782-
"success": RECEIPT_KEYS,
783-
"ordinary failure": RECEIPT_KEYS,
784-
"exit-8 exhaustion": EXHAUSTED_RECEIPT_KEYS,
785-
"typed refusal with M=2": TYPED_REFUSAL_RECEIPT_KEYS,
834+
"success": RECEIPT_KEYS, "ordinary failure": RECEIPT_KEYS,
835+
"exit-8 exhaustion": EXHAUSTED_RECEIPT_KEYS, "typed refusal with M=2": TYPED_REFUSAL_RECEIPT_KEYS,
786836
}
837+
assert finalizer.__doc__.count("on stderr only when --receipt-fd is not 2") == 2
838+
assert finalizer.__doc__.count("ordinary failure receipt only, indistinguishable from a malformed stream") == 2
787839

788840

789841
@pytest.mark.parametrize("stdout_mode", [False, True])

‎docs/pages/memory-management.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -292,9 +292,9 @@ trailer raises `missing header or trailer`.
292292
and exit status 1
293293
(2 for the refusals above). The separate sink finalizer tool,
294294
`docker/pineforge_sink_finalize.py`, has its own exit statuses: 0 complete and verified, 2 usage error,
295-
3 malformed stream, 4 verification failure, 5 input or output failure, 6 end of input without a
295+
3 malformed stream or host-profile refusal, 4 verification failure, 5 input or output failure, 6 end of input without a
296296
trailer, 7 internal error, 8 `sink_exhausted` (the stream exceeds `--max-bytes`;
297-
`sink_exhausted` at `docker/pineforge_sink_finalize.py:100-109`). Its output is provisional until
297+
`sink_exhausted` at `docker/pineforge_sink_finalize.py:104-113`). Its output is provisional until
298298
exit 0, and a consumer must discard everything it received on exit 3 to 8 (`PROVISIONAL` at
299299
`docker/pineforge_sink_finalize.py:24-26`). The failure catalog maps exit 8 to `report_sink_failed`
300300
with `reason` `sink_exhausted` (`sink_exhausted` at `docker/run_failure_codes.json:1105`). A sink failure carries code `report_sink_failed` and a `reason`

‎docs/pages/scratch-tables.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,25 @@ APP acceptance tests remain **planned**, not established by engine host fakes.
262262

263263
## Jail responsibilities and exclusions
264264

265+
`PINEFORGE_RUN_JSON_PROFILE` accepts exactly `unjailed` (default) or
266+
`jailed` for these tools; neither has a CLI profile override.
267+
268+
`scripts/run_strategy.py` is excluded from jailed execution:
269+
trace-name dedup, Docker feed trim and reverse-CSV staging need writable storage.
270+
It raises the existing typed RuntimeError (exit 1), reason
271+
`strategy_driver_no_jailed_route`, before any native result or output staging.
272+
`pineforge-sink-finalize` file-output mode is excluded before output removal and spool creation: exit 3,
273+
with a text diagnostic naming `finalizer_file_output_no_jailed_route` on stderr only when `--receipt-fd` is not 2 (no typed code or arguments).
274+
With `--receipt-fd 2`, the caller sees exit 3 and the ordinary failure receipt only, indistinguishable from a malformed stream.
275+
The strategy driver uses `run_request_invalid` and `{"option":"arguments"}`;
276+
its reason is in diagnostic text, without new typed argument keys.
277+
Unjailed file mode needs writable output and spool directories.
278+
279+
The finalizer's stdout mode (`--out -` with a positive `--max-bytes`) remains
280+
accepted and writes no file. In valid jailed `docker/run_json.py` runs, the
281+
final JSON spool and output-column/series temporaries are unreachable:
282+
reports stay resident, fd outputs stay in the stream, and path sinks are refused.
283+
265284
**The HOST must configure SIGXFSZ.** APP's `pf-jail.c` must install
266285
`SIG_IGN` for SIGXFSZ **before** applying `RLIMIT_FSIZE`, including the zero
267286
limit, and check that setup succeeds. A successful new inherited-arena attach sets SIGXFSZ to `SIG_IGN` process-wide; detach does not restore it.

‎scripts/run_strategy.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@
5959
starts on TradingView's first computed bar, before that bar -- and the
6060
trades written are those entered inside the tape's span —
6161
_tv_entry_emit_window states the exact rule.
62+
63+
PINEFORGE_RUN_JSON_PROFILE=jailed excludes this driver before any run or
64+
output staging: run_request_invalid, strategy_driver_no_jailed_route.
6265
"""
6366
from __future__ import annotations
6467

@@ -4519,7 +4522,26 @@ def _run_via_docker(strategy_dir: Path, ohlcv_path: Path, params: dict,
45194522

45204523
# --- CLI ---------------------------------------------------------------
45214524

4525+
def _refuse_jailed_profile() -> None:
4526+
"""Read run_json's host selector once, before any driver side effects."""
4527+
profile = os.environ.get("PINEFORGE_RUN_JSON_PROFILE", "unjailed")
4528+
if profile == "jailed":
4529+
error = RuntimeError(
4530+
"jailed profile excludes scripts/run_strategy.py: "
4531+
"strategy_driver_no_jailed_route")
4532+
elif profile != "unjailed":
4533+
error = RuntimeError("PINEFORGE_RUN_JSON_PROFILE must be unjailed or jailed")
4534+
else:
4535+
return
4536+
# Preserve _with_run_failure_code's attributes and the CLI's reporting.
4537+
# Keep the named exclusion in the text: this code's args are closed.
4538+
error.run_failure_code = "run_request_invalid"
4539+
error.run_failure_args = '{"option":"arguments"}'
4540+
raise error
4541+
4542+
45224543
def main() -> int:
4544+
_refuse_jailed_profile()
45234545
ap = argparse.ArgumentParser(description=__doc__,
45244546
formatter_class=argparse.RawDescriptionHelpFormatter)
45254547
ap.add_argument("strategy_dir", type=Path,

‎tests/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7441,6 +7441,14 @@ endif()
74417441

74427442
add_test(NAME test_run_json_sink_fd_skip_accounting COMMAND ${Python3_EXECUTABLE} -B ${PROJECT_SOURCE_DIR}/tests/test_run_json_sink_fd_skip_accounting.py --artifacts ${CMAKE_CURRENT_BINARY_DIR}/pytest-no-skips/test_run_json_sink_fd_skip_accounting)
74437443

7444+
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
7445+
add_test(NAME test_jailed_temp_files_witness
7446+
COMMAND ${Python3_EXECUTABLE} -B ${PROJECT_SOURCE_DIR}/scripts/run_pytest_no_skips.py
7447+
--artifacts ${CMAKE_CURRENT_BINARY_DIR}/pytest-no-skips/test_jailed_temp_files_witness
7448+
-- -q -p no:cacheprovider
7449+
${PROJECT_SOURCE_DIR}/tests/test_jailed_temp_files_witness.py)
7450+
endif()
7451+
74447452
# Property reads and the diagnostic belong before the lane loop;
74457453
# the fence grammar permits target definitions, not get_target_property/message.
74467454
if(PINEFORGE_BUILD_SOURCE_LAYER)

0 commit comments

Comments
 (0)