Skip to content

Commit 8901e83

Browse files
committed
feat: add native live runner with durable webhooks and broker parser plugins
1 parent 6bfe983 commit 8901e83

31 files changed

Lines changed: 4341 additions & 2 deletions

‎.github/workflows/native-live.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: Native live runner
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
push:
7+
branches: [main]
8+
workflow_dispatch:
9+
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
native-live:
15+
runs-on: ubuntu-latest
16+
timeout-minutes: 25
17+
steps:
18+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
19+
with:
20+
persist-credentials: false
21+
- name: Install native build dependencies
22+
run: |
23+
sudo apt-get update
24+
sudo apt-get install -y cmake ninja-build libeigen3-dev libsqlite3-dev libssl-dev zlib1g-dev
25+
- name: Build pinned WebSocket-enabled libcurl
26+
run: |
27+
mkdir -p build-native-deps
28+
cd build-native-deps
29+
curl --fail --location --proto '=https' --tlsv1.2 https://curl.se/download/curl-8.14.1.tar.xz -o curl.tar.xz
30+
echo 'f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd curl.tar.xz' | sha256sum -c -
31+
tar -xf curl.tar.xz
32+
cmake -S curl-8.14.1 -B curl-build -G Ninja \
33+
-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PWD/curl-install" \
34+
-DBUILD_CURL_EXE=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON \
35+
-DBUILD_TESTING=OFF -DCURL_USE_OPENSSL=ON -DENABLE_WEBSOCKETS=ON \
36+
-DHTTP_ONLY=ON -DCURL_USE_LIBPSL=OFF -DUSE_LIBIDN2=OFF \
37+
-DCURL_USE_LIBSSH2=OFF -DCURL_BROTLI=OFF -DCURL_ZSTD=OFF
38+
cmake --build curl-build -j 4
39+
cmake --install curl-build
40+
- name: Build native runner and tests
41+
run: |
42+
cmake -S . -B build-live -G Ninja -DCMAKE_BUILD_TYPE=Release \
43+
-DPINEFORGE_BUILD_LIVE_RUNNER=ON -DPINEFORGE_BUILD_TUTORIAL=OFF \
44+
-DCURL_DIR="$PWD/build-native-deps/curl-install/lib/cmake/CURL"
45+
cmake --build build-live -j 4
46+
- name: Test native engine, parser, journal and transports
47+
run: |
48+
ctest --test-dir build-live --output-on-failure -j 4
49+
# This required lane uses the pinned WS-enabled curl. A skip is not
50+
# sufficient here even though ordinary system-curl builds may skip.
51+
build-live/bin/test_native_live_websocket
52+
python3 scripts/check_c_abi_runtime.py
53+
python3 scripts/check_broker_state_hash_coverage.py
54+
- name: Install native executable
55+
run: |
56+
cmake --install build-live --prefix "$PWD/build-live/install"
57+
build-live/install/bin/pineforge-live --help

‎CMakeLists.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ option(PINEFORGE_BUILD_CORPUS_STRATEGIES
3939
"Build a strategy.so per .pine in corpus/ from its generated.cpp" OFF)
4040
option(PINEFORGE_BUILD_TUTORIAL
4141
"Build the tutorial MACD strategy.so under tutorial/" ON)
42+
option(PINEFORGE_BUILD_LIVE_RUNNER
43+
"Build the native live runner (requires SQLite3, libcurl and OpenSSL)" OFF)
4244
option(PINEFORGE_ENABLE_COVERAGE
4345
"Instrument the runtime + tests for source-level coverage (Clang or GCC)" OFF)
4446
# Opt-in AddressSanitizer + UndefinedBehaviorSanitizer. Off by default
@@ -310,6 +312,10 @@ install(FILES
310312
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/PineForge
311313
)
312314

315+
if(PINEFORGE_BUILD_LIVE_RUNNER)
316+
add_subdirectory(runner)
317+
endif()
318+
313319
# === Summary ===========================================================
314320
message(STATUS "")
315321
message(STATUS "PineForge ${PINEFORGE_VERSION_FULL}")
@@ -319,5 +325,6 @@ message(STATUS " install prefix: ${CMAKE_INSTALL_PREFIX}")
319325
message(STATUS " build tests: ${PINEFORGE_BUILD_TESTS}")
320326
message(STATUS " build tutorial: ${PINEFORGE_BUILD_TUTORIAL}")
321327
message(STATUS " build corpus: ${PINEFORGE_BUILD_CORPUS_STRATEGIES}")
328+
message(STATUS " build live runner: ${PINEFORGE_BUILD_LIVE_RUNNER}")
322329
message(STATUS " build bench strat: ${PINEFORGE_BUILD_BENCH_STRATEGIES}")
323330
message(STATUS "")

‎NOTICE‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,15 @@ Eigen (https://eigen.tuxfamily.org/)
1515
Licensed under the Mozilla Public License 2.0 (MPL-2.0).
1616
SPDX-License-Identifier: MPL-2.0
1717

18+
Optional native live runner dependencies (not part of libpineforge)
19+
-------------------------------------------------------------------
20+
SQLite (https://sqlite.org/): public domain.
21+
libcurl (https://curl.se/): curl license (MIT/X derivative).
22+
OpenSSL (https://www.openssl.org/): Apache License 2.0 for OpenSSL 3.x;
23+
older supported installations retain their applicable upstream terms.
24+
These libraries are linked only by the opt-in native runner. Preserve their
25+
upstream notices when distributing binaries that bundle them.
26+
1827
Optional benchmark harness dependencies (not part of libpineforge)
1928
-------------------------------------------------------------------
2029
The contents under benchmarks/ may install separate Node.js and Python packages

‎README.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,21 @@ Lifecycle-aware compiled modules reset Pine variables, indicator/history buffers
108108
109109
---
110110
111+
## Native live runner
112+
113+
The optional C++17 `pineforge-live` executable uses this engine's native
114+
warmup-to-stream lifecycle. It accepts tick or confirmed 1m OHLCV input,
115+
supports user-defined C++ parsers for broker/provider messages, and commits
116+
inputs plus order-action webhooks to a durable SQLite ledger. Hand-written
117+
C++ and codegen-generated strategies use the same strategy interface.
118+
119+
Build with `-DPINEFORGE_BUILD_LIVE_RUNNER=ON`; the option is off by default,
120+
so core-only users do not acquire SQLite/libcurl/OpenSSL dependencies. See
121+
the [native runner guide](runner/README.md) for feed modes, symbol metadata,
122+
parser ABI, recovery and execution limitations. The existing validation
123+
scoreboard below describes batch backtests; it does not certify new native
124+
live behavior or real broker fills.
125+
111126
## Validation scoreboard
112127
113128
**Round 39 · 2026-09-09:** **4,182 excellent / 8 strong / zero moderate** across all **4,190 scored probes**. This round adds one excellent result, with zero regressions on any canonical metric.
@@ -226,6 +241,7 @@ TradingView ties some day-boundary logic (intraday order caps, session rollovers
226241
| `strategy_set_trace_enabled` | Toggle per-bar trace recording |
227242
| `strategy_set_trade_start_time` | Suppress historical order placement before a time |
228243
| `strategy_stream_begin` / `_push_tick` / `_push_ticks` / `_advance_time` / `_end` / `_fill_report` | Warm on OHLCV, then run realtime on ordered trades |
244+
| `strategy_stream_api_version` / `_push_bar` / `_order_actions_len` / `_order_action_get` / `_order_actions_clear` / `_state_hash` | Native live extension v1: confirmed input bars, physical fill events and observable replay state |
229245
| `strategy_set_chart_timezone` / `strategy_set_syminfo_timezone` / `strategy_set_syminfo_session` | Chart and exchange time |
230246
| `strategy_set_syminfo_mintick` / `_pointvalue` / `_metadata` / `_type` / `_string` | Symbol tick size, point value, numeric metadata, instrument class, string members |
231247
| `strategy_set_native_security_feed` / `strategy_set_aux_security_feed` | Feed `request.security()` from a native higher-timeframe series / an auxiliary bar-aligned feed |

‎include/pineforge/engine.hpp‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,28 @@
3030
// against a runtime providing this hook. This is an internal C++ capability;
3131
// it does not change any public C POD or exported C function signature.
3232
#define PINEFORGE_HAS_SCRIPT_RUN_PREPARE_V1 1
33+
#define PINEFORGE_HAS_NATIVE_LIVE_V1 1
3334

3435
namespace pineforge {
3536

3637
enum class PositionSide { FLAT, LONG, SHORT };
3738

39+
// One physical emulator lot entry/exit, in actual execution order. A reversal
40+
// may produce several exits followed by an entry. No range-end report rows.
41+
struct StreamOrderAction {
42+
uint64_t sequence = 0;
43+
int64_t timestamp_ms = 0;
44+
int32_t bar_index = 0;
45+
bool is_entry = false;
46+
bool is_long = false;
47+
double quantity = 0.0;
48+
double price = 0.0;
49+
std::string order_id;
50+
std::string comment;
51+
uint64_t entry_incarnation = 0;
52+
size_t closed_trade_index = static_cast<size_t>(-1); // engine-private provenance
53+
};
54+
3855
// round 8 family R (OANDA:EURUSD@15, 45 strong probes; campaign notes
3956
// log-20260905t164404z-85800609 (diagnosis), log-20260905t180248z-0dce5ab0
4057
// (entry-leg admission), log-20260905t180249z-10358e84 (margin-call
@@ -3386,6 +3403,11 @@ class BacktestEngine {
33863403
int stream_next_script_bar_index_ = 0;
33873404
bool stream_script_bar_had_tick_ = false;
33883405
bool stream_script_tick_seen_ = false;
3406+
enum class StreamInputMode { UNSET, TICKS, BARS };
3407+
StreamInputMode stream_input_mode_ = StreamInputMode::UNSET;
3408+
bool stream_observe_actions_ = false;
3409+
uint64_t stream_action_sequence_ = 0;
3410+
std::vector<StreamOrderAction> stream_order_actions_;
33893411

33903412
// --- request.security state ---
33913413
struct HistoricalSecurityProjection {
@@ -4531,6 +4553,9 @@ class BacktestEngine {
45314553
bool bar_magnifier,
45324554
int magnifier_samples,
45334555
MagnifierDistribution magnifier_dist);
4556+
void stream_observe_entry(const PyramidEntry& pe);
4557+
void stream_observe_exit(size_t trade_index);
4558+
void stream_refresh_action_metadata(size_t first_action, size_t first_trade);
45344559
bool stream_finalize_until(int64_t timestamp_ms);
45354560
void stream_feed_input_bar(const Bar& bar, bool had_tick);
45364561
void stream_dispatch_script_bar(const Bar& bar, bool had_tick);
@@ -4594,11 +4619,23 @@ class BacktestEngine {
45944619
bool stream_begin(const Bar* warmup_bars, int n_warmup,
45954620
const std::string& input_tf,
45964621
const std::string& script_tf = "");
4622+
// Confirmed input bars use the same OHLC broker kernel as batch. The first
4623+
// tick/bar locks the input mode; mixing them is rejected. Missing in-session
4624+
// bars are rejected rather than padded. Closed-session gaps may be skipped.
4625+
bool stream_push_bar(const Bar& bar);
45974626
bool stream_push_tick(const TradeTick& tick);
45984627
bool stream_push_ticks(const TradeTick* ticks, int n);
45994628
bool stream_advance_time(int64_t timestamp_ms);
46004629
bool stream_end(bool finalize_partial_input_bar = false);
46014630
bool stream_is_realtime() const { return stream_phase_ == StreamPhase::REALTIME; }
4631+
int stream_order_actions_len() const { return static_cast<int>(stream_order_actions_.size()); }
4632+
const StreamOrderAction& stream_order_action_at(int i) const {
4633+
return stream_order_actions_.at(static_cast<size_t>(i));
4634+
}
4635+
void stream_order_actions_clear() { stream_order_actions_.clear(); }
4636+
// Observable broker + stream cursors/forming bars, not a serialization of
4637+
// arbitrary strategy members. Deterministic strategy code is required.
4638+
uint64_t stream_state_hash() const;
46024639

46034640
// Install an effective-time FX curve (account-currency units per one unit
46044641
// of symbol quote currency). Points are copied and must have strictly

‎include/pineforge/live_parser.h‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/* SPDX-License-Identifier: Apache-2.0 */
2+
#ifndef PINEFORGE_LIVE_PARSER_H
3+
#define PINEFORGE_LIVE_PARSER_H
4+
5+
#include <stddef.h>
6+
#include <stdint.h>
7+
8+
#define PF_LIVE_PARSER_ABI_VERSION 1u
9+
#define PF_LIVE_PARSER_MAX_MESSAGE_BYTES (1024u * 1024u)
10+
#define PF_LIVE_PARSER_MAX_EVENTS 1024u
11+
#define PF_LIVE_PARSER_MAX_OUTPUT_BYTES (1024u * 1024u)
12+
13+
#if defined(_WIN32) || defined(__CYGWIN__)
14+
#define PF_LIVE_PARSER_API __declspec(dllexport)
15+
#elif defined(__GNUC__) || defined(__clang__)
16+
#define PF_LIVE_PARSER_API __attribute__((visibility("default")))
17+
#else
18+
#define PF_LIVE_PARSER_API
19+
#endif
20+
21+
#ifdef __cplusplus
22+
extern "C" {
23+
#endif
24+
25+
enum pf_live_parser_event_kind {
26+
PF_LIVE_PARSER_TICK = 1,
27+
PF_LIVE_PARSER_BAR = 2,
28+
PF_LIVE_PARSER_TIME = 3
29+
};
30+
31+
/* ABI v1 is fixed-layout POD; a changed layout requires a new ABI version.
32+
* Initialize the whole struct to zero before assigning its active fields.
33+
* All timestamps are Unix milliseconds and must be nonnegative.
34+
*
35+
* TICK: timestamp, strictly positive source sequence, price and quantity.
36+
* BAR: timestamp is a confirmed one-minute bar's open, aligned to 60000ms;
37+
* open/high/low/close are positive finite prices; volume is finite >= 0.
38+
* TIME: timestamp is the provider's explicit completeness boundary, declaring
39+
* that no earlier ticks remain. A wall clock/heartbeat is not sufficient.
40+
*
41+
* reserved must be zero. Inactive fields are ignored and normalized to zero by
42+
* the host. Sequence continuity, mode and stream ordering are checked by the
43+
* runner across messages; parsers must preserve source sequence, never invent
44+
* a mutable counter. BAR/TIME do not use sequence.
45+
*/
46+
typedef struct pf_live_parser_event_v1 {
47+
uint32_t kind;
48+
uint32_t reserved;
49+
int64_t timestamp;
50+
uint64_t sequence;
51+
double open;
52+
double high;
53+
double low;
54+
double close;
55+
double volume;
56+
double price;
57+
double quantity;
58+
} pf_live_parser_event_v1_t;
59+
60+
/* The event pointer is borrowed only until this callback returns. The callback
61+
* copies it synchronously and returns 0 on acceptance or -1 on failure. A plugin
62+
* MUST stop and propagate a callback failure; it must not retain callback/user
63+
* pointers, call concurrently, or emit after parse_message returns.
64+
*/
65+
typedef int (*pf_live_parser_emit_v1_fn)(
66+
const pf_live_parser_event_v1_t* event, void* user);
67+
68+
PF_LIVE_PARSER_API uint32_t pf_live_parser_abi_version(void);
69+
70+
/* Parse ONE complete provider message into zero or more ordered events.
71+
* message/config_json are length-delimited borrowed bytes, not NUL-terminated
72+
* strings. config_json is an immutable JSON object; it remains in host memory
73+
* and must not be logged or copied into normalized events. Heartbeats/control
74+
* messages may succeed with zero events. Malformed input MUST fail.
75+
*
76+
* Return 0 on success, -1 on any failure. Never throw across this C ABI. The
77+
* host stages events until success, so a failed message emits no partial output
78+
* to the strategy or journal. Limits: 1MiB message, 1024 events, 1MiB POD output.
79+
* With the v1 event size the event-count bound is the tighter output bound.
80+
*
81+
* REQUIRED: stateless and deterministic for identical (message, config) bytes.
82+
* No I/O, clock, random values, mutable globals or retained state may influence
83+
* output. The journal stores canonical normalized events, not parser state.
84+
* Providers requiring cross-message assembly/state need an upstream adapter.
85+
* Plugins are trusted native executable code, not sandboxed. The host cannot
86+
* enforce this contract against malicious code or recover from native crashes.
87+
*/
88+
PF_LIVE_PARSER_API int pf_live_parse_message(
89+
const char* message, size_t message_size,
90+
const char* config_json, size_t config_size,
91+
pf_live_parser_emit_v1_fn emit, void* user);
92+
93+
#ifdef __cplusplus
94+
} /* extern "C" */
95+
#endif
96+
#endif /* PINEFORGE_LIVE_PARSER_H */

‎include/pineforge/pineforge.h‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -589,13 +589,62 @@ PF_API uint64_t strategy_closed_trade_entry_incarnation(
589589
* lifecycle uses close-only strategy calculation (the Pine strategy default)
590590
* while resting broker orders are evaluated on every normalized trade.
591591
*
592+
* calc_on_order_fills, historical probe/tail overrides, timestamped FX,
593+
* auxiliary and native security feeds are rejected. No every-tick strategy
594+
* callback is provided; hand-written strategies follow the same lifecycle.
592595
* @return 0 on success, -1 on failure. Inspect #strategy_get_last_error. */
593596
PF_API int strategy_stream_begin(pf_strategy_t s,
594597
const pf_bar_t* warmup_bars,
595598
int n_warmup,
596599
const char* input_tf,
597600
const char* script_tf);
598601

602+
/** Native live extension version (1). Additive to ABI v4; callers must probe
603+
* this symbol before using the confirmed-bar/action API in older modules. */
604+
PF_API int strategy_stream_api_version(void);
605+
606+
/** One physical emulator lot action. is_long describes the position side:
607+
* buy = is_entry == is_long. A reversal is exits followed by an entry.
608+
* price/time are emulator reference values, not external broker fills.
609+
* Strings are borrowed until the next stream mutation or queue clear. */
610+
typedef struct pf_stream_order_action {
611+
uint64_t sequence;
612+
int64_t timestamp_ms;
613+
int32_t bar_index;
614+
int32_t is_entry;
615+
int32_t is_long;
616+
double quantity;
617+
double price;
618+
const char* order_id;
619+
const char* comment;
620+
uint64_t entry_incarnation;
621+
} pf_stream_order_action_t;
622+
623+
/** Consume one confirmed input-timeframe bar. Its script-timeframe aggregate
624+
* uses the existing batch OHLC fill kernel when complete. Close-only strategy
625+
* calculation; no bar magnifier or synthetic trade ticks. The first tick/bar
626+
* locks the feed mode. Off-grid/out-of-order data, invalid OHLCV and missing
627+
* in-session bars fail. Calendar-closed intervals may be skipped. No padding.
628+
* Returns 0/-1. On ANY input-processing failure discard/recover the instance:
629+
* the operation is not transactional and can have advanced native state. */
630+
PF_API int strategy_stream_push_bar(pf_strategy_t s, const pf_bar_t* bar);
631+
632+
/** Queued physical fills since the last clear, in execution order. Historical
633+
* warmup and report-only range-end rows never enqueue. Sequence starts at 1
634+
* after warmup and is not reset by clear. Returns -1 on a null handle. */
635+
PF_API int strategy_stream_order_actions_len(pf_strategy_t s);
636+
/** Copy one action; return 0 on success, -1 on invalid handle/index/output. */
637+
PF_API int strategy_stream_order_action_get(pf_strategy_t s, int index,
638+
pf_stream_order_action_t* out);
639+
/** Clear observed events after the caller durably journals them. */
640+
PF_API void strategy_stream_order_actions_clear(pf_strategy_t s);
641+
/** Versioned deterministic fingerprint of observable broker/stream state.
642+
* Excludes the consumable queue and arbitrary private strategy members.
643+
* This is a replay check, not a complete state snapshot or cryptographic hash.
644+
* Fresh replay must use deterministic strategy code and pinned configuration.
645+
* Returns 0 for NULL. */
646+
PF_API uint64_t strategy_stream_state_hash(pf_strategy_t s);
647+
599648
/** Push one normalized realtime trade. Returns 0 on success, -1 on failure. */
600649
PF_API int strategy_stream_push_tick(pf_strategy_t s,
601650
const pf_trade_tick_t* tick);

0 commit comments

Comments
 (0)