Skip to content

Commit d21ea8d

Browse files
luisleo526claude
andcommitted
TradeLedger: per-field accessors and a 65,536-row default read cache for closedtrades loops
Round 3 cached decoded rows, but hungpixi (24,238 closed trades, a loop over all of them each bar) thrashed the 4096-row cache and every Pine accessor still built a whole Trade by value to read one field. Now: - TradeLedger::<field>_at(i) returns one field straight from the cached record (numeric: no Trade, no string; string: only the requested packed bytes), with the same checked-row contract as at(); the closed_trade_* accessors in engine.hpp use them (values and error behaviour unchanged, percent formulas unchanged). - Default budget 65,536 rows (max 1,048,576); entry = 168-byte record + 16-byte key/generation + 128 inline string bytes = 312 bytes (20,447,232 B at the default). A row whose four strings exceed 128 bytes keeps its numeric fields cached and reads strings from scratch. - The cache array grows lazily to min(budget, next power of two >= max(64, rows)), so a run with few trades allocates a few KiB, and memory is bounded by the budget independent of bars. - seal() no longer invalidates the cache: publication changes no row content and the checked accessors refuse sealed rows before reaching the cache; otherwise every published trade would make the next loop cold again. Tests: hungpixi-shaped 24,238-row loop (zero reads/writes after warm-up, oracle-compared), allocation check for numeric hits, lazy growth boundaries, wide-string fallback, replace/rewind/move invalidation, sealed-row refusal for every new accessor. Refusal still uses setting_rejected until the shared resident-budget code lands. Edit-only lane: nothing built or run at commit time; lab verification follows. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
1 parent 6548775 commit d21ea8d

6 files changed

Lines changed: 473 additions & 109 deletions

File tree

‎include/pineforge/engine.hpp‎

Lines changed: 22 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1605,94 +1605,92 @@ class BacktestEngine {
16051605
// --- Trade history accessors (for strategy.closedtrades.*) ---
16061606
double closed_trade_profit(int index) const {
16071607
if (index >= 0 && index < (int)trades_.size())
1608-
return trades_[index].pnl;
1608+
return trades_.pnl_at(index);
16091609
return 0.0;
16101610
}
16111611
double closed_trade_profit_percent(int idx) const {
16121612
if (idx < 0 || idx >= (int)trades_.size()) return std::numeric_limits<double>::quiet_NaN();
1613-
return trades_[idx].pnl_pct;
1613+
return trades_.pnl_pct_at(idx);
16141614
}
16151615
double closed_trade_commission(int idx) const {
16161616
if (idx < 0 || idx >= (int)trades_.size()) return std::numeric_limits<double>::quiet_NaN();
1617-
return trades_[idx].commission;
1617+
return trades_.commission_at(idx);
16181618
}
16191619
int closed_trade_entry_bar_index(int idx) const {
16201620
if (idx < 0 || idx >= (int)trades_.size()) return na<int>();
1621-
return trades_[idx].entry_bar_index;
1621+
return trades_.entry_bar_index_at(idx);
16221622
}
16231623
int closed_trade_exit_bar_index(int idx) const {
16241624
if (idx < 0 || idx >= (int)trades_.size()) return na<int>();
1625-
return trades_[idx].exit_bar_index;
1625+
return trades_.exit_bar_index_at(idx);
16261626
}
16271627
std::string closed_trade_entry_comment(int idx) const {
16281628
if (idx < 0 || idx >= (int)trades_.size()) return std::string();
1629-
return trades_[idx].entry_comment;
1629+
return trades_.entry_comment_at(idx);
16301630
}
16311631
std::string closed_trade_exit_comment(int idx) const {
16321632
if (idx < 0 || idx >= (int)trades_.size()) return std::string();
1633-
return trades_[idx].exit_comment;
1633+
return trades_.exit_comment_at(idx);
16341634
}
16351635
std::string closed_trade_entry_id(int idx) const {
16361636
if (idx < 0 || idx >= (int)trades_.size()) return std::string();
1637-
return trades_[idx].entry_id;
1637+
return trades_.entry_id_at(idx);
16381638
}
16391639
std::string closed_trade_exit_id(int idx) const {
16401640
if (idx < 0 || idx >= (int)trades_.size()) return std::string();
1641-
return trades_[idx].exit_id;
1641+
return trades_.exit_id_at(idx);
16421642
}
16431643
uint64_t closed_trade_entry_incarnation(int idx) const {
16441644
if (idx < 0 || idx >= (int)trades_.size()) return 0;
1645-
return trades_[idx].entry_incarnation;
1645+
return trades_.entry_incarnation_at(idx);
16461646
}
16471647
double closed_trade_entry_price(int idx) const {
16481648
if (idx < 0 || idx >= (int)trades_.size()) return std::numeric_limits<double>::quiet_NaN();
1649-
return trades_[idx].entry_price;
1649+
return trades_.entry_price_at(idx);
16501650
}
16511651
double closed_trade_exit_price(int idx) const {
16521652
if (idx < 0 || idx >= (int)trades_.size()) return std::numeric_limits<double>::quiet_NaN();
1653-
return trades_[idx].exit_price;
1653+
return trades_.exit_price_at(idx);
16541654
}
16551655
int64_t closed_trade_entry_time(int idx) const {
16561656
if (idx < 0 || idx >= (int)trades_.size()) return 0;
1657-
return trades_[idx].entry_time;
1657+
return trades_.entry_time_at(idx);
16581658
}
16591659
int64_t closed_trade_exit_time(int idx) const {
16601660
if (idx < 0 || idx >= (int)trades_.size()) return 0;
1661-
return trades_[idx].exit_time;
1661+
return trades_.exit_time_at(idx);
16621662
}
16631663
double closed_trade_size(int idx) const {
16641664
if (idx < 0 || idx >= (int)trades_.size()) return std::numeric_limits<double>::quiet_NaN();
1665-
return trades_[idx].qty;
1665+
return trades_.qty_at(idx);
16661666
}
16671667
double closed_trade_max_runup(int idx) const {
16681668
if (idx < 0 || idx >= (int)trades_.size()) return 0.0;
1669-
return trades_[idx].max_runup;
1669+
return trades_.max_runup_at(idx);
16701670
}
16711671
// Percent excursions: trade.max_runup / max_drawdown are stored in
16721672
// account currency (× pointvalue, see build_close_trade_with_costs), so the entry
16731673
// cost denominator must be in currency too (entry × qty × pointvalue).
16741674
// pointvalue=1 cancels out and matches the legacy ratio bit-for-bit.
16751675
double closed_trade_max_runup_percent(int idx) const {
16761676
if (idx < 0 || idx >= (int)trades_.size()) return 0.0;
1677-
const Trade& t = trades_[idx];
1678-
double cost = t.entry_price * t.qty * syminfo_.pointvalue;
1679-
return (cost > 0.0) ? (t.max_runup / cost) * 100.0 : 0.0;
1677+
double cost = trades_.entry_price_at(idx) * trades_.qty_at(idx) * syminfo_.pointvalue;
1678+
return (cost > 0.0) ? (trades_.max_runup_at(idx) / cost) * 100.0 : 0.0;
16801679
}
16811680
double closed_trade_max_drawdown(int idx) const {
16821681
if (idx < 0 || idx >= (int)trades_.size()) return 0.0;
1683-
return trades_[idx].max_drawdown;
1682+
return trades_.max_drawdown_at(idx);
16841683
}
16851684
double closed_trade_max_drawdown_percent(int idx) const {
16861685
if (idx < 0 || idx >= (int)trades_.size()) return 0.0;
1687-
const Trade& t = trades_[idx];
1688-
double cost = t.entry_price * t.qty * syminfo_.pointvalue;
1689-
return (cost > 0.0) ? (t.max_drawdown / cost) * 100.0 : 0.0;
1686+
double cost = trades_.entry_price_at(idx) * trades_.qty_at(idx) * syminfo_.pointvalue;
1687+
return (cost > 0.0) ? (trades_.max_drawdown_at(idx) / cost) * 100.0 : 0.0;
16901688
}
16911689

16921690
// --- Direction accessors ---
16931691
std::string closed_trade_direction(int idx) const {
16941692
if (idx < 0 || idx >= (int)trades_.size()) return "";
1695-
return trades_[idx].is_long ? "long" : "short";
1693+
return trades_.is_long_at(idx) ? "long" : "short";
16961694
}
16971695
std::string open_trade_direction(int idx) const {
16981696
if (position_side_ == PositionSide::FLAT || idx < 0 || idx >= (int)pyramid_entries_.size()) return "";

‎include/pineforge/trade_ledger.hpp‎

Lines changed: 41 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#pragma once
22

33
#include <cstddef>
4+
#include <cstdint>
45
#include <functional>
56
#include <iterator>
67
#include <memory>
@@ -19,7 +20,8 @@ inline namespace engine_script_run_v19 {
1920
// the string payload stay in memory. All four string payloads and all older
2021
// mutable/sealed descriptors use the existing run-owned positional scratch
2122
// files. A separate, budgeted read cache retains numeric records and up to
22-
// kCacheStringBytes of decoded string bytes per row. Wider rows use scratch.
23+
// kCacheStringBytes of decoded string bytes per row. Wider rows retain their
24+
// numeric records in the cache and use scratch only for string reads.
2325
// Owning Trade values still belong to readers.
2426
//
2527
// Reads own their values. No reference or contiguous pointer into an
@@ -36,13 +38,15 @@ class TradeLedger {
3638
static constexpr size_type kResidentRows = 64;
3739
// One knob for this cache kind: ledger_rows. Each entry contains a
3840
// 168-byte numeric/extent record, a global key and validity generation,
39-
// and exactly 1,024 inline string bytes (all four strings combined).
41+
// and exactly 128 inline string bytes (all four strings combined).
42+
// The string-validity flag shares the generation word. On 64-bit targets:
43+
// 168 + 16 + 128 = 312 bytes, with no additional alignment padding.
4044
// No per-entry heap allocation or growing offset/string directory.
41-
static constexpr size_type kCacheStringBytes = 1024;
45+
static constexpr size_type kCacheStringBytes = 128;
4246
static constexpr size_type kLedgerEntryBytes =
4347
kRecordBytes + 2 * sizeof(size_type) + kCacheStringBytes;
44-
static constexpr size_type kDefaultCacheRows = 4096;
45-
static constexpr size_type kMaxCacheRows = 65536;
48+
static constexpr size_type kDefaultCacheRows = 65536;
49+
static constexpr size_type kMaxCacheRows = 1048576;
4650
using Visitor = std::function<bool(size_type, const Trade&)>;
4751

4852
class const_iterator {
@@ -80,17 +84,21 @@ class TradeLedger {
8084
void set_scratch_directory(const std::string&);
8185
const std::string& scratch_directory() const;
8286
// Configure before the first append/spill; clear() preserves the budget.
83-
// Valid rows: [1,65536]; invalid/late settings throw a coded
87+
// Valid rows: [1,1048576]; invalid/late settings throw a coded
8488
// std::invalid_argument (setting_rejected, reason value_below_minimum,
85-
// value_above_maximum or frozen). Default: 4,096 rows, 4,947,968 entry
86-
// bytes on 64-bit targets; maximum: 65,536 rows, 79,167,488 entry bytes.
89+
// value_above_maximum or frozen). Default: 65,536 rows, 20,447,232 entry
90+
// bytes on 64-bit targets; maximum: 1,048,576 rows, 327,155,712 entry bytes.
8791
void set_cache_rows(size_type rows);
8892
size_type cache_rows() const;
8993
// Accounted ledger/cache storage: rows*kLedgerEntryBytes + a fixed base
9094
// (sizeof ledger + State). Excludes allocator overhead, scratch/OS pages,
9195
// configured directory strings, returned owning Trades and the explicit
9296
// legacy materialize() map. Validates the same row range as the setter.
9397
static size_type resident_cache_bytes(size_type rows);
98+
// Actual allocation: fixed base plus allocated capacity*kLedgerEntryBytes.
99+
// Allocated on the first read, at min(cache_rows(), the next power of two
100+
// >= max(64,size())). Further growth invalidates entries; capacity stays
101+
// bounded by cache_rows() and independent of the number of bars.
94102
size_type cache_resident_bytes() const;
95103
// Occupied decoded entries only; separate from resident_size()'s legacy
96104
// descriptor/materialization accounting.
@@ -108,17 +116,38 @@ class TradeLedger {
108116
// read() spans the whole exact source; at()/[] retain the checked
109117
// unsealed-window contract for existing settlement callers.
110118
// Hits allocate only the owning strings already required by read().
111-
// Direct-mapped eviction above cache_rows(), or an oversized string set,
112-
// retains the existing scratch path; a history loop can still cost O(T).
119+
// Direct-mapped eviction above cache_rows(), or oversized strings, retains
120+
// the existing scratch path; a history loop can still cost O(T).
113121
Trade read(size_type global) const;
114122
Trade operator[](size_type global) const;
115123
Trade at(size_type global) const;
116124
Trade back() const;
117125
// Same checked unsealed-row contract as at(). numeric_at() returns every
118-
// numeric field and flag with empty strings, reading only the descriptor;
119-
// entry_id_at() reads just that one string when a predicate needs it.
126+
// numeric field and flag with empty strings, reading only the descriptor.
127+
// entry_id_at() returns only that string and shares the field read cache.
120128
Trade numeric_at(size_type global) const;
121129
std::string entry_id_at(size_type global) const;
130+
// Pine field reads use the same check(global,true) as at(). A numeric
131+
// hit loads just its Record member, without constructing/copying a Trade
132+
// or any string. A string hit copies only the requested packed bytes.
133+
// Misses retain the numeric record and all strings fitting the allowance.
134+
double pnl_at(size_type global) const;
135+
double pnl_pct_at(size_type global) const;
136+
double commission_at(size_type global) const;
137+
std::int32_t entry_bar_index_at(size_type global) const;
138+
std::int32_t exit_bar_index_at(size_type global) const;
139+
std::uint64_t entry_incarnation_at(size_type global) const;
140+
double entry_price_at(size_type global) const;
141+
double exit_price_at(size_type global) const;
142+
std::int64_t entry_time_at(size_type global) const;
143+
std::int64_t exit_time_at(size_type global) const;
144+
double qty_at(size_type global) const;
145+
double max_runup_at(size_type global) const;
146+
double max_drawdown_at(size_type global) const;
147+
bool is_long_at(size_type global) const;
148+
std::string entry_comment_at(size_type global) const;
149+
std::string exit_comment_at(size_type global) const;
150+
std::string exit_id_at(size_type global) const;
122151
void replace(size_type global, const Trade&);
123152
void modify(size_type global, const std::function<void(Trade&)>&);
124153

0 commit comments

Comments
 (0)